mvv/personal-finances
1
0
Fork 0
mirror of https://github.com/mvvasilev/personal-finances.git synced 2025-04-19 14:19:52 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Bank statement upload/parsing

Browse source
This commit is contained in:
Miroslav Vasilev 2023-12-15 20:04:57 +02:00
parent da3f2f6169
commit 6e190d09f0
35 changed files with 675 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
.idea/dataSources.xml generated Normal file
View file

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourceManagerImpl" format="xml" multifile-model="true">
<data-source source="LOCAL" name="finances@localhost" uuid="fa2f05d4-8222-487a-b3c3-3e6fa0c7164c">
<driver-ref>postgresql</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.postgresql.Driver</jdbc-driver>
<jdbc-url>jdbc:postgresql://localhost:5432/finances</jdbc-url>
<jdbc-additional-properties>
<property name="com.intellij.clouds.kubernetes.db.host.port" />
<property name="com.intellij.clouds.kubernetes.db.enabled" value="false" />
<property name="com.intellij.clouds.kubernetes.db.resource.type" value="Deployment" />
<property name="com.intellij.clouds.kubernetes.db.container.port" />
</jdbc-additional-properties>
<working-dir>$ProjectFileDir$</working-dir>
</data-source>
</component>
</project>

1
.idea/gradle.xml generated
View file

@ -10,6 +10,7 @@
<set>
<option value="$PROJECT_DIR$" />
<option value="$PROJECT_DIR$/APIGateway" />
<option value="$PROJECT_DIR$/Common" />
<option value="$PROJECT_DIR$/PersonalFinancesService" />
</set>
</option>

2
APIGateway/.env.example
View file

@ -1,3 +1,5 @@
PROFILE= production/development
AUTHENTIK_CLIENT_ID= authentik oauth2 client id
AUTHENTIK_CLIENT_SECRET= authentik oauth2 client secret
AUTHENTIK_ISSUER_URL= authentik issuer url ( dev: https://auth.mvvasilev.dev/application/o/personal-finances/ )

3
APIGateway/src/main/resources/application.yml
View file

@ -3,6 +3,8 @@ logging:
web: trace
core: trace
spring:
profiles:
active: ${PROFILE}
data:
redis:
host: ${REDIS_HOST}
@ -30,6 +32,7 @@ spring:
predicates:
- Path=/api/**
filters:
- RewritePath=/api/(?<segment>.*), /$\{segment}
- TokenRelay=
- id: spa
order: 10

21
Common/build.gradle Normal file
View file

@ -0,0 +1,21 @@
plugins {
id 'java'
}
group = 'dev.mvvasilev'
version = '1.0-SNAPSHOT'
repositories {
mavenCentral()
}
dependencies {
implementation 'jakarta.persistence:jakarta.persistence-api:3.1.0'
testImplementation platform('org.junit:junit-bom:5.9.1')
testImplementation 'org.junit.jupiter:junit-jupiter'
}
test {
useJUnitPlatform()
}

22
Common/src/main/java/dev/mvvasilev/common/controller/AbstractRestController.java Normal file
View file

@ -0,0 +1,22 @@
package dev.mvvasilev.common.controller;
import dev.mvvasilev.common.web.APIErrorDTO;
import dev.mvvasilev.common.web.APIResponseDTO;
import java.util.List;
public class AbstractRestController {
protected <T> APIResponseDTO<T> withStatus(int statusCode, String statusText, T body) {
return new APIResponseDTO<>(body, null, statusCode, statusText);
}
protected <T> APIResponseDTO<T> withSingleError(int statusCode, String statusText, String errorMessage, String errorCode, String stacktrace) {
return new APIResponseDTO<>(null, List.of(new APIErrorDTO(errorMessage, errorCode, stacktrace)), statusCode, statusText);
}
protected <T> APIResponseDTO<T> ok(T body) {
return withStatus(200, "ok", body);
}
}

54
Common/src/main/java/dev/mvvasilev/common/data/AbstractEntity.java Normal file
View file

@ -0,0 +1,54 @@
package dev.mvvasilev.common.data;
import jakarta.persistence.*;
import java.time.LocalDateTime;
@MappedSuperclass
public abstract class AbstractEntity implements DatabaseStorable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(columnDefinition = "bigserial")
private Long id;
@Column(name = "time_created", insertable = false, updatable = false)
private LocalDateTime timeCreated;
@Column(name = "time_last_modified", insertable = false)
private LocalDateTime timeLastModified;
protected AbstractEntity() {}
@Override
public long getId() {
return id;
}
@Override
public LocalDateTime getTimeCreated() {
return timeCreated;
}
@Override
public LocalDateTime getTimeLastModified() {
return timeLastModified;
}
public void setId(Long id) {
this.id = id;
}
public void setTimeCreated(LocalDateTime timeCreated) {
this.timeCreated = timeCreated;
}
public void setTimeLastModified(LocalDateTime timeLastModified) {
this.timeLastModified = timeLastModified;
}
@PreUpdate
protected void onUpdate() {
this.timeLastModified = LocalDateTime.now();
}
}

4
Common/src/main/java/dev/mvvasilev/common/data/DataNamingStrategy.java Normal file
View file

@ -0,0 +1,4 @@
package dev.mvvasilev.common.data;
public class DataNamingStrategy {
}

13
Common/src/main/java/dev/mvvasilev/common/data/DatabaseStorable.java Normal file
View file

@ -0,0 +1,13 @@
package dev.mvvasilev.common.data;
import java.time.LocalDateTime;
public interface DatabaseStorable {
long getId();
LocalDateTime getTimeCreated();
LocalDateTime getTimeLastModified();
}

4
Common/src/main/java/dev/mvvasilev/common/exceptions/CommonFinancesException.java Normal file
View file

@ -0,0 +1,4 @@
package dev.mvvasilev.common.exceptions;
public class CommonFinancesException extends RuntimeException {
}

4
Common/src/main/java/dev/mvvasilev/common/exceptions/InvalidUserIdException.java Normal file
View file

@ -0,0 +1,4 @@
package dev.mvvasilev.common.exceptions;
public class InvalidUserIdException extends CommonFinancesException {
}

3
Common/src/main/java/dev/mvvasilev/common/web/APIErrorDTO.java Normal file
View file

@ -0,0 +1,3 @@
package dev.mvvasilev.common.web;
public record APIErrorDTO(String message, String errorCode, String stacktrace) { }

5
Common/src/main/java/dev/mvvasilev/common/web/APIResponseDTO.java Normal file
View file

@ -0,0 +1,5 @@
package dev.mvvasilev.common.web;
import java.util.Collection;
public record APIResponseDTO<T>(T result, Collection<APIErrorDTO> errors, int statusCode, String statusText) { }

4
PersonalFinancesService/.env.example
View file

@ -1,5 +1,7 @@
PROFILE= production/development
AUTHENTIK_ISSUER_URL= auth server configuration url for fetching JWKs ( dev: https://auth.mvvasilev.dev/application/o/personal-finances/ )
DATASOURCE_URL= database jdbc url ( postgres only, example: jdbc:postgresql://localhost:45093/mydatabase )
DATASOURCE_URL= database jdbc url ( postgres only, example: jdbc:postgresql://localhost:5432/mydatabase )
DATASOURCE_USER= database user
DATASOURCE_PASSWORD= database password

11
PersonalFinancesService/build.gradle
View file

@ -26,10 +26,19 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-oauth2-resource-server'
implementation 'org.springframework.boot:spring-boot-starter-security'
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.flywaydb:flyway-core'
implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.2.0'
implementation 'org.springdoc:springdoc-openapi-starter-common:2.3.0'
implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.3.0'
implementation 'org.apache.poi:poi:5.2.5'
implementation 'org.apache.poi:poi-ooxml:5.2.5'
implementation project(":Common")
runtimeOnly 'com.h2database:h2'
runtimeOnly 'org.postgresql:postgresql'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation 'org.springframework.security:spring-security-test'
}

42
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/advice/APIResponseAdvice.java Normal file
View file

@ -0,0 +1,42 @@
package dev.mvvasilev.finances.advice;
import dev.mvvasilev.common.web.APIErrorDTO;
import dev.mvvasilev.common.web.APIResponseDTO;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.*;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import java.util.List;
@RestControllerAdvice(basePackages = {"dev.mvvasilev"})
public class APIResponseAdvice {
@Value("${debug}")
private boolean isDebug;
private Logger logger = LoggerFactory.getLogger(this.getClass());
@ExceptionHandler(Exception.class)
public APIResponseDTO<Object> processGenericException(Exception ex) {
List<APIErrorDTO> errors = List.of(
new APIErrorDTO(
ex.getMessage(),
isDebug ? ex.getClass().getCanonicalName() : null,
isDebug ? ExceptionUtils.getStackTrace(ex) : null
)
);
logger.error("Exception", ex);
return new APIResponseDTO<>(
null,
errors,
HttpStatus.INTERNAL_SERVER_ERROR.value(),
HttpStatus.INTERNAL_SERVER_ERROR.getReasonPhrase()
);
}
}

19
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/configuration/SecurityConfiguration.java
View file

@ -11,6 +11,7 @@ import org.springframework.security.oauth2.jwt.JwtDecoders;
import org.springframework.security.oauth2.server.resource.web.BearerTokenResolver;
import org.springframework.security.oauth2.server.resource.web.DefaultBearerTokenResolver;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.web.filter.CommonsRequestLoggingFilter;
@Configuration
public class SecurityConfiguration {
@ -18,20 +19,32 @@ public class SecurityConfiguration {
@Value("${jwt.issuer-url}")
public String jwtIssuerUrl;
private static final String[] SWAGGER_URLS = {
private static final String[] WHITELISTED_URLS = {
"/v3/api-docs/**",
"/swagger-ui/**",
"/v2/api-docs/**",
"/swagger-resources/**"
"/swagger-resources/**",
"/actuator/**"
};
@Bean
public CommonsRequestLoggingFilter requestLoggingFilter() {
CommonsRequestLoggingFilter loggingFilter = new CommonsRequestLoggingFilter();
loggingFilter.setIncludeClientInfo(true);
loggingFilter.setIncludeHeaders(true);
loggingFilter.setIncludeQueryString(true);
loggingFilter.setIncludePayload(true);
loggingFilter.setMaxPayloadLength(64000);
return loggingFilter;
}
@Bean
public SecurityFilterChain securityFilterChain(HttpSecurity httpSecurity, BearerTokenResolver bearerTokenResolver) throws Exception {
return httpSecurity
.cors(c -> c.disable()) // won't be needing cors, as the API will be completely hidden behind an api gateway
.csrf(c -> c.disable())
.authorizeHttpRequests(c -> {
c.requestMatchers(SWAGGER_URLS).permitAll();
c.requestMatchers(WHITELISTED_URLS).permitAll();
c.anyRequest().authenticated();
})
.oauth2ResourceServer(c -> {

24
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/configuration/SwaggerConfiguration.java
View file

@ -7,25 +7,25 @@ import io.swagger.v3.oas.models.security.SecurityScheme;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.List;
@Configuration
public class SwaggerConfiguration {
@Bean
public OpenAPI customizeOpenAPI() {
final String securitySchemeName = "Bearer";
final String securitySchemeName = "bearerAuth";
return new OpenAPI()
.addSecurityItem(new SecurityRequirement().addList(securitySchemeName))
.components(
new Components().addSecuritySchemes(
securitySchemeName,
new SecurityScheme()
.name(securitySchemeName)
.type(SecurityScheme.Type.APIKEY)
.scheme("bearer")
.bearerFormat("JWT")
)
);
new Components()
.addSecuritySchemes(securitySchemeName,
new SecurityScheme()
.type(SecurityScheme.Type.HTTP)
.scheme("bearer")
.bearerFormat("JWT")
)
)
.security(List.of(new SecurityRequirement().addList(securitySchemeName)));
}
}

36
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/controllers/ApiController.java
View file

@ -1,36 +0,0 @@
package dev.mvvasilev.finances.controllers;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.oauth2.jwt.JwtClaimNames;
import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.Optional;
@RestController
public class ApiController {
Logger logger = LoggerFactory.getLogger(ApiController.class);
private final ObjectMapper objectMapper;
@Autowired
public ApiController(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
@PostMapping("/api/user-info")
public ResponseEntity<Authentication> userInfo(JwtAuthenticationToken authenticationToken) {
logger.info(authenticationToken.getToken().getClaimAsString(JwtClaimNames.SUB));
return ResponseEntity.of(Optional.of(SecurityContextHolder.getContext().getAuthentication()));
}
}

18
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/controllers/DebugController.java Normal file
View file

@ -0,0 +1,18 @@
package dev.mvvasilev.finances.controllers;
import dev.mvvasilev.common.controller.AbstractRestController;
import dev.mvvasilev.common.web.APIResponseDTO;
import org.springframework.context.annotation.Profile;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RestController;
@RestController
@Profile("development")
public class DebugController extends AbstractRestController {
@GetMapping("/token")
public ResponseEntity<APIResponseDTO<String>> fetchToken(@RequestHeader("Authorization") String authHeader) {
return ResponseEntity.ofNullable(ok(authHeader));
}
}

36
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/controllers/TransactionsController.java Normal file
View file

@ -0,0 +1,36 @@
package dev.mvvasilev.finances.controllers;
import dev.mvvasilev.common.controller.AbstractRestController;
import dev.mvvasilev.common.web.APIResponseDTO;
import dev.mvvasilev.finances.services.TransactionsService;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
@RestController("/transactions")
public class TransactionsController extends AbstractRestController {
private TransactionsService transactionsService;
@Autowired
public TransactionsController(TransactionsService transactionsService) {
this.transactionsService = transactionsService;
}
@PostMapping(value = "/transactions/uploadSheet", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<APIResponseDTO<Integer>> uploadTransactions(@RequestParam("file") MultipartFile file, Authentication authentication) throws IOException {
transactionsService.uploadMultipleTransactionsFromExcelSheetForUser(
file.getInputStream(),
authentication.getName()
);
return ResponseEntity.ofNullable(ok(1));
}
}

23
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/entity/RawStatement.java Normal file
View file

@ -0,0 +1,23 @@
package dev.mvvasilev.finances.entity;
import dev.mvvasilev.common.data.AbstractEntity;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
@Entity
@Table(schema = "transactions")
public class RawStatement extends AbstractEntity {
private Integer userId;
public RawStatement() {
}
public Integer getUserId() {
return userId;
}
public void setUserId(Integer userId) {
this.userId = userId;
}
}

64
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/entity/RawTransactionValue.java Normal file
View file

@ -0,0 +1,64 @@
package dev.mvvasilev.finances.entity;
import dev.mvvasilev.common.data.AbstractEntity;
import jakarta.persistence.*;
import java.time.LocalDateTime;
@Entity
@Table(schema = "transactions")
public class RawTransactionValue extends AbstractEntity {
private Long groupId;
private String stringValue;
private LocalDateTime timestampValue;
private Double numericValue;
private Boolean booleanValue;
public RawTransactionValue() {
}
public Long getGroupId() {
return groupId;
}
public void setGroupId(Long groupId) {
this.groupId = groupId;
}
public String getStringValue() {
return stringValue;
}
public void setStringValue(String stringValue) {
this.stringValue = stringValue;
}
public LocalDateTime getTimestampValue() {
return timestampValue;
}
public void setTimestampValue(LocalDateTime timestampValue) {
this.timestampValue = timestampValue;
}
public Double getNumericValue() {
return numericValue;
}
public void setNumericValue(Double numericValue) {
this.numericValue = numericValue;
}
public Boolean getBooleanValue() {
return booleanValue;
}
public void setBooleanValue(Boolean booleanValue) {
this.booleanValue = booleanValue;
}
}

44
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/entity/RawTransactionValueGroup.java Normal file
View file

@ -0,0 +1,44 @@
package dev.mvvasilev.finances.entity;
import dev.mvvasilev.common.data.AbstractEntity;
import dev.mvvasilev.finances.enums.RawTransactionValueType;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
@Entity
@Table(schema = "transactions")
public class RawTransactionValueGroup extends AbstractEntity {
private Long statementId;
private String name;
private RawTransactionValueType type;
public RawTransactionValueGroup() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public RawTransactionValueType getType() {
return type;
}
public void setType(RawTransactionValueType type) {
this.type = type;
}
public Long getStatementId() {
return statementId;
}
public void setStatementId(Long statementId) {
this.statementId = statementId;
}
}

8
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/enums/RawTransactionValueType.java Normal file
View file

@ -0,0 +1,8 @@
package dev.mvvasilev.finances.enums;
public enum RawTransactionValueType {
STRING,
NUMERIC,
TIMESTAMP,
BOOLEAN
}

9
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/persistence/RawStatementRepository.java Normal file
View file

@ -0,0 +1,9 @@
package dev.mvvasilev.finances.persistence;
import dev.mvvasilev.finances.entity.RawStatement;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface RawStatementRepository extends JpaRepository<RawStatement, Long> {
}

9
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/persistence/RawTransactionValueGroupRepository.java Normal file
View file

@ -0,0 +1,9 @@
package dev.mvvasilev.finances.persistence;
import dev.mvvasilev.finances.entity.RawTransactionValueGroup;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface RawTransactionValueGroupRepository extends JpaRepository<RawTransactionValueGroup, Long> {
}

9
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/persistence/RawTransactionValueRepository.java Normal file
View file

@ -0,0 +1,9 @@
package dev.mvvasilev.finances.persistence;
import dev.mvvasilev.finances.entity.RawTransactionValue;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface RawTransactionValueRepository extends JpaRepository<RawTransactionValue, Long> {
}

169
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/services/TransactionsService.java Normal file
View file

@ -0,0 +1,169 @@
package dev.mvvasilev.finances.services;
import dev.mvvasilev.finances.entity.RawStatement;
import dev.mvvasilev.finances.entity.RawTransactionValue;
import dev.mvvasilev.finances.entity.RawTransactionValueGroup;
import dev.mvvasilev.finances.enums.RawTransactionValueType;
import dev.mvvasilev.finances.persistence.RawStatementRepository;
import dev.mvvasilev.finances.persistence.RawTransactionValueGroupRepository;
import dev.mvvasilev.finances.persistence.RawTransactionValueRepository;
import jakarta.transaction.Transactional;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.ss.util.CellUtil;
import org.hibernate.type.descriptor.DateTimeUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.time.temporal.ChronoField;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.IntStream;
@Service
@Transactional
public class TransactionsService {
private static final DateTimeFormatter DATE_FORMAT = new DateTimeFormatterBuilder()
.appendPattern("dd.MM.yyyy[ [HH][:mm][:ss]]")
.parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
.parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
.parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
.toFormatter()
.withResolverStyle(ResolverStyle.LENIENT);
private RawStatementRepository rawStatementRepository;
private RawTransactionValueGroupRepository rawTransactionValueGroupRepository;
private RawTransactionValueRepository rawTransactionValueRepository;
@Autowired
public TransactionsService(RawStatementRepository rawStatementRepository, RawTransactionValueGroupRepository rawTransactionValueGroupRepository, RawTransactionValueRepository rawTransactionValueRepository) {
this.rawStatementRepository = rawStatementRepository;
this.rawTransactionValueGroupRepository = rawTransactionValueGroupRepository;
this.rawTransactionValueRepository = rawTransactionValueRepository;
}
public void uploadMultipleTransactionsFromExcelSheetForUser(InputStream workbookInputStream, String userId) throws IOException {
var workbook = WorkbookFactory.create(workbookInputStream);
var firstWorksheet = workbook.getSheetAt(0);
var lastRowIndex = firstWorksheet.getLastRowNum();
var statement = new RawStatement();
statement.setUserId(Integer.parseInt(userId));
statement = rawStatementRepository.saveAndFlush(statement);
var firstRow = firstWorksheet.getRow(0);
List<RawTransactionValueGroup> valueGroups = new ArrayList<>();
for (var c : firstRow) {
if (c == null || c.getCellType() == CellType.BLANK) {
break;
}
var transactionValueGroup = new RawTransactionValueGroup();
transactionValueGroup.setStatementId(statement.getId());
transactionValueGroup.setName(c.getStringCellValue());
// group type is string by default, if no other type could have been determined
var groupType = RawTransactionValueType.STRING;
for (int y = c.getRowIndex() + 1; y <= lastRowIndex; y++) {
var typeResult = determineGroupType(firstWorksheet, y, c.getColumnIndex());
if (typeResult.isPresent()) {
groupType = typeResult.get();
break;
}
}
transactionValueGroup.setType(groupType);
valueGroups.add(transactionValueGroup);
}
valueGroups = rawTransactionValueGroupRepository.saveAllAndFlush(valueGroups);
var column = 0;
for (var group : valueGroups) {
var valueList = new ArrayList<RawTransactionValue>();
for (int y = 1; y < lastRowIndex; y++) {
var value = new RawTransactionValue();
value.setGroupId(group.getId());
switch (group.getType()) {
case STRING -> value.setStringValue(firstWorksheet.getRow(y).getCell(column).getStringCellValue());
case NUMERIC -> value.setNumericValue(firstWorksheet.getRow(y).getCell(column).getNumericCellValue());
case TIMESTAMP -> value.setTimestampValue(LocalDateTime.parse(firstWorksheet.getRow(y).getCell(column).getStringCellValue().trim(), DATE_FORMAT));
case BOOLEAN -> value.setBooleanValue(firstWorksheet.getRow(y).getCell(column).getBooleanCellValue());
}
valueList.add(value);
}
rawTransactionValueRepository.saveAllAndFlush(valueList);
column++;
}
}
private Optional<RawTransactionValueType> determineGroupType(Sheet worksheet, int rowIndex, int columnIndex) {
var cell = worksheet.getRow(rowIndex).getCell(columnIndex);
if (cell == null || cell.getCellType() == CellType.BLANK) {
return Optional.empty();
}
if (cell.getCellType() == CellType.BOOLEAN) {
return Optional.of(RawTransactionValueType.BOOLEAN);
}
if (cell.getCellType() == CellType.NUMERIC) {
return Optional.of(RawTransactionValueType.NUMERIC);
}
var cellValue = cell.getStringCellValue();
if (isValidDate(cellValue)) {
return Optional.of(RawTransactionValueType.TIMESTAMP);
}
return Optional.empty();
}
private boolean isValidDate(String inDate) {
try {
DATE_FORMAT.parse(inDate.trim());
} catch (DateTimeParseException e) {
return false;
}
return true;
}
}

7
PersonalFinancesService/src/main/java/dev/mvvasilev/finances/services/WorkbookParsingService.java Normal file
View file

@ -0,0 +1,7 @@
package dev.mvvasilev.finances.services;
import org.springframework.stereotype.Service;
@Service
public class WorkbookParsingService {
}

9
PersonalFinancesService/src/main/resources/application.properties
View file

@ -1,6 +1,9 @@
server.port=8081
debug=true
logging.level.org.springframework.web=DEBUG
logging.level.org.springframework.boot.autoconfigure=ERROR
spring.profiles.active=${PROFILE}
# Database
spring.datasource.url=${DATASOURCE_URL}
@ -8,5 +11,9 @@ spring.datasource.username=${DATASOURCE_USER}
spring.datasource.password=${DATASOURCE_PASSWORD}
spring.datasource.driver-class-name=org.postgresql.Driver
spring.jpa.generate-ddl=false
spring.jpa.show-sql=true
spring.jpa.hibernate.ddl-auto=validate
# Security
jwt.issuer-url=${AUTHENTIK_ISSUER_URL}

31
PersonalFinancesService/src/main/resources/db/migration/V1.0__CreateTransactionTables.sql Normal file
View file

@ -0,0 +1,31 @@
CREATE SCHEMA IF NOT EXISTS transactions;
CREATE TABLE IF NOT EXISTS transactions.raw_statement (
id BIGSERIAL,
time_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
time_last_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
CONSTRAINT PK_raw_statement PRIMARY KEY (id)
);
CREATE TABLE IF NOT EXISTS transactions.raw_transaction_value_group (
id BIGSERIAL,
statement_id BIGINT,
name VARCHAR(255),
type SMALLINT,
time_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
time_last_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
CONSTRAINT PK_raw_transaction_value_group PRIMARY KEY (id),
CONSTRAINT FK_raw_transaction_value_group_raw_statement FOREIGN KEY (statement_id) REFERENCES transactions.raw_statement(id)
);
CREATE TABLE IF NOT EXISTS transactions.raw_transaction_value (
id BIGSERIAL,
group_id BIGINT,
string_value VARCHAR(1024),
timestamp_value TIMESTAMP,
numeric_value FLOAT,
time_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
time_last_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
CONSTRAINT PK_raw_transaction_value PRIMARY KEY (id),
CONSTRAINT FK_raw_transaction_value_raw_transaction_value_group FOREIGN KEY (group_id) REFERENCES transactions.raw_transaction_value_group(id)
)

1
PersonalFinancesService/src/main/resources/db/migration/V1.1__AddUserIdToRawStatement.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE transactions.raw_statement ADD user_id INTEGER NOT NULL;

1
PersonalFinancesService/src/main/resources/db/migration/V1.2__AddBooleanValueToRawTransactionValue.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE transactions.raw_transaction_value ADD boolean_value BOOLEAN;

1
settings.gradle
View file

@ -2,4 +2,5 @@ rootProject.name = 'personal-finances'
include 'PersonalFinancesService'
include 'APIGateway'
include 'Common'