Add secure source ingestion pipeline

.env.example

@@ -9,3 +9,7 @@ ORGMEMORY_DB_USER=orgmemory
 ORGMEMORY_DB_PASSWORD=orgmemory
 ORGMEMORY_API_PORT=8080
 ORGMEMORY_MCP_PORT=8081
+
+# Generate separate local object-storage credentials; never reuse production values.
+ORGMEMORY_OBJECT_STORAGE_ACCESS_KEY=replace-with-local-minio-user
+ORGMEMORY_OBJECT_STORAGE_SECRET_KEY=replace-with-a-long-random-local-secret

apps/api/build.gradle.kts

@@ -1,10 +1,13 @@
+import org.gradle.api.tasks.testing.Test
+
 plugins {
     id("orgmemory.spring-boot-app-conventions")
 }
 
 dependencies {
     implementation(project(":core"))
     implementation(project(":integrations:authorization-openfga"))
+    implementation(project(":integrations:object-storage-minio"))
 
     implementation("org.springframework.boot:spring-boot-starter-webmvc")
     implementation("org.springframework.boot:spring-boot-starter-security-oauth2-client")
@@ -27,3 +30,7 @@ dependencies {
     testImplementation("org.testcontainers:testcontainers-postgresql")
     testRuntimeOnly("org.junit.platform:junit-platform-launcher")
 }
+
+tasks.withType<Test>().configureEach {
+    systemProperty("spring.session.jdbc.cleanup-cron", "-")
+}

apps/api/src/main/java/com/orgmemory/api/OrgMemoryApiApplication.java

@@ -1,7 +1,9 @@
 package com.orgmemory.api;
 
+import com.orgmemory.core.knowledge.SourceIngestionProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.boot.persistence.autoconfigure.EntityScan;
 import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 
@@ -12,9 +14,10 @@
 })
 @EntityScan("com.orgmemory.core")
 @EnableJpaRepositories("com.orgmemory.core")
+@EnableConfigurationProperties(SourceIngestionProperties.class)
 public class OrgMemoryApiApplication {
 
-    public static void main(String[] args) {
+    static void main(String[] args) {
         SpringApplication.run(OrgMemoryApiApplication.class, args);
     }
 }

apps/api/src/main/java/com/orgmemory/api/source/SourceController.java

@@ -0,0 +1,68 @@
+package com.orgmemory.api.source;
+
+import com.orgmemory.api.security.CurrentActorProvider;
+import com.orgmemory.core.knowledge.CreateUploadSourceCommand;
+import com.orgmemory.core.knowledge.SourceQueryService;
+import com.orgmemory.core.knowledge.SourceUploadService;
+import com.orgmemory.core.organization.CurrentActor;
+import com.orgmemory.core.permission.KnowledgeClassification;
+import io.swagger.v3.oas.annotations.Operation;
+import java.io.IOException;
+import java.util.List;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.security.core.Authentication;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.bind.annotation.ResponseStatus;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.server.ResponseStatusException;
+
+@RestController
+@RequestMapping("/api/sources")
+class SourceController {
+
+    private final SourceQueryService sources;
+    private final SourceUploadService uploads;
+    private final CurrentActorProvider actors;
+
+    SourceController(SourceQueryService sources, SourceUploadService uploads, CurrentActorProvider actors) {
+        this.sources = sources;
+        this.uploads = uploads;
+        this.actors = actors;
+    }
+
+    @GetMapping
+    @Operation(operationId = "listSources", summary = "List sources uploaded by the current user")
+    List<SourceResponse> list(Authentication authentication) {
+        CurrentActor actor = actors.current(authentication);
+        return sources.listOwn(actor).stream().map(SourceResponse::from).toList();
+    }
+
+    @PostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
+    @Operation(operationId = "uploadSource", summary = "Upload a source for asynchronous ingestion")
+    @ResponseStatus(HttpStatus.CREATED)
+    SourceResponse upload(
+            @RequestPart("file") MultipartFile file,
+            @RequestParam(defaultValue = "CONFIDENTIAL") KnowledgeClassification classification,
+            Authentication authentication) {
+        CurrentActor actor = actors.current(authentication);
+        try (var content = file.getInputStream()) {
+            return SourceResponse.from(uploads.upload(
+                    new CreateUploadSourceCommand(
+                            actor,
+                            file.getOriginalFilename(),
+                            file.getContentType(),
+                            file.getSize(),
+                            classification),
+                    content));
+        } catch (IOException exception) {
+            throw new ResponseStatusException(
+                    HttpStatus.BAD_REQUEST, "The uploaded file could not be read", exception);
+        }
+    }
+}

apps/api/src/main/java/com/orgmemory/api/source/SourceResponse.java

@@ -0,0 +1,44 @@
+package com.orgmemory.api.source;
+
+import com.orgmemory.core.knowledge.SourceSummary;
+import java.time.Instant;
+import java.util.UUID;
+
+record SourceResponse(
+        UUID id,
+        String title,
+        String sourceType,
+        String status,
+        String classification,
+        String fileName,
+        String mediaType,
+        long contentLength,
+        String failureCode,
+        String failureMessage,
+        String embeddingProfileKey,
+        String embeddingProvider,
+        String embeddingModel,
+        Integer embeddingDimensions,
+        Instant createdAt,
+        Instant updatedAt) {
+
+    static SourceResponse from(SourceSummary source) {
+        return new SourceResponse(
+                source.id(),
+                source.title(),
+                source.sourceType().name(),
+                source.status().name(),
+                source.classification().name(),
+                source.fileName(),
+                source.mediaType(),
+                source.contentLength(),
+                source.failureCode(),
+                source.failureMessage(),
+                source.embeddingProfileKey(),
+                source.embeddingProvider(),
+                source.embeddingModel(),
+                source.embeddingDimensions(),
+                source.createdAt(),
+                source.updatedAt());
+    }
+}

apps/api/src/main/resources/application.yml

@@ -38,8 +38,22 @@ spring:
     timeout: ${ORGMEMORY_WEB_SESSION_TIMEOUT:8h}
     jdbc:
       initialize-schema: never
+  servlet:
+    multipart:
+      max-file-size: ${ORGMEMORY_MAX_UPLOAD_SIZE:25MB}
+      max-request-size: ${ORGMEMORY_MAX_UPLOAD_SIZE:25MB}
 
 orgmemory:
+  ingestion:
+    maximum-upload-size: ${ORGMEMORY_MAX_UPLOAD_SIZE:25MB}
+    maximum-attempts: ${ORGMEMORY_INGESTION_MAX_ATTEMPTS:5}
+  storage:
+    object:
+      endpoint: ${ORGMEMORY_OBJECT_STORAGE_ENDPOINT:http://localhost:9000}
+      access-key: ${ORGMEMORY_OBJECT_STORAGE_ACCESS_KEY:orgmemory-local}
+      secret-key: ${ORGMEMORY_OBJECT_STORAGE_SECRET_KEY:orgmemory-local-secret}
+      bucket: ${ORGMEMORY_OBJECT_STORAGE_BUCKET:orgmemory-evidence}
+      maximum-object-size: ${ORGMEMORY_MAX_UPLOAD_SIZE:25MB}
   security:
     oidc:
       issuer-uri: ${ORGMEMORY_OIDC_ISSUER_URI:http://localhost:8180/realms/orgmemory}

apps/api/src/test/java/com/orgmemory/api/OrgMemoryApiContextLoadTests.java

@@ -3,12 +3,14 @@
 import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
+import org.springframework.test.annotation.DirtiesContext;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 import org.testcontainers.postgresql.PostgreSQLContainer;
 
 @SpringBootTest
 @Testcontainers
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 class OrgMemoryApiContextLoadTests {
 
     @Container

apps/api/src/test/java/com/orgmemory/api/capability/CapabilityAssetServiceIntegrationTests.java

@@ -41,6 +41,7 @@
 import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc;
 import org.springframework.http.MediaType;
 import org.springframework.security.core.authority.SimpleGrantedAuthority;
+import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.web.servlet.MockMvc;
 import org.springframework.test.context.bean.override.mockito.MockitoBean;
 import org.springframework.test.web.servlet.request.RequestPostProcessor;
@@ -52,6 +53,7 @@
 @SpringBootTest(properties = "spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://localhost/unused")
 @AutoConfigureMockMvc
 @Testcontainers
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 class CapabilityAssetServiceIntegrationTests {
 
     private static final String ISSUER = "http://localhost:8180/realms/orgmemory";

apps/api/src/test/java/com/orgmemory/api/knowledge/KnowledgeIngestionIntegrationTests.java

@@ -38,12 +38,14 @@
 import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
 import org.springframework.dao.DataAccessException;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.annotation.DirtiesContext;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 import org.testcontainers.postgresql.PostgreSQLContainer;
 
 @SpringBootTest
 @Testcontainers
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 class KnowledgeIngestionIntegrationTests {
 
     private static final UUID ORGANIZATION_ID = UUID.fromString("11111111-1111-1111-1111-111111111111");
@@ -370,13 +372,11 @@ void sourceUriDropsQueryAndFragmentBeforePersistence() {
                         String.class,
                         raw.rawSourceObjectId()));
 
-        RegisterRawSourceCommand unsafeScheme = withSourceUri(
-                completeCommand(
-                        "unsafe-source-uri-doc",
-                        "Source body with an unsafe citation scheme.",
-                        KnowledgeClassification.PUBLIC,
-                        DeclaredAccessScope.ALL),
-                "javascript:alert('unsafe')");
+        RegisterRawSourceCommand unsafeScheme = withUnsafeSourceUri(completeCommand(
+                "unsafe-source-uri-doc",
+                "Source body with an unsafe citation scheme.",
+                KnowledgeClassification.PUBLIC,
+                DeclaredAccessScope.ALL));
         assertThrows(IllegalArgumentException.class, () -> ingestion.registerRawSource(unsafeScheme));
     }
 
@@ -495,22 +495,19 @@ private Map<String, Object> snapshotEvidence(UUID snapshotId) {
     }
 
     private static <T> List<T> runConcurrently(Callable<T> action) throws Exception {
-        var executor = Executors.newFixedThreadPool(2);
         var barrier = new CyclicBarrier(2);
-        try {
+        try (var executor = Executors.newFixedThreadPool(2)) {
             Callable<T> synchronizedAction = () -> {
                 assertTrue(barrier.await(10, TimeUnit.SECONDS) >= 0);
                 return action.call();
             };
             var first = executor.submit(synchronizedAction);
             var second = executor.submit(synchronizedAction);
             return List.of(first.get(30, TimeUnit.SECONDS), second.get(30, TimeUnit.SECONDS));
-        } finally {
-            executor.shutdownNow();
         }
     }
 
-    private static RegisterRawSourceCommand withSourceUri(RegisterRawSourceCommand base, String sourceUri) {
+    private static RegisterRawSourceCommand withUnsafeSourceUri(RegisterRawSourceCommand base) {
         return new RegisterRawSourceCommand(
                 base.organizationId(),
                 base.departmentId(),
@@ -521,7 +518,7 @@ private static RegisterRawSourceCommand withSourceUri(RegisterRawSourceCommand b
                 base.objectType(),
                 base.title(),
                 base.rawContent(),
-                sourceUri,
+                "javascript:alert('unsafe')",
                 base.sourceModifiedAt(),
                 base.classification(),
                 base.declaredAccess(),

apps/api/src/test/java/com/orgmemory/api/knowledge/KnowledgeRetrievalIntegrationTests.java

@@ -40,6 +40,7 @@
 import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.security.core.authority.SimpleGrantedAuthority;
+import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.web.servlet.MockMvc;
 import org.springframework.test.context.bean.override.mockito.MockitoBean;
 import org.springframework.test.web.servlet.MvcResult;
@@ -51,6 +52,7 @@
 @SpringBootTest(properties = "spring.security.oauth2.resourceserver.jwt.jwk-set-uri=http://localhost/unused")
 @AutoConfigureMockMvc
 @Testcontainers
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 class KnowledgeRetrievalIntegrationTests {
 
     private static final String ISSUER = "http://localhost:8180/realms/orgmemory";

apps/api/src/test/java/com/orgmemory/api/permission/PermissionAuditIntegrationTests.java

@@ -12,13 +12,15 @@
 import com.orgmemory.core.permission.PermissionAuditDecision;
 import com.orgmemory.core.permission.PermissionAuditService;
 import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
 import org.springframework.dao.DataAccessException;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.support.TransactionTemplate;
 import org.testcontainers.junit.jupiter.Container;
@@ -27,6 +29,7 @@
 
 @SpringBootTest
 @Testcontainers
+@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
 class PermissionAuditIntegrationTests {
 
     private static final UUID ORGANIZATION_ID = UUID.fromString("11111111-1111-1111-1111-111111111111");
@@ -89,7 +92,8 @@ void requiresNewAuditCommitSurvivesOuterRollback() {
     @Test
     void databaseRejectsUpdateDeleteAndTruncate() {
         UUID eventId = audit.record(command("DOC032", null));
-        long countBefore = jdbc.queryForObject("SELECT count(*) FROM permission_audit_events", Long.class);
+        long countBefore = Objects.requireNonNull(
+                jdbc.queryForObject("SELECT count(*) FROM permission_audit_events", Long.class));
 
         assertThrows(
                 DataAccessException.class,