From e0b408c033b49ce51a63f073485360530fafd7a2 Mon Sep 17 00:00:00 2001 From: "A. Anil Sinaci" Date: Thu, 24 Sep 2026 22:03:28 +0300 Subject: [PATCH 1/3] :construction_worker: build: publish community artifacts to SRDC Nexus for the enterprise edition --- .github/maven-settings.xml | 18 ++++++++++ .github/workflows/maven.yml | 34 +++++++++++++++++++ ignifyr-cli/pom.xml | 6 ++++ pom.xml | 68 ++++++++++++++++++++++++++----------- 4 files changed, 106 insertions(+), 20 deletions(-) create mode 100644 .github/maven-settings.xml diff --git a/.github/maven-settings.xml b/.github/maven-settings.xml new file mode 100644 index 00000000..fae6dc71 --- /dev/null +++ b/.github/maven-settings.xml @@ -0,0 +1,18 @@ + + + + + + srdc-maven-snapshots + ${env.NEXUS_USERNAME} + ${env.NEXUS_PASSWORD} + + + srdc-maven-releases + ${env.NEXUS_USERNAME} + ${env.NEXUS_PASSWORD} + + + diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b90601fe..1cf596bb 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -7,6 +7,9 @@ # stays short and Docker-free; run `mvn verify -DskipITs=false` when you want it) and mandatory on # every pull request. Pushes only get the two fast jobs, since anything reaching a protected branch # has already been through a PR that ran the long tier. +# +# A push to main also deploys the community -SNAPSHOT modules to SRDC Nexus, which is where the +# enterprise edition resolves them from between releases. Releases are deployed by hand (RELEASING.md). name: Java CI with Maven @@ -71,3 +74,34 @@ jobs: # lets ${revision} stay a snapshot. RELEASING.md runs it with --release at release time. - name: Release readiness - jar contents and licensing run: bash test-flow/check-release-ready.sh + + # The enterprise edition builds against whatever community snapshot is on SRDC Nexus, so main's + # snapshot is republished after every push that passed the short tier. Needs the NEXUS_USERNAME / + # NEXUS_PASSWORD repository secrets; without them the job says so and succeeds, rather than turning + # main red on a fork or before the secrets are configured. + deploy-snapshot: + name: Deploy snapshot to SRDC Nexus + needs: [ tiers, short ] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + env: + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + - name: Deploy + run: | + if [ -z "$NEXUS_USERNAME" ] || [ -z "$NEXUS_PASSWORD" ]; then + echo "::warning::NEXUS_USERNAME/NEXUS_PASSWORD secrets are not set; skipping the snapshot deploy" + exit 0 + fi + case "$(grep -m1 -oE '[^<]+' pom.xml)" in + *-SNAPSHOT*) ;; + *) echo "::notice::\${revision} is a release version; releases are deployed by hand (RELEASING.md)"; exit 0 ;; + esac + mvn -B -s .github/maven-settings.xml -DskipTests deploy diff --git a/ignifyr-cli/pom.xml b/ignifyr-cli/pom.xml index db963c7f..4b2e638e 100644 --- a/ignifyr-cli/pom.xml +++ b/ignifyr-cli/pom.xml @@ -13,6 +13,12 @@ ignifyr-cli_2.13 jar + + + true + + + resolveCiFriendliesOnly + true @@ -573,101 +582,103 @@ ${akka.version} - + io.ignifyr ignifyr-engine_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-testkit_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-connector-sql_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-connector-file_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-sink-fhir_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-sink-file_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-sink-omop_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-format-json_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-format-delta_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-connector-fhir-server_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-connector-kafka_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-runtime-streaming_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-runtime-scheduling_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-redcap_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-terminology-tools_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-observability_2.13 - ${project.version} + ${revision} io.ignifyr ignifyr-common - ${project.version} + ${revision} io.ignifyr ignifyr-server-common - ${project.version} + ${revision} io.ignifyr ignifyr-rxnorm_2.13 - ${project.version} + ${revision} @@ -807,4 +818,21 @@ + + + + srdc-maven-releases + SRDC Nexus Releases + https://nexus.srdc.com.tr/repository/maven-releases/ + + + srdc-maven-snapshots + SRDC Nexus Snapshots + https://nexus.srdc.com.tr/repository/maven-snapshots/ + + + From 4cd879537922744c840cf6a054e915735382c739 Mon Sep 17 00:00:00 2001 From: "A. Anil Sinaci" Date: Thu, 24 Sep 2026 22:03:28 +0300 Subject: [PATCH 2/3] :recycle: refactor: move the enterprise modules to the private enterprise repository --- docker/docker-compose-efk.yml | 35 - docker/docker-compose.yml | 70 - docker/fluentd/Dockerfile | 17 - docker/fluentd/build.sh | 3 - docker/fluentd/conf/fluent.conf | 23 - docker/kibana/Dockerfile | 15 - docker/kibana/build.sh | 3 - docker/kibana/exports/dashboards.ndjson | 15 - .../exports/ignore_above_index_template.json | 23 - .../required_fields_index_template.json | 106 - docker/kibana/init-script.sh | 45 - docker/kibana/kibana.yml | 12 - docker/server/Dockerfile | 14 - docker/server/build.sh | 4 - docker/server/docker-entrypoint.sh | 121 - ignifyr-connector-fhir-server/pom.xml | 63 - .../io.ignifyr.engine.spi.IgnifyrExtension | 2 - .../FhirServerConnectorExtension.scala | 28 - .../FhirServerDataSourceReader.scala | 109 - .../FhirServerConnectorExtensionSpec.scala | 19 - ignifyr-connector-kafka/pom.xml | 64 - .../io.ignifyr.engine.spi.IgnifyrExtension | 2 - .../kafka/KafkaConnectorExtension.scala | 80 - .../connector/kafka/KafkaSourceReader.scala | 199 - .../kafka/KafkaConnectorExtensionSpec.scala | 23 - .../kafka/KafkaSourceReaderTest.scala | 110 - ignifyr-format-delta/pom.xml | 65 - .../io.ignifyr.engine.spi.IgnifyrExtension | 4 - ...io.ignifyr.sink.file.format.FileSinkFormat | 2 - .../format/delta/DeltaFormatExtension.scala | 23 - .../format/delta/DeltaSinkFormat.scala | 39 - .../src/test/resources/application.conf | 35 - .../format/delta/DeltaSinkFormatTest.scala | 78 - ignifyr-format-json/pom.xml | 58 - ...fyr.connector.file.format.FileSourceFormat | 2 - .../format/json/JsonSourceFormat.scala | 33 - .../src/test/resources/application.conf | 35 - .../folder-test/json/patient1.json | 3 - .../folder-test/json/patient2.json | 3 - .../folder-test/json/patient3.json | 3 - .../folder-test/txt-ndjson/patient1.txt | 3 - .../folder-test/txt-ndjson/patient2.txt | 3 - .../folder-test/txt-ndjson/patient3.txt | 3 - .../single-file-test/patients-ndjson.txt | 11 - .../single-file-test/patients.json | 11 - .../zip-test/json.zip | Bin 690 -> 0 bytes .../zip-test/txt-ndjson.zip | Bin 681 -> 0 bytes .../format/json/JsonSourceFormatTest.scala | 157 - ignifyr-observability/pom.xml | 78 - .../MapMarkerToLogstashMarkerEncoder.scala | 101 - ...MapMarkerToLogstashMarkerEncoderSpec.scala | 96 - ignifyr-redcap/CLAUDE.md | 67 - ignifyr-redcap/pom.xml | 64 - .../io.ignifyr.engine.spi.IgnifyrExtension | 2 - ...r.server.common.spi.IgnifyrServerExtension | 2 - .../ignifyr/redcap/ExtractRedCapSchemas.scala | 64 - .../io/ignifyr/redcap/RedCapEndpoint.scala | 108 - .../io/ignifyr/redcap/RedCapExtension.scala | 29 - .../ignifyr/redcap/RedCapProjectConfig.scala | 10 - .../redcap/RedCapSchemaImportEndpoint.scala | 53 - .../redcap/RedCapServerExtension.scala | 54 - .../io/ignifyr/redcap/RedCapService.scala | 109 - .../ignifyr/redcap/RedCapServiceConfig.scala | 51 - .../scala/io/ignifyr/redcap/RedCapUtil.scala | 432 -- .../ignifyr/redcap/RedCapExtensionSpec.scala | 63 - .../redcap/RedCapUtilDataTypeSpec.scala | 256 -- ignifyr-runtime-scheduling/CLAUDE.md | 55 - ignifyr-runtime-scheduling/pom.xml | 101 - .../io.ignifyr.engine.spi.IgnifyrExtension | 2 - .../scheduling/Cron4jSchedulerProvider.scala | 176 - .../scheduling/ScheduledJobRegistry.scala | 129 - .../SchedulingRuntimeExtension.scala | 13 - .../src/test/resources/application.conf | 39 - .../test/resources/sql/scheduling-drop.sql | 3 - .../resources/sql/scheduling-populate.sql | 49 - .../resources/test-schedule-mappingjob.json | 48 - .../integrationtest/SchedulingTest.scala | 145 - .../scheduling/ScheduledTimeRangeSpec.scala | 102 - .../SchedulingRuntimeExtensionSpec.scala | 26 - ignifyr-runtime-streaming/CLAUDE.md | 80 - ignifyr-runtime-streaming/pom.xml | 118 - .../io.ignifyr.engine.spi.IgnifyrExtension | 2 - .../streaming/StreamingJobExecutor.scala | 71 - .../streaming/StreamingRuntimeExtension.scala | 13 - .../streaming/StreamingSinkHandler.scala | 52 - .../src/test/resources/application.conf | 51 - .../streaming-folder-watch/patients.csv | 11 - .../KafkaStreamingRedcapTest.scala | 167 - .../StreamingFolderWatchTest.scala | 159 - .../streaming/StreamingSinkHandlerTest.scala | 119 - ignifyr-rxnorm/CLAUDE.md | 63 - ignifyr-rxnorm/pom.xml | 110 - .../rxnorm/PullRxNormNdcMedDetails.scala | 196 - .../io/ignifyr/rxnorm/RxNormApiClient.scala | 205 - .../rxnorm/RxNormApiFunctionLibrary.scala | 307 -- .../ignifyr/rxnorm/RxNormApiClientTest.scala | 104 - .../rxnorm/RxNormApiFunctionLibraryTest.scala | 44 - .../io/ignifyr/rxnorm/RxNormApiStub.scala | 63 - ignifyr-server-common/CLAUDE.md | 51 - ignifyr-server-common/pom.xml | 91 - .../common/config/WebServerConfig.scala | 33 - .../common/interceptor/ICORSHandler.scala | 67 - .../common/interceptor/IErrorHandler.scala | 47 - .../server/common/model/IgnifyrError.scala | 81 - .../server/common/model/IgnifyrRestCall.scala | 31 - .../common/spi/IgnifyrServerExtension.scala | 49 - .../common/spi/IgnifyrServerExtensions.scala | 32 - .../server/common/spi/SchemaImportSink.scala | 20 - ignifyr-server/CLAUDE.md | 100 - ignifyr-server/api.yaml | 3593 ----------------- ignifyr-server/pom.xml | 374 -- .../src/main/resources/application.conf | 223 - ignifyr-server/src/main/resources/logback.xml | 124 - .../src/main/resources/version.properties | 1 - .../main/scala/io/ignifyr/server/Boot.scala | 11 - .../io/ignifyr/server/IgnifyrHttpServer.scala | 69 - .../io/ignifyr/server/IgnifyrServer.scala | 23 - .../server/endpoint/CodeSystemEndpoint.scala | 206 - .../server/endpoint/ConceptMapEndpoint.scala | 206 - .../FileSystemTreeStructureEndpoint.scala | 44 - .../endpoint/IgnifyrServerEndpoint.scala | 116 - .../ignifyr/server/endpoint/JobEndpoint.scala | 306 -- .../endpoint/MappingContextEndpoint.scala | 235 -- .../server/endpoint/MappingEndpoint.scala | 92 - .../server/endpoint/MetadataEndpoint.scala | 54 - .../server/endpoint/ProjectEndpoint.scala | 162 - .../server/endpoint/ReloadEndpoint.scala | 50 - .../endpoint/SchemaDefinitionEndpoint.scala | 281 -- .../TerminologyServiceManagerEndpoint.scala | 163 - .../ignifyr/server/model/ExecuteJobTask.scala | 7 - .../ignifyr/server/model/FilePathNode.scala | 10 - .../server/model/ImportSchemaSettings.scala | 16 - .../io/ignifyr/server/model/InferTask.scala | 16 - .../io/ignifyr/server/model/Metadata.scala | 64 - .../io/ignifyr/server/model/Project.scala | 79 - .../server/model/TerminologySystem.scala | 48 - .../model/TestResourceCreationRequest.scala | 31 - .../ignifyr/server/model/csv/CsvHeader.scala | 9 - .../repository/FolderDBInitializer.scala | 199 - .../repository/FolderRepositoryManager.scala | 63 - .../repository/IRepositoryManager.scala | 35 - .../repository/job/IJobRepository.scala | 72 - .../repository/job/JobFolderRepository.scala | 265 -- .../mapping/IMappingRepository.scala | 74 - .../ProjectMappingFolderRepository.scala | 309 -- .../IMappingContextRepository.scala | 102 - .../MappingContextFolderRepository.scala | 299 -- .../repository/project/IProjectList.scala | 16 - .../project/IProjectRepository.scala | 151 - .../project/ProjectFolderRepository.scala | 373 -- .../schema/AbstractSchemaRepository.scala | 35 - .../repository/schema/ISchemaRepository.scala | 97 - .../schema/SchemaFolderRepository.scala | 683 ---- .../schema/SchemaManagementUtil.scala | 37 - .../ITerminologySystemRepository.scala | 52 - .../TerminologySystemFolderRepository.scala | 294 -- .../CodeSystemFolderRepository.scala | 313 -- .../codesystem/ICodeSystemRepository.scala | 108 - .../ConceptMapFolderRepository.scala | 313 -- .../conceptmap/IConceptMapRepository.scala | 108 - .../server/service/ExecutionService.scala | 425 -- .../ignifyr/server/service/JobService.scala | 89 - .../service/MappingContextService.scala | 111 - .../server/service/MappingService.scala | 97 - .../server/service/MetadataService.scala | 107 - .../server/service/ProjectService.scala | 111 - .../server/service/ReloadService.scala | 34 - .../service/SchemaDefinitionService.scala | 283 -- .../fhir/FileSystemTreeStructureService.scala | 74 - .../terminology/CodeSystemService.scala | 136 - .../terminology/ConceptMapService.scala | 136 - .../TerminologySystemService.scala | 120 - .../io/ignifyr/server/util/CsvUtil.scala | 231 -- .../ignifyr/server/util/DataFrameUtil.scala | 27 - .../ignifyr/server/util/FileOperations.scala | 220 - .../server/util/IgnifyrRejectionHandler.scala | 106 - .../src/test/resources/application.conf | 138 - .../fhir-resources/condition-resource.json | 24 - .../custom-patient-profile.json | 73 - .../fhir-resources/custom-profiles.zip | Bin 1347 -> 0 bytes ...ir-profiles-missing-referenced-schemas.zip | Bin 1118 -> 0 bytes .../invalid-condition-resource.json | 31 - .../fhir-resources/patient-bundle.json | 108 - .../fhir-resources/patient-resource.json | 39 - .../resources/file-system-structure/file1.txt | 1 - .../resources/file-system-structure/file2.txt | 1 - .../file-system-structure/subdir/file3.txt | 1 - ignifyr-server/src/test/resources/logback.xml | 94 - .../src/test/resources/redcap/instrument.csv | 10 - .../test/resources/sql/sql-source-drop.sql | 2 - .../resources/sql/sql-source-populate.sql | 122 - .../patients/patients.csv | 11 - .../test-data/other-observations.csv | 15 - .../test-data/patient-gender-simple.csv | 2 - .../resources/test-data/patient-simple.csv | 2 - .../src/test/resources/test-data/patients.csv | 11 - .../sample-unit-conversion.csv | 5 - .../patient-mapping-job-with-two-sources.json | 51 - .../other-observation-concept-map.csv | 14 - .../other-observation-mapping.json | 213 - .../other-observation-mapping2.json | 63 - .../test-mappings/patient-flat-mapping.json | 38 - .../patient-mapping-with-two-sources.json | 42 - .../test-mappings/patient-mapping.json | 47 - .../test-mappings/patient-mapping2.json | 47 - .../test-schemas/blood-pressure-schema.json | 112 - .../other-observation-schema.json | 117 - .../test-schemas/patient-schema.json | 118 - .../sample-code-system.csv | 3 - .../sample-concept-map.csv | 5 - .../FhirDefinitionsEndpointTest.scala | 216 - .../MappingExecutionEndpointTest.scala | 851 ---- .../integrationtest/SchemaEndpointTest.scala | 1215 ------ .../io/ignifyr/server/BaseEndpointTest.scala | 106 - .../FileSystemTreeStructureEndpointTest.scala | 91 - .../endpoint/IgnifyrErrorHandlerTest.scala | 87 - .../IgnifyrRejectionHandlerTest.scala | 66 - .../server/endpoint/JobEndpointTest.scala | 331 -- .../JobExecutionControlEndpointTest.scala | 110 - .../endpoint/MappingContextEndpointTest.scala | 132 - .../server/endpoint/MappingEndpointTest.scala | 243 -- .../endpoint/MetadataEndpointTest.scala | 38 - .../server/endpoint/ProjectEndpointTest.scala | 149 - .../endpoint/ReloadingEndpointTest.scala | 58 - ...erminologyServiceManagerEndpointTest.scala | 535 --- .../repository/FolderDBInitializerTest.scala | 266 -- .../server/service/ExecutionServiceTest.scala | 138 - .../server/service/MetadataServiceTest.scala | 81 - .../io/ignifyr/server/util/CsvUtilTest.scala | 201 - .../server/util/DataFrameUtilTest.scala | 58 - .../io/ignifyr/server/util/TestUtil.scala | 21 - ignifyr-sink-omop/pom.xml | 67 - .../io.ignifyr.engine.spi.IgnifyrExtension | 1 - .../ignifyr/sink/omop/OmopSinkExtension.scala | 17 - .../sink/omop/OmopSinkExtensionSpec.scala | 17 - ignifyr-terminology-tools/pom.xml | 72 - .../ignifyr/tools/terminology/Concept.scala | 15 - .../terminology/GeneratorDBAdapter.scala | 103 - .../TerminologyMappingGenerator.scala | 250 -- .../TerminologyMappingGeneratorTest.scala | 57 - pom.xml | 73 - test-flow/.gitignore | 5 - test-flow/check-editions.sh | 39 +- test-flow/check-release-ready.sh | 51 +- test-flow/config/ignifyr-server.conf | 85 - test-flow/config/jobs/archive-file-job.json | 32 - test-flow/config/jobs/batch-file-job.json | 32 - .../config/jobs/scheduling-file-job.json | 36 - test-flow/config/jobs/sql-patient-job.json | 33 - test-flow/create-ui-testflow.sh | 156 - test-flow/data/patients.csv | 11 - test-flow/data/redcap-patients.ndjson | 3 - test-flow/data/sql-source-seed.sql | 21 - test-flow/data/stream-patients.csv | 6 - test-flow/docker-compose.test.yml | 176 - test-flow/nginx.conf | 56 - test-flow/run-automated-tests.sh | 23 +- test-flow/run-manual-flow.sh | 340 -- 258 files changed, 45 insertions(+), 28043 deletions(-) delete mode 100644 docker/docker-compose-efk.yml delete mode 100644 docker/docker-compose.yml delete mode 100644 docker/fluentd/Dockerfile delete mode 100644 docker/fluentd/build.sh delete mode 100644 docker/fluentd/conf/fluent.conf delete mode 100644 docker/kibana/Dockerfile delete mode 100644 docker/kibana/build.sh delete mode 100644 docker/kibana/exports/dashboards.ndjson delete mode 100644 docker/kibana/exports/ignore_above_index_template.json delete mode 100644 docker/kibana/exports/required_fields_index_template.json delete mode 100644 docker/kibana/init-script.sh delete mode 100644 docker/kibana/kibana.yml delete mode 100644 docker/server/Dockerfile delete mode 100644 docker/server/build.sh delete mode 100644 docker/server/docker-entrypoint.sh delete mode 100644 ignifyr-connector-fhir-server/pom.xml delete mode 100644 ignifyr-connector-fhir-server/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension delete mode 100644 ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtension.scala delete mode 100644 ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerDataSourceReader.scala delete mode 100644 ignifyr-connector-fhir-server/src/test/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtensionSpec.scala delete mode 100644 ignifyr-connector-kafka/pom.xml delete mode 100644 ignifyr-connector-kafka/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension delete mode 100644 ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaConnectorExtension.scala delete mode 100644 ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaSourceReader.scala delete mode 100644 ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaConnectorExtensionSpec.scala delete mode 100644 ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaSourceReaderTest.scala delete mode 100644 ignifyr-format-delta/pom.xml delete mode 100644 ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension delete mode 100644 ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.sink.file.format.FileSinkFormat delete mode 100644 ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaFormatExtension.scala delete mode 100644 ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaSinkFormat.scala delete mode 100644 ignifyr-format-delta/src/test/resources/application.conf delete mode 100644 ignifyr-format-delta/src/test/scala/io/ignifyr/format/delta/DeltaSinkFormatTest.scala delete mode 100644 ignifyr-format-json/pom.xml delete mode 100644 ignifyr-format-json/src/main/resources/META-INF/services/io.ignifyr.connector.file.format.FileSourceFormat delete mode 100644 ignifyr-format-json/src/main/scala/io/ignifyr/format/json/JsonSourceFormat.scala delete mode 100644 ignifyr-format-json/src/test/resources/application.conf delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient1.json delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient2.json delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient3.json delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient1.txt delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient2.txt delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient3.txt delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients-ndjson.txt delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients.json delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/zip-test/json.zip delete mode 100644 ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/zip-test/txt-ndjson.zip delete mode 100644 ignifyr-format-json/src/test/scala/io/ignifyr/format/json/JsonSourceFormatTest.scala delete mode 100644 ignifyr-observability/pom.xml delete mode 100644 ignifyr-observability/src/main/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoder.scala delete mode 100644 ignifyr-observability/src/test/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoderSpec.scala delete mode 100644 ignifyr-redcap/CLAUDE.md delete mode 100644 ignifyr-redcap/pom.xml delete mode 100644 ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension delete mode 100644 ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.server.common.spi.IgnifyrServerExtension delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/ExtractRedCapSchemas.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapEndpoint.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapExtension.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapProjectConfig.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapSchemaImportEndpoint.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServerExtension.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapService.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServiceConfig.scala delete mode 100644 ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapUtil.scala delete mode 100644 ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapExtensionSpec.scala delete mode 100644 ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapUtilDataTypeSpec.scala delete mode 100644 ignifyr-runtime-scheduling/CLAUDE.md delete mode 100644 ignifyr-runtime-scheduling/pom.xml delete mode 100644 ignifyr-runtime-scheduling/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension delete mode 100644 ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/Cron4jSchedulerProvider.scala delete mode 100644 ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/ScheduledJobRegistry.scala delete mode 100644 ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtension.scala delete mode 100644 ignifyr-runtime-scheduling/src/test/resources/application.conf delete mode 100644 ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-drop.sql delete mode 100644 ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-populate.sql delete mode 100644 ignifyr-runtime-scheduling/src/test/resources/test-schedule-mappingjob.json delete mode 100644 ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/integrationtest/SchedulingTest.scala delete mode 100644 ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/ScheduledTimeRangeSpec.scala delete mode 100644 ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtensionSpec.scala delete mode 100644 ignifyr-runtime-streaming/CLAUDE.md delete mode 100644 ignifyr-runtime-streaming/pom.xml delete mode 100644 ignifyr-runtime-streaming/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension delete mode 100644 ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingJobExecutor.scala delete mode 100644 ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingRuntimeExtension.scala delete mode 100644 ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingSinkHandler.scala delete mode 100644 ignifyr-runtime-streaming/src/test/resources/application.conf delete mode 100644 ignifyr-runtime-streaming/src/test/resources/streaming-folder-watch/patients.csv delete mode 100644 ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/KafkaStreamingRedcapTest.scala delete mode 100644 ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/StreamingFolderWatchTest.scala delete mode 100644 ignifyr-runtime-streaming/src/test/scala/io/ignifyr/runtime/streaming/StreamingSinkHandlerTest.scala delete mode 100644 ignifyr-rxnorm/CLAUDE.md delete mode 100644 ignifyr-rxnorm/pom.xml delete mode 100644 ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/PullRxNormNdcMedDetails.scala delete mode 100644 ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiClient.scala delete mode 100644 ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibrary.scala delete mode 100644 ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiClientTest.scala delete mode 100644 ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibraryTest.scala delete mode 100644 ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiStub.scala delete mode 100644 ignifyr-server-common/CLAUDE.md delete mode 100644 ignifyr-server-common/pom.xml delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/config/WebServerConfig.scala delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/ICORSHandler.scala delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/IErrorHandler.scala delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrError.scala delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrRestCall.scala delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtension.scala delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtensions.scala delete mode 100644 ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/SchemaImportSink.scala delete mode 100644 ignifyr-server/CLAUDE.md delete mode 100644 ignifyr-server/api.yaml delete mode 100644 ignifyr-server/pom.xml delete mode 100644 ignifyr-server/src/main/resources/application.conf delete mode 100644 ignifyr-server/src/main/resources/logback.xml delete mode 100644 ignifyr-server/src/main/resources/version.properties delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/Boot.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrHttpServer.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrServer.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/CodeSystemEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ConceptMapEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/IgnifyrServerEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/JobEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingContextEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MetadataEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ProjectEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ReloadEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/SchemaDefinitionEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpoint.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/ExecuteJobTask.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/FilePathNode.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/ImportSchemaSettings.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/InferTask.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/Metadata.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/Project.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/TerminologySystem.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/TestResourceCreationRequest.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/model/csv/CsvHeader.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderDBInitializer.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderRepositoryManager.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/IRepositoryManager.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/IJobRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/JobFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/IMappingRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/ProjectMappingFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/IMappingContextRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/MappingContextFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectList.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/ProjectFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/AbstractSchemaRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/ISchemaRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaManagementUtil.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/ITerminologySystemRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/TerminologySystemFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/CodeSystemFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/ICodeSystemRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/ConceptMapFolderRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/IConceptMapRepository.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/ExecutionService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/JobService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingContextService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/MetadataService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/ProjectService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/ReloadService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/SchemaDefinitionService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/fhir/FileSystemTreeStructureService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/CodeSystemService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/ConceptMapService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/TerminologySystemService.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/util/CsvUtil.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/util/DataFrameUtil.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/util/FileOperations.scala delete mode 100644 ignifyr-server/src/main/scala/io/ignifyr/server/util/IgnifyrRejectionHandler.scala delete mode 100644 ignifyr-server/src/test/resources/application.conf delete mode 100644 ignifyr-server/src/test/resources/fhir-resources/condition-resource.json delete mode 100644 ignifyr-server/src/test/resources/fhir-resources/custom-patient-profile.json delete mode 100644 ignifyr-server/src/test/resources/fhir-resources/custom-profiles.zip delete mode 100644 ignifyr-server/src/test/resources/fhir-resources/fhir-profiles-missing-referenced-schemas.zip delete mode 100644 ignifyr-server/src/test/resources/fhir-resources/invalid-condition-resource.json delete mode 100644 ignifyr-server/src/test/resources/fhir-resources/patient-bundle.json delete mode 100644 ignifyr-server/src/test/resources/fhir-resources/patient-resource.json delete mode 100644 ignifyr-server/src/test/resources/file-system-structure/file1.txt delete mode 100644 ignifyr-server/src/test/resources/file-system-structure/file2.txt delete mode 100644 ignifyr-server/src/test/resources/file-system-structure/subdir/file3.txt delete mode 100644 ignifyr-server/src/test/resources/logback.xml delete mode 100644 ignifyr-server/src/test/resources/redcap/instrument.csv delete mode 100644 ignifyr-server/src/test/resources/sql/sql-source-drop.sql delete mode 100644 ignifyr-server/src/test/resources/sql/sql-source-populate.sql delete mode 100644 ignifyr-server/src/test/resources/streaming-parent-folder/patients/patients.csv delete mode 100644 ignifyr-server/src/test/resources/test-data/other-observations.csv delete mode 100644 ignifyr-server/src/test/resources/test-data/patient-gender-simple.csv delete mode 100644 ignifyr-server/src/test/resources/test-data/patient-simple.csv delete mode 100644 ignifyr-server/src/test/resources/test-data/patients.csv delete mode 100644 ignifyr-server/src/test/resources/test-mapping-context/sample-unit-conversion.csv delete mode 100644 ignifyr-server/src/test/resources/test-mapping-job/patient-mapping-job-with-two-sources.json delete mode 100644 ignifyr-server/src/test/resources/test-mappings/other-observation-concept-map.csv delete mode 100644 ignifyr-server/src/test/resources/test-mappings/other-observation-mapping.json delete mode 100644 ignifyr-server/src/test/resources/test-mappings/other-observation-mapping2.json delete mode 100644 ignifyr-server/src/test/resources/test-mappings/patient-flat-mapping.json delete mode 100644 ignifyr-server/src/test/resources/test-mappings/patient-mapping-with-two-sources.json delete mode 100644 ignifyr-server/src/test/resources/test-mappings/patient-mapping.json delete mode 100644 ignifyr-server/src/test/resources/test-mappings/patient-mapping2.json delete mode 100644 ignifyr-server/src/test/resources/test-schemas/blood-pressure-schema.json delete mode 100644 ignifyr-server/src/test/resources/test-schemas/other-observation-schema.json delete mode 100644 ignifyr-server/src/test/resources/test-schemas/patient-schema.json delete mode 100644 ignifyr-server/src/test/resources/test-terminology-service/sample-code-system.csv delete mode 100644 ignifyr-server/src/test/resources/test-terminology-service/sample-concept-map.csv delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/integrationtest/FhirDefinitionsEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/integrationtest/MappingExecutionEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/integrationtest/SchemaEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/BaseEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrErrorHandlerTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrRejectionHandlerTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobExecutionControlEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingContextEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MetadataEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ProjectEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ReloadingEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpointTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/repository/FolderDBInitializerTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/service/ExecutionServiceTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/service/MetadataServiceTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/util/CsvUtilTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/util/DataFrameUtilTest.scala delete mode 100644 ignifyr-server/src/test/scala/io/ignifyr/server/util/TestUtil.scala delete mode 100644 ignifyr-sink-omop/pom.xml delete mode 100644 ignifyr-sink-omop/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension delete mode 100644 ignifyr-sink-omop/src/main/scala/io/ignifyr/sink/omop/OmopSinkExtension.scala delete mode 100644 ignifyr-sink-omop/src/test/scala/io/ignifyr/sink/omop/OmopSinkExtensionSpec.scala delete mode 100644 ignifyr-terminology-tools/pom.xml delete mode 100644 ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/Concept.scala delete mode 100644 ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/GeneratorDBAdapter.scala delete mode 100644 ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/TerminologyMappingGenerator.scala delete mode 100644 ignifyr-terminology-tools/src/test/scala/io/ignifyr/tools/terminology/TerminologyMappingGeneratorTest.scala delete mode 100644 test-flow/config/ignifyr-server.conf delete mode 100644 test-flow/config/jobs/archive-file-job.json delete mode 100644 test-flow/config/jobs/batch-file-job.json delete mode 100644 test-flow/config/jobs/scheduling-file-job.json delete mode 100644 test-flow/config/jobs/sql-patient-job.json delete mode 100644 test-flow/create-ui-testflow.sh delete mode 100644 test-flow/data/patients.csv delete mode 100644 test-flow/data/redcap-patients.ndjson delete mode 100644 test-flow/data/sql-source-seed.sql delete mode 100644 test-flow/data/stream-patients.csv delete mode 100644 test-flow/docker-compose.test.yml delete mode 100644 test-flow/nginx.conf delete mode 100644 test-flow/run-manual-flow.sh diff --git a/docker/docker-compose-efk.yml b/docker/docker-compose-efk.yml deleted file mode 100644 index 7936772a..00000000 --- a/docker/docker-compose-efk.yml +++ /dev/null @@ -1,35 +0,0 @@ -services: - # Elasticsearch, Fluentd and Kibana stack for mapping job log management - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.19 - container_name: elasticsearch - environment: - - discovery.type=single-node - ports: - - "9200:9200" - volumes: - - elasticsearch_data:/usr/share/elasticsearch/data - fluentd: - build: ./fluentd - container_name: fluentd - image: docker.srdc.com.tr/srdc/ignifyr-fluentd:latest - environment: - - FORWARD_PORT=24224 - - HOST=elasticsearch - - PORT=9200 - ports: - - "24224:24224" - - "24224:24224/udp" - kibana: - build: - context: ./kibana - dockerfile: ./Dockerfile - image: docker.srdc.com.tr/srdc/ignifyr-kibana:latest - container_name: kibana - ports: - - "5601:5601" - depends_on: - - elasticsearch -volumes: - elasticsearch_data: - driver: local \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 98930ff3..00000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: '3.9' -services: - ignifyr-server: - image: srdc/ignifyr-server:latest - container_name: ignifyr-server - hostname: ignifyr-server - environment: - - FHIR_DEFINITIONS_ENDPOINT=http://onfhir:8080/fhir - - FHIR_REPO_URL=http://onfhir:8080/fhir - - APP_CONF_FILE=/usr/local/ignifyr/conf/docker/ignifyr-server.conf - - FLUENT_HOST=fluentd - - CONTEXT_PATH=conf - ports: - - "8085:8085" - networks: - - ignifyr-network - volumes: - - './data-integration-suite:/usr/local/ignifyr/conf' - - './ignifyr-docker-logs:/usr/local/ignifyr/logs' - ignifyr-web: - image: docker.srdc.com.tr/srdc/tofhir-web:latest - container_name: ignifyr-web - ports: - - "8087:80" - networks: - - ignifyr-network - # Elasticsearch, Fluentd and Kibana stack for mapping job log management - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.19 - container_name: elasticsearch - environment: - - discovery.type=single-node - ports: - - "9200:9200" - networks: - - ignifyr-network - volumes: - - elasticsearch_data:/usr/share/elasticsearch/data - fluentd: - build: ./fluentd - container_name: fluentd - image: docker.srdc.com.tr/srdc/ignifyr-fluentd:latest - environment: - - FORWARD_PORT=24224 - - HOST=elasticsearch - - PORT=9200 - ports: - - "24224:24224" - - "24224:24224/udp" - networks: - - ignifyr-network - kibana: - build: - context: ./kibana - dockerfile: ./Dockerfile - image: docker.srdc.com.tr/srdc/ignifyr-kibana:latest - container_name: kibana - ports: - - "5601:5601" - networks: - - ignifyr-network - depends_on: - - elasticsearch -networks: - ignifyr-network: - name: onfhir-network - external: true -volumes: - elasticsearch_data: - driver: local \ No newline at end of file diff --git a/docker/fluentd/Dockerfile b/docker/fluentd/Dockerfile deleted file mode 100644 index 4aa68d0b..00000000 --- a/docker/fluentd/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -#Specifies the base image for building the Fluentd Docker image -FROM fluent/fluentd:v1.17-1 -#Switches to the root user account to perform package installations and configurations. -USER root -#Installs necessary packages and dependencies using the Alpine package manager (apk). -RUN apk add --no-cache --update --virtual .build-deps \ - sudo build-base ruby-dev \ - && sudo gem install faraday \ - && sudo gem install faraday-net_http \ - && sudo gem install fluent-plugin-elasticsearch \ - && sudo gem sources --clear-all \ - && apk del .build-deps \ - && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem -# Switches back to the fluent user account to run Fluentd. -USER fluent -# Copies Fluentd configuration file into the image -COPY ./docker/fluentd/conf/fluent.conf /fluentd/etc/fluent.conf \ No newline at end of file diff --git a/docker/fluentd/build.sh b/docker/fluentd/build.sh deleted file mode 100644 index 680d9e07..00000000 --- a/docker/fluentd/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Execute the following command from the project.root.directory (../../) - -docker build -f docker/fluentd/Dockerfile -t srdc/ignifyr-fluentd:latest . \ No newline at end of file diff --git a/docker/fluentd/conf/fluent.conf b/docker/fluentd/conf/fluent.conf deleted file mode 100644 index 57e7c647..00000000 --- a/docker/fluentd/conf/fluent.conf +++ /dev/null @@ -1,23 +0,0 @@ -// Defines a Fluentd configuration for forwarding logs to Elasticsearch. - - - // Specifies the input plugin type as 'forward', which receives log entries forwarded by Fluentd clients. - @type forward - // Specifies the port on which the forward input plugin listens for incoming log entries. - port "#{ENV['FORWARD_PORT']}" - - - - // Specifies the output plugin type as 'elasticsearch', which sends log entries to an Elasticsearch cluster. - @type elasticsearch - // Specifies the host address of the Elasticsearch cluster. - host "#{ENV['HOST']}" - // Specifies the port number of the Elasticsearch cluster. - port "#{ENV['PORT']}" - // Configures the output format to be compatible with Logstash. - logstash_format true - // Specifies the prefix to be added to the index name in Elasticsearch. - logstash_prefix fluentd - // Specifies the interval at which buffered log entries are flushed to Elasticsearch. - flush_interval 5s - \ No newline at end of file diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile deleted file mode 100644 index 398b04a6..00000000 --- a/docker/kibana/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM docker.elastic.co/kibana/kibana:7.17.19 - -RUN mkdir -p /tmp/sample/ -# Copy default dashboards -COPY ./docker/kibana/exports/ /tmp/sample/ -# Copy custom kibana.yml file -COPY ./docker/kibana/kibana.yml /usr/share/kibana/config/ -# Copy initialization script -COPY ./docker/kibana/init-script.sh /tmp/sample/init-script.sh -USER root -# give access to files -RUN chmod -R 777 /tmp/sample/* -# Run -CMD ["/bin/bash", "-c", "kibana-docker --allow-root & /tmp/sample/init-script.sh"] - diff --git a/docker/kibana/build.sh b/docker/kibana/build.sh deleted file mode 100644 index 6fcc034e..00000000 --- a/docker/kibana/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Execute the following command from the project.root.directory (../../) - -docker build -f docker/kibana/Dockerfile -t srdc/ignifyr-kibana:latest . \ No newline at end of file diff --git a/docker/kibana/exports/dashboards.ndjson b/docker/kibana/exports/dashboards.ndjson deleted file mode 100644 index 69ce28bf..00000000 --- a/docker/kibana/exports/dashboards.ndjson +++ /dev/null @@ -1,15 +0,0 @@ -{"attributes":{"buildNum":44266,"defaultIndex":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5"},"coreMigrationVersion":"7.17.19","id":"7.15.2","migrationVersion":{"config":"7.13.0"},"references":[],"type":"config","updated_at":"2026-03-02T10:15:06.315Z","version":"WzYsMV0="} -{"attributes":{"buildNum":47514,"defaultIndex":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5"},"coreMigrationVersion":"7.17.19","id":"7.17.19","migrationVersion":{"config":"7.13.0"},"references":[],"type":"config","updated_at":"2026-03-02T10:15:06.315Z","version":"WzcsMV0="} -{"attributes":{"fieldAttrs":"{\"result\":{\"count\":7},\"isCompleted\":{\"count\":3},\"isScheduled\":{\"count\":5},\"isScheduledJob\":{\"count\":3},\"normalizedNumOfFhirResources\":{\"count\":3},\"chunkResult\":{\"count\":5},\"mappingTaskName\":{\"count\":2},\"isStreamingJob\":{\"count\":2},\"@timestamp-original\":{\"count\":2},\"chunkProgress\":{\"count\":2},\"batchProgress\":{\"count\":3},\"message\":{\"count\":1}}","fieldFormatMap":"{\"@timestamp-original\":{\"id\":\"string\",\"params\":{\"parsedUrl\":{\"origin\":\"http://localhost:5601\",\"pathname\":\"/app/management/data/index_management/indices\",\"basePath\":\"\"},\"pattern\":\"0,0.[000]\"}},\"isScheduled\":{\"id\":\"boolean\",\"params\":{\"parsedUrl\":{\"origin\":\"http://localhost:5601\",\"pathname\":\"/app/home\",\"basePath\":\"\"},\"pattern\":\"0,0.[000]\"}},\"isCompleted\":{\"id\":\"boolean\",\"params\":{\"parsedUrl\":{\"origin\":\"http://localhost:5601\",\"pathname\":\"/app/home\",\"basePath\":\"\"},\"pattern\":\"0,0.[000]\"}}}","fields":"[{\"count\":2,\"script\":\"doc['@timestamp']\",\"lang\":\"painless\",\"name\":\"@timestamp-original\",\"type\":\"string\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"script\":\"if(doc['numOfNotMapped'].size() == 0){\\n return 0;\\n}\\nelse if(doc['numOfNotMapped'].value == -1){\\n return 0;\\n}\\nelse {\\n return doc['numOfNotMapped'].value;\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfNotMapped\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"script\":\"if(doc['numOfFailedWrites'].size() == 0){\\r\\n return 0;\\r\\n}\\r\\nelse if(doc['numOfFailedWrites'].value == -1){\\r\\n return 0;\\r\\n}\\r\\nelse {\\r\\n return doc['numOfFailedWrites'].value;\\r\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfFailedWrites\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"script\":\"if(doc['numOfInvalids'].size() == 0){\\r\\n return 0;\\r\\n}\\r\\nelse if(doc['numOfInvalids'].value == -1){\\r\\n return 0;\\r\\n}\\r\\nelse {\\r\\n return doc['numOfInvalids'].value;\\r\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfInvalids\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":3,\"script\":\"if(doc['numOfFhirResources'].size() == 0){\\r\\n return 0;\\r\\n}\\r\\nelse if(doc['numOfFhirResources'].value == -1){\\r\\n return 0;\\r\\n}\\r\\nelse {\\r\\n return doc['numOfFhirResources'].value;\\r\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfFhirResources\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":5,\"script\":\"if(doc['result.keyword'].size() == 0){ \\r\\n return null; \\r\\n} \\r\\n\\r\\nif(doc['result.keyword'].value == \\\"DESCHEDULED\\\") { \\r\\n return false; \\r\\n \\r\\n} else if(doc['isScheduledJob'].size() > 0 && doc['isScheduledJob'].value){ \\r\\n return true; \\r\\n \\r\\n} else {\\r\\n return false;\\r\\n}\",\"lang\":\"painless\",\"name\":\"isScheduled\",\"type\":\"boolean\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":3,\"script\":\"if(doc['result.keyword'].size() == 0) {\\r\\n return false;\\r\\n}\\r\\nif(doc['isStreamingJob'].size() == 0 || doc['isStreamingJob'].value == false){\\r\\n return (doc['chunkResult'].size() > 0 && doc['chunkResult'].value == false && doc['result.keyword'].value != \\\"STARTED\\\") || doc['result.keyword'].value == \\\"STOPPED\\\";\\r\\n}\\r\\nelse {\\r\\n return doc['result.keyword'].value == \\\"FAILURE\\\" || doc['result.keyword'].value == \\\"STOPPED\\\";\\r\\n}\",\"lang\":\"painless\",\"name\":\"isCompleted\",\"type\":\"boolean\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"fluentd*","typeMeta":"{}"},"coreMigrationVersion":"7.17.19","id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2026-03-02T10:15:06.315Z","version":"WzExLDFd"} -{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.17.19\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":3,\"i\":\"ecd350a2-0317-4f66-b2cb-80596e4a0689\"},\"panelIndex\":\"ecd350a2-0317-4f66-b2cb-80596e4a0689\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"\",\"description\":\"\",\"type\":\"markdown\",\"params\":{\"fontSize\":12,\"openLinksInNewTab\":false,\"markdown\":\"Go back to [Executions Dashboard](http://localhost:5601/goto/2a517919d8081cb04ccd2c61a0e1c74a)\\n\"},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}}},\"enhancements\":{}}},{\"version\":\"7.17.19\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":3,\"w\":48,\"h\":14,\"i\":\"9f083d19-1515-40d5-b078-918dd8b0f5ed\"},\"panelIndex\":\"9f083d19-1515-40d5-b078-918dd8b0f5ed\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-layer-045875b2-5639-4e09-a9f0-06a2bc94627c\"}],\"state\":{\"visualization\":{\"layerId\":\"045875b2-5639-4e09-a9f0-06a2bc94627c\",\"layerType\":\"data\",\"columns\":[{\"isTransposed\":false,\"columnId\":\"a9105d68-f41f-4a6a-bcf2-88afb2753911\",\"width\":297.83333333333337},{\"isTransposed\":false,\"columnId\":\"200d4609-5416-4b35-9eb7-bbf58e4e940b\",\"hidden\":true},{\"isTransposed\":false,\"columnId\":\"e71b0e35-d05f-47c0-bac4-80e0c501d6cc\",\"width\":157.83333333333334},{\"isTransposed\":false,\"columnId\":\"5d44faff-382c-484d-9fa8-d4660f26805e\",\"width\":166.63333333333333},{\"isTransposed\":false,\"columnId\":\"bda664e6-f527-4e55-880a-be71839aef3a\"},{\"isTransposed\":false,\"columnId\":\"08108aa9-c895-48be-b8c6-5888ef415407\"},{\"columnId\":\"f431fd4e-b333-4ec2-a4b6-2cebf08b179c\",\"isTransposed\":false},{\"columnId\":\"73b24b5b-26ec-44d2-9f43-206fb27d7b72\",\"isTransposed\":false},{\"columnId\":\"74a3433e-26ac-4b72-bf24-cb034c7e74d5\",\"isTransposed\":false,\"hidden\":false,\"width\":224.16666666666666,\"alignment\":\"left\"},{\"columnId\":\"fc661f33-4342-444b-8cd4-1ab9f0170e55\",\"isTransposed\":false}],\"sorting\":{\"columnId\":\"74a3433e-26ac-4b72-bf24-cb034c7e74d5\",\"direction\":\"desc\"}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"045875b2-5639-4e09-a9f0-06a2bc94627c\":{\"columns\":{\"a9105d68-f41f-4a6a-bcf2-88afb2753911\":{\"label\":\"Mapping Task Name\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"mappingTaskName.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"200d4609-5416-4b35-9eb7-bbf58e4e940b\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"200d4609-5416-4b35-9eb7-bbf58e4e940b\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\"},\"08108aa9-c895-48be-b8c6-5888ef415407X0\":{\"label\":\"Part of Num Of Fhir Resources\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfFhirResources\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"chunkResult : true\",\"language\":\"kuery\"},\"customLabel\":true},\"08108aa9-c895-48be-b8c6-5888ef415407\":{\"label\":\"Num Of Fhir Resources\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"sum(normalizedNumOfFhirResources, kql='chunkResult : true')\",\"isFormulaBroken\":false},\"references\":[\"08108aa9-c895-48be-b8c6-5888ef415407X0\"],\"customLabel\":true},\"bda664e6-f527-4e55-880a-be71839aef3aX0\":{\"label\":\"Part of Num Of Failed Writes\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfFailedWrites\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"chunkResult : true\",\"language\":\"kuery\"},\"customLabel\":true},\"bda664e6-f527-4e55-880a-be71839aef3a\":{\"label\":\"Num Of Failed Writes\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"sum(normalizedNumOfFailedWrites, kql='chunkResult : true')\",\"isFormulaBroken\":false},\"references\":[\"bda664e6-f527-4e55-880a-be71839aef3aX0\"],\"customLabel\":true},\"5d44faff-382c-484d-9fa8-d4660f26805eX0\":{\"label\":\"Part of Num Of Not Mapped\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfNotMapped\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"chunkResult : true\",\"language\":\"kuery\"},\"customLabel\":true},\"5d44faff-382c-484d-9fa8-d4660f26805e\":{\"label\":\"Num Of Not Mapped\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"sum(normalizedNumOfNotMapped, kql='chunkResult : true')\",\"isFormulaBroken\":false},\"references\":[\"5d44faff-382c-484d-9fa8-d4660f26805eX0\"],\"customLabel\":true},\"73b24b5b-26ec-44d2-9f43-206fb27d7b72\":{\"label\":\"Chunk Progress\",\"dataType\":\"string\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ordinal\",\"sourceField\":\"chunkProgress.keyword\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true,\"filter\":{\"query\":\"chunkProgress.keyword: * \",\"language\":\"kuery\"}},\"f431fd4e-b333-4ec2-a4b6-2cebf08b179c\":{\"label\":\"Result\",\"dataType\":\"string\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ordinal\",\"sourceField\":\"result.keyword\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"74a3433e-26ac-4b72-bf24-cb034c7e74d5\":{\"label\":\"Timestamp\",\"dataType\":\"string\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ordinal\",\"sourceField\":\"@timestamp-original\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"e71b0e35-d05f-47c0-bac4-80e0c501d6ccX0\":{\"label\":\"Part of Num Of Invalids\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfInvalids\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"chunkResult : true\",\"language\":\"kuery\"},\"customLabel\":true},\"e71b0e35-d05f-47c0-bac4-80e0c501d6cc\":{\"label\":\"Num Of Invalids\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"sum(normalizedNumOfInvalids, kql='chunkResult : true')\",\"isFormulaBroken\":false},\"references\":[\"e71b0e35-d05f-47c0-bac4-80e0c501d6ccX0\"],\"customLabel\":true},\"fc661f33-4342-444b-8cd4-1ab9f0170e55\":{\"label\":\"Batch Progress\",\"dataType\":\"string\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ordinal\",\"sourceField\":\"batchProgress.keyword\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true,\"filter\":{\"query\":\"batchProgress.keyword : *\",\"language\":\"kuery\"}}},\"columnOrder\":[\"a9105d68-f41f-4a6a-bcf2-88afb2753911\",\"200d4609-5416-4b35-9eb7-bbf58e4e940b\",\"74a3433e-26ac-4b72-bf24-cb034c7e74d5\",\"f431fd4e-b333-4ec2-a4b6-2cebf08b179c\",\"fc661f33-4342-444b-8cd4-1ab9f0170e55\",\"73b24b5b-26ec-44d2-9f43-206fb27d7b72\",\"e71b0e35-d05f-47c0-bac4-80e0c501d6cc\",\"5d44faff-382c-484d-9fa8-d4660f26805e\",\"bda664e6-f527-4e55-880a-be71839aef3a\",\"08108aa9-c895-48be-b8c6-5888ef415407\",\"08108aa9-c895-48be-b8c6-5888ef415407X0\",\"bda664e6-f527-4e55-880a-be71839aef3aX0\",\"5d44faff-382c-484d-9fa8-d4660f26805eX0\",\"e71b0e35-d05f-47c0-bac4-80e0c501d6ccX0\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Mapping Tasks\"},{\"version\":\"7.17.19\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":17,\"w\":48,\"h\":15,\"i\":\"35afbd9b-edce-4523-ba9c-60f93a24e7d3\"},\"panelIndex\":\"35afbd9b-edce-4523-ba9c-60f93a24e7d3\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"\",\"description\":\"\",\"type\":\"table\",\"params\":{\"perPage\":1000,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"showToolbar\":false,\"autoFitRowToContent\":true},\"uiState\":{},\"data\":{\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"mappingTaskName.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Task\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"errorCode.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Code\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"errorDesc.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Description\"},\"schema\":\"bucket\"},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"errorExpr.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"\",\"customLabel\":\"Expression\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source\"},\"schema\":\"bucket\"},{\"id\":\"7\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"mappedResource.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Mapped Resource\"},\"schema\":\"bucket\"}],\"searchSource\":{\"index\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}}},\"hidePanelTitles\":false,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}},\"table\":null,\"enhancements\":{}},\"title\":\"Mapping Errors\"},{\"version\":\"7.17.19\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":32,\"w\":48,\"h\":15,\"i\":\"c2380600-bd3e-4cd8-a87f-824ea16d2bfb\"},\"panelIndex\":\"c2380600-bd3e-4cd8-a87f-824ea16d2bfb\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"\",\"description\":\"\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"showToolbar\":false,\"autoFitRowToContent\":true},\"uiState\":{},\"data\":{\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"mappingTaskName.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Task\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"throwable.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Stack Trace\"},\"schema\":\"bucket\"}],\"searchSource\":{\"index\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Mapping Task Execution Errors\"}]","refreshInterval":{"pause":false,"value":5000},"timeFrom":"now/d","timeRestore":true,"timeTo":"now/d","title":"Execution Details","version":1},"coreMigrationVersion":"7.17.19","id":"994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5","migrationVersion":{"dashboard":"7.17.3"},"references":[{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"9f083d19-1515-40d5-b078-918dd8b0f5ed:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"9f083d19-1515-40d5-b078-918dd8b0f5ed:indexpattern-datasource-layer-045875b2-5639-4e09-a9f0-06a2bc94627c","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"35afbd9b-edce-4523-ba9c-60f93a24e7d3:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"c2380600-bd3e-4cd8-a87f-824ea16d2bfb:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"dashboard","updated_at":"2026-03-02T13:26:51.699Z","version":"WzM0NzcsMV0="} -{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.17.19\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":16,\"i\":\"d62f4d3d-027b-408d-b8ad-62404b90a057\"},\"panelIndex\":\"d62f4d3d-027b-408d-b8ad-62404b90a057\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca\"}],\"state\":{\"visualization\":{\"layerId\":\"a67e6635-b164-4c45-a563-bbe4ca458fca\",\"layerType\":\"data\",\"columns\":[{\"isTransposed\":false,\"columnId\":\"73954412-33dd-4460-9ed0-25fb11272df8\",\"width\":297.66666666666663},{\"columnId\":\"6c1cf201-6770-49ee-9056-9bc1f009f5da\",\"isTransposed\":false},{\"columnId\":\"567b3cde-4bc5-4155-82bb-8c0f1097af27\",\"isTransposed\":false},{\"columnId\":\"6e4b599c-1d09-46c1-83b7-d66df0337009\",\"isTransposed\":false},{\"columnId\":\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\",\"isTransposed\":false,\"width\":213.66666666666666},{\"columnId\":\"76358e4d-51dc-4bb7-9763-45c422e51e5b\",\"isTransposed\":false},{\"columnId\":\"620de4f7-8ba5-4e46-80ee-2d71bdb3bf01\",\"isTransposed\":false,\"alignment\":\"left\"},{\"columnId\":\"b432ec70-9913-40c5-ac01-37b29490916f\",\"isTransposed\":false}],\"sorting\":{\"columnId\":\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\",\"direction\":\"desc\"}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"a67e6635-b164-4c45-a563-bbe4ca458fca\":{\"columns\":{\"73954412-33dd-4460-9ed0-25fb11272df8\":{\"label\":\"Execution ID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"executionId.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\":{\"label\":\"Timestamp\",\"dataType\":\"string\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ordinal\",\"sourceField\":\"@timestamp-original\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"620de4f7-8ba5-4e46-80ee-2d71bdb3bf01\":{\"label\":\"Streaming Job\",\"dataType\":\"boolean\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"isStreamingJob\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true,\"filter\":{\"query\":\"isStreamingJob : true or isStreamingJob : false \",\"language\":\"kuery\"}},\"b432ec70-9913-40c5-ac01-37b29490916f\":{\"label\":\"Scheduled\",\"dataType\":\"boolean\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"isScheduled\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"6c1cf201-6770-49ee-9056-9bc1f009f5daX0\":{\"label\":\"Part of Success Count\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : SUCCESS and chunkResult : false \",\"language\":\"kuery\"},\"customLabel\":true},\"6c1cf201-6770-49ee-9056-9bc1f009f5da\":{\"label\":\"Success Count\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result : SUCCESS and chunkResult : false ')\",\"isFormulaBroken\":false},\"references\":[\"6c1cf201-6770-49ee-9056-9bc1f009f5daX0\"],\"customLabel\":true},\"567b3cde-4bc5-4155-82bb-8c0f1097af27X0\":{\"label\":\"Part of Failure Count\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result: FAILURE and chunkResult : false \",\"language\":\"kuery\"},\"customLabel\":true},\"567b3cde-4bc5-4155-82bb-8c0f1097af27\":{\"label\":\"Failure Count\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result: FAILURE and chunkResult : false ')\",\"isFormulaBroken\":false},\"references\":[\"567b3cde-4bc5-4155-82bb-8c0f1097af27X0\"],\"customLabel\":true},\"6e4b599c-1d09-46c1-83b7-d66df0337009X0\":{\"label\":\"Part of Partial Success Count\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : PARTIAL_SUCCESS and chunkResult : false \",\"language\":\"kuery\"},\"customLabel\":true},\"6e4b599c-1d09-46c1-83b7-d66df0337009\":{\"label\":\"Partial Success Count\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result : PARTIAL_SUCCESS and chunkResult : false ')\",\"isFormulaBroken\":false},\"references\":[\"6e4b599c-1d09-46c1-83b7-d66df0337009X0\"],\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\":{\"label\":\"Part of Ongoing Tasks\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : STARTED and chunkResult : false\",\"language\":\"kuery\"},\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\":{\"label\":\"Part of Ongoing Tasks\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"isCompleted : true\",\"language\":\"kuery\"},\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX2\":{\"label\":\"Part of Ongoing Tasks\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"subtract\",\"args\":[\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\"],\"location\":{\"min\":0,\"max\":87},\"text\":\"count(kql='result : STARTED and chunkResult : false') - count(kql='isCompleted : true')\"}},\"references\":[\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\"],\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5b\":{\"label\":\"Ongoing Tasks\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result : STARTED and chunkResult : false') - count(kql='isCompleted : true')\",\"isFormulaBroken\":false},\"references\":[\"76358e4d-51dc-4bb7-9763-45c422e51e5bX2\"],\"customLabel\":true}},\"columnOrder\":[\"73954412-33dd-4460-9ed0-25fb11272df8\",\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\",\"620de4f7-8ba5-4e46-80ee-2d71bdb3bf01\",\"b432ec70-9913-40c5-ac01-37b29490916f\",\"76358e4d-51dc-4bb7-9763-45c422e51e5b\",\"6c1cf201-6770-49ee-9056-9bc1f009f5da\",\"567b3cde-4bc5-4155-82bb-8c0f1097af27\",\"6e4b599c-1d09-46c1-83b7-d66df0337009\",\"6c1cf201-6770-49ee-9056-9bc1f009f5daX0\",\"567b3cde-4bc5-4155-82bb-8c0f1097af27X0\",\"6e4b599c-1d09-46c1-83b7-d66df0337009X0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX2\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{\"dynamicActions\":{\"events\":[{\"eventId\":\"78a728c0-3ead-4511-8742-5840b4e891b3\",\"triggers\":[\"FILTER_TRIGGER\"],\"action\":{\"factoryId\":\"DASHBOARD_TO_DASHBOARD_DRILLDOWN\",\"name\":\"Go to Dashboard\",\"config\":{\"useCurrentFilters\":true,\"useCurrentDateRange\":true}}}]}}}}]","refreshInterval":{"pause":false,"value":5000},"timeFrom":"now/d","timeRestore":true,"timeTo":"now/d","title":"Executions","version":1},"coreMigrationVersion":"7.17.19","id":"0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5","migrationVersion":{"dashboard":"7.17.3"},"references":[{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"d62f4d3d-027b-408d-b8ad-62404b90a057:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"d62f4d3d-027b-408d-b8ad-62404b90a057:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca","type":"index-pattern"},{"id":"994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5","name":"d62f4d3d-027b-408d-b8ad-62404b90a057:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId","type":"dashboard"}],"type":"dashboard","updated_at":"2026-03-02T10:15:06.315Z","version":"WzEzLDFd"} -{"attributes":{"accessCount":1,"accessDate":1713774467694,"createDate":1713774391835,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:asc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:edit)"},"coreMigrationVersion":"7.17.19","id":"10e336e54901315dfa88c7b31b0a46e0","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzE0LDFd"} -{"attributes":{"accessCount":3,"accessDate":1720505861508,"createDate":1714045492705,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(%2773954412-33dd-4460-9ed0-25fb11272df8%27,%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27,%276e4b599c-1d09-46c1-83b7-d66df0337009%27,%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),columns:(%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Failure%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result:%20FAILURE!%27)%27,isFormulaBroken:!f),references:!(%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27),scale:ratio),%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Failure%20Count%27,operationType:count,scale:ratio,sourceField:Records),%276c1cf201-6770-49ee-9056-9bc1f009f5da%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27),scale:ratio),%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20count(kql%3D!%27result%20:%20SUCCESS!%27)%27,operationType:count,scale:ratio,sourceField:Records),%276e4b599c-1d09-46c1-83b7-d66df0337009%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Partial%20Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27),scale:ratio),%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Partial%20Success%20Count%27,operationType:count,scale:ratio,sourceField:Records),%2773954412-33dd-4460-9ed0-25fb11272df8%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(fallback:!t,type:alphabetical),orderDirection:asc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword),%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Ongoing%20Tasks%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),scale:ratio),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20STARTED%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:math,params:(tinymathAst:(args:!((args:!((args:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),location:(max:132,min:0),name:subtract,text:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,type:function)),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),scale:ratio),%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27:(customLabel:!t,dataType:string,isBucketed:!f,label:Timestamp,operationType:last_value,params:(sortField:%27@timestamp%27),scale:ordinal,sourceField:%27@timestamp-original%27)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:%2773954412-33dd-4460-9ed0-25fb11272df8%27,isTransposed:!f,width:297.66666666666663),(columnId:%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,isTransposed:!f),(columnId:%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,isTransposed:!f),(columnId:%276e4b599c-1d09-46c1-83b7-d66df0337009%27,isTransposed:!f),(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,isTransposed:!f,width:213.66666666666666),(columnId:%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:16,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:48,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.17.19","id":"2a517919d8081cb04ccd2c61a0e1c74a","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzE1LDFd"} -{"attributes":{"accessCount":4,"accessDate":1713268236271,"createDate":1713268122764,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:edit)&show-time-filter=true"},"coreMigrationVersion":"7.17.19","id":"30f7ebeef8b48c64c79da05ff1588bba","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzE2LDFd"} -{"attributes":{"accessCount":0,"accessDate":1713268354177,"createDate":1713268354177,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)"},"coreMigrationVersion":"7.17.19","id":"37a997d487481a422d52a3588547cb58","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzE3LDFd"} -{"attributes":{"accessCount":1,"accessDate":1713271630902,"createDate":1713271613093,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:asc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-top-menu=true&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.17.19","id":"41b1fa963c2ae0c20701f43be4c3a7e6","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzE4LDFd"} -{"attributes":{"accessCount":21,"accessDate":1714045364167,"createDate":1713271648005,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:asc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.17.19","id":"797a37b8b559f872a4933989160014cb","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzE5LDFd"} -{"attributes":{"accessCount":0,"accessDate":1713268357275,"createDate":1713268357275,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-time-filter=true"},"coreMigrationVersion":"7.17.19","id":"97e24daa9135b3f09857761fa490e663","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzIwLDFd"} -{"attributes":{"accessCount":2,"accessDate":1714045457802,"createDate":1714045392621,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(%2773954412-33dd-4460-9ed0-25fb11272df8%27,%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27,%276e4b599c-1d09-46c1-83b7-d66df0337009%27,%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),columns:(%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Failure%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result:%20FAILURE!%27)%27,isFormulaBroken:!f),references:!(%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27),scale:ratio),%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Failure%20Count%27,operationType:count,scale:ratio,sourceField:Records),%276c1cf201-6770-49ee-9056-9bc1f009f5da%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27),scale:ratio),%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20count(kql%3D!%27result%20:%20SUCCESS!%27)%27,operationType:count,scale:ratio,sourceField:Records),%276e4b599c-1d09-46c1-83b7-d66df0337009%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Partial%20Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27),scale:ratio),%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Partial%20Success%20Count%27,operationType:count,scale:ratio,sourceField:Records),%2773954412-33dd-4460-9ed0-25fb11272df8%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(fallback:!t,type:alphabetical),orderDirection:asc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword),%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Ongoing%20Tasks%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),scale:ratio),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20STARTED%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:math,params:(tinymathAst:(args:!((args:!((args:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),location:(max:132,min:0),name:subtract,text:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,type:function)),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),scale:ratio),%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27:(customLabel:!t,dataType:string,isBucketed:!f,label:Timestamp,operationType:last_value,params:(sortField:%27@timestamp%27),scale:ordinal,sourceField:%27@timestamp-original%27)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:%2773954412-33dd-4460-9ed0-25fb11272df8%27,isTransposed:!f,width:297.66666666666663),(columnId:%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,isTransposed:!f),(columnId:%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,isTransposed:!f),(columnId:%276e4b599c-1d09-46c1-83b7-d66df0337009%27,isTransposed:!f),(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,isTransposed:!f,width:213.66666666666666),(columnId:%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:16,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:48,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)"},"coreMigrationVersion":"7.17.19","id":"c8e40b39e912579ca0ccfd05c90d3302","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzIxLDFd"} -{"attributes":{"accessCount":22,"accessDate":1714047836559,"createDate":1713268358357,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.17.19","id":"0c52344e47af8f96bbb54b2c911cbdf7","references":[],"type":"url","updated_at":"2026-03-02T10:15:06.315Z","version":"WzIyLDFd"} -{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":14,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/docker/kibana/exports/ignore_above_index_template.json b/docker/kibana/exports/ignore_above_index_template.json deleted file mode 100644 index c4d379f6..00000000 --- a/docker/kibana/exports/ignore_above_index_template.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "index_patterns": [ - "fluentd*" - ], - "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "match_mapping_type": "string", - "mapping": { - "type": "text", - "fields": { - "keyword": { - "ignore_above": 10000, - "type": "keyword" - } - } - } - } - } - ] - } -} \ No newline at end of file diff --git a/docker/kibana/exports/required_fields_index_template.json b/docker/kibana/exports/required_fields_index_template.json deleted file mode 100644 index eb71e69c..00000000 --- a/docker/kibana/exports/required_fields_index_template.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "index_patterns": [ - "fluentd*" - ], - "mappings": { - "properties": { - "executionId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "mappingTaskName": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "mappedResource": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "result": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "errorCode": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "errorDesc": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "isStreamingJob": { - "type": "boolean" - }, - "isScheduledJob": { - "type": "boolean" - }, - "chunkResult": { - "type": "boolean" - }, - "chunkProgress": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "batchProgress": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "errorExpr": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "source": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - }, - "throwable": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword" - } - } - } - } - } -} \ No newline at end of file diff --git a/docker/kibana/init-script.sh b/docker/kibana/init-script.sh deleted file mode 100644 index 706bb2e0..00000000 --- a/docker/kibana/init-script.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Wait for Elasticsearch to start up before doing anything. -until curl -s http://elasticsearch:9200/_cat/health -o /dev/null; do - echo Waiting for Elasticsearch... - sleep 5 -done - -# The following index templates should be established prior to importing the Saved Objects to ensure their configurations are applied to new indexes correctly. - -# Send a request to apply the ignore_above template in Elasticsearch. -# This template increases the default maximum length of (256 characters) text types, and thanks to it, we can -# utilize them in dashboard visualizations. -curl -X PUT "http://elasticsearch:9200/_template/ignore_above" -H 'Content-Type: application/json' -d @/tmp/sample/ignore_above_index_template.json -# Certain fields are required for Kibana Dashboards to initialize properly. -# These fields are populated using the index template. Without such an index template, the dashboards cannot be initialized -# properly until the first data entry is sent to Elasticsearch. Since the dashboards are not initialized correctly, -# the refresh functionality does not work, even if new data is sent to Elasticsearch (You have the reload the page). -curl -X PUT "http://elasticsearch:9200/_template/required_fields" -H 'Content-Type: application/json' -d @/tmp/sample/required_fields_index_template.json - -# Function to check if Kibana is ready -check_kibana() { - # Make a request to Kibana's status API and check the response - status=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5601/api/status) - # If the status is 200 (OK), Kibana is ready - if [ "$status" -eq 200 ]; then - return 0 - else - return 1 - fi -} - -# Wait for Kibana to be ready -until check_kibana; do - echo "Waiting for Kibana..." - sleep 5 -done - -# Import default dashboards using the Saved Objects API -curl -X POST "http://localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" -H 'Content-Type: multipart/form-data' --form file=@/tmp/sample/dashboards.ndjson -# Disable telemetry i.e. Help us improve the Elastic Stack notification -curl -X POST "http://localhost:5601/api/telemetry/v2/optIn" -H "kbn-xsrf: true" -H 'Content-Type: application/json' -d '{"enabled":false}' - -echo $'\nInitialization complete. Keeping container running for Kibana...' -tail -f /dev/null \ No newline at end of file diff --git a/docker/kibana/kibana.yml b/docker/kibana/kibana.yml deleted file mode 100644 index 1764c85d..00000000 --- a/docker/kibana/kibana.yml +++ /dev/null @@ -1,12 +0,0 @@ -# The host of the Kibana server. "0" binds to all network interfaces. -server.host: "0" -# The duration for Kibana to wait for in-flight requests before shutting down. -server.shutdownTimeout: "5s" -# Elasticsearch hosts Kibana will connect to -elasticsearch.hosts: [ "http://elasticsearch:9200" ] -# Enables the monitoring UI in Kibana for Elasticsearch containers -monitoring.ui.container.elasticsearch.enabled: true -# Disable insecure cluster warning -security.showInsecureClusterWarning: false -# Request maximum payload size. Default = 1048576 byte (~1MB) -server.maxPayload: "104857600" \ No newline at end of file diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile deleted file mode 100644 index c9abae05..00000000 --- a/docker/server/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM eclipse-temurin:11-jre - -# We need curl for healthcheck -RUN apt -y update; apt -y install curl - -ENV IGNIFYR_HOME=/usr/local/ignifyr -RUN mkdir -p "$IGNIFYR_HOME" -WORKDIR $IGNIFYR_HOME - -COPY ./ignifyr-server/target/ignifyr-server-standalone.jar . -COPY ./docker/server/docker-entrypoint.sh . -RUN chmod +x docker-entrypoint.sh - -ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/docker/server/build.sh b/docker/server/build.sh deleted file mode 100644 index 23d3caad..00000000 --- a/docker/server/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Execute one of the following commands from the project.root.directory (../../) - -docker build -f docker/server/Dockerfile -t srdc/ignifyr-server:latest . - diff --git a/docker/server/docker-entrypoint.sh b/docker/server/docker-entrypoint.sh deleted file mode 100644 index 18a99148..00000000 --- a/docker/server/docker-entrypoint.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash - -# Set default Java options -DEFAULT_JAVA_OPTIONS="-Xms256m -Xmx4g" - -# Use environment variable if provided, otherwise use default value -JAVA_OPTIONS="${JAVA_OPTIONS:-$DEFAULT_JAVA_OPTIONS}" - -# Construct JAVA_CMD with Java options -JAVA_CMD="java $JAVA_OPTIONS -jar " - -# Configure application.conf path -if [ ! -z "$APP_CONF_FILE" ]; then - JAVA_CMD+="-Dconfig.file=$APP_CONF_FILE " -fi - -# Configure logback configuration file -if [ ! -z "$LOGBACK_CONF_FILE" ]; then - JAVA_CMD+="-Dlogback.configurationFile=$LOGBACK_CONF_FILE " -fi -if [ ! -z "$FLUENT_HOST" ]; then - JAVA_CMD+="-Defk.fluent-host=$FLUENT_HOST " -fi - -# Configure Spark -if [ ! -z "$SPARK_APPNAME" ]; then - JAVA_CMD+="-Dspark.app-name=$SPARK_APPNAME " -fi -if [ ! -z "$SPARK_MASTER" ]; then - JAVA_CMD+="-Dspark.master=$SPARK_MASTER " -fi -if [ ! -z "$SPARK_UIENABLED" ]; then - JAVA_CMD+="-Dspark.ui.enabled=$SPARK_UIENABLED " -fi - -# Configure Ignifyr mapping-related paths -if [ ! -z "$CONTEXT_PATH" ]; then - JAVA_CMD+="-Dignifyr.context-path=$CONTEXT_PATH " -fi -if [ ! -z "$MAPPINGS_FOLDER" ]; then - JAVA_CMD+="-Dignifyr.mappings.repository.folder-path=$MAPPINGS_FOLDER " -fi -if [ ! -z "$SCHEMAS_FOLDER" ]; then - JAVA_CMD+="-Dignifyr.mappings.schemas.repository.folder-path=$SCHEMAS_FOLDER " -fi -if [ ! -z "$MAPPING_JOB" ]; then - JAVA_CMD+="-Dignifyr.mapping-jobs.initial-job-file-path=$MAPPING_JOB " -fi -if [ ! -z "$FHIR_BATCH_SIZE" ]; then - JAVA_CMD+="-Dignifyr.fhir-writer.batch-group-size=$FHIR_BATCH_SIZE " -fi -if [ ! -z "$DB_PATH" ]; then - JAVA_CMD+="-Dignifyr.db-path=$DB_PATH " -fi - -# Configure the FHIR endpoint to which this Ignifyr can connect to retrieve resource definitions etc. -if [ ! -z "$FHIR_DEFINITIONS_FHIRVERSION" ]; then - JAVA_CMD+="-Dfhir.fhir-version=$FHIR_DEFINITIONS_FHIRVERSION " -fi -if [ ! -z "$FHIR_DEFINITIONS_ENDPOINT" ]; then - JAVA_CMD+="-Dfhir.definitions-fhir-endpoint=$FHIR_DEFINITIONS_ENDPOINT " -fi -if [ ! -z "$FHIR_DEFINITIONS_ROOT_URL_0" ]; then - JAVA_CMD+="-Dfhir.definitions-root-urls.0=$FHIR_DEFINITIONS_ROOT_URL_0 " -fi -if [ ! -z "$FHIR_DEFINITIONS_ROOT_URL_1" ]; then - JAVA_CMD+="-Dfhir.definitions-root-urls.1=$FHIR_DEFINITIONS_ROOT_URL_1 " -fi -if [ ! -z "$FHIR_DEFINITIONS_ROOT_URL_2" ]; then - JAVA_CMD+="-Dfhir.definitions-root-urls.2=$FHIR_DEFINITIONS_ROOT_URL_2 " -fi -if [ ! -z "$FHIR_DEFINITIONS_PROFILES_PATH" ]; then - JAVA_CMD+="-Dfhir.profiles-path=$FHIR_DEFINITIONS_PROFILES_PATH " -fi -if [ ! -z "$FHIR_DEFINITIONS_VALUESETS_PATH" ]; then - JAVA_CMD+="-Dfhir.valuesets-path=$FHIR_DEFINITIONS_VALUESETS_PATH " -fi -if [ ! -z "$FHIR_DEFINITIONS_CODESYSTEMS_URL" ]; then - JAVA_CMD+="-Dfhir.codesystems-path=$FHIR_DEFINITIONS_CODESYSTEMS_URL " -fi - -# Configure ignifyr-server web server -if [ ! -z "$WEBSERVER_HOST" ]; then - JAVA_CMD+="-Dwebserver.host=$WEBSERVER_HOST " -fi -if [ ! -z "$WEBSERVER_PORT" ]; then - JAVA_CMD+="-Dwebserver.port=$WEBSERVER_PORT " -fi -if [ ! -z "$WEBSERVER_BASEURI" ]; then - JAVA_CMD+="-Dwebserver.base-uri=$WEBSERVER_BASEURI " -fi - -# Configure ignifyr-redcap -if [ ! -z "$IGNIFYR_REDCAP_ENDPOINT" ]; then - JAVA_CMD+="-Dignifyr-redcap.endpoint=$IGNIFYR_REDCAP_ENDPOINT " -fi -if [ ! -z "$IGNIFYR_REDCAP_NOTIFICATION_PATH" ]; then - JAVA_CMD+="-Dignifyr-redcap.paths.notification=$IGNIFYR_REDCAP_NOTIFICATION_PATH " -fi -if [ ! -z "$IGNIFYR_REDCAP_PROJECTS_PATH" ]; then - JAVA_CMD+="-Dignifyr-redcap.paths.projects=$IGNIFYR_REDCAP_PROJECTS_PATH " -fi -if [ ! -z "$IGNIFYR_REDCAP_PROJECT_DATA_PATH" ]; then - JAVA_CMD+="-Dignifyr-redcap.paths.projectData=$IGNIFYR_REDCAP_PROJECT_DATA_PATH " -fi -if [ ! -z "$IGNIFYR_REDCAP_RELOAD_PARAMETER" ]; then - JAVA_CMD+="-Dignifyr-redcap.parameters.reload=$IGNIFYR_REDCAP_RELOAD_PARAMETER " -fi - -# Delay the execution for this amount of seconds -if [ ! -z "$DELAY_EXECUTION" ]; then - sleep $DELAY_EXECUTION -fi - -# Finally, tell which jar to run -JAVA_CMD+="ignifyr-server-standalone.jar" - -# Use 'exec' to replace the shell process with the Java process. -# This ensures that the Java application becomes the main process in the container, -# allowing signals such as SIGTERM to be properly handled. -exec $JAVA_CMD "$@" diff --git a/ignifyr-connector-fhir-server/pom.xml b/ignifyr-connector-fhir-server/pom.xml deleted file mode 100644 index 978156cf..00000000 --- a/ignifyr-connector-fhir-server/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-connector-fhir-server_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - io.ignifyr.connector.fhirserver - - - - - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - - io.onfhir.spark - spark-on-fhir-connector-api_${scala.binary.version} - - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - - diff --git a/ignifyr-connector-fhir-server/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension b/ignifyr-connector-fhir-server/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension deleted file mode 100644 index 765e5a32..00000000 --- a/ignifyr-connector-fhir-server/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension +++ /dev/null @@ -1,2 +0,0 @@ -# Ignifyr FHIR-server-as-source connector. -io.ignifyr.connector.fhirserver.FhirServerConnectorExtension diff --git a/ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtension.scala b/ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtension.scala deleted file mode 100644 index 0420c0e1..00000000 --- a/ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtension.scala +++ /dev/null @@ -1,28 +0,0 @@ -package io.ignifyr.connector.fhirserver - -import io.ignifyr.engine.data.read.BaseDataSourceReader -import io.ignifyr.engine.model.{ - FhirServerSource, - FhirServerSourceSettings, - MappingJobSourceSettings, - MappingSourceBinding -} -import io.ignifyr.engine.spi.{IgnifyrExtension, SourceConnector} -import org.apache.spark.sql.SparkSession - -/** - * Registers the FHIR-server-as-source connector with the engine via ServiceLoader. - */ -class FhirServerConnectorExtension extends IgnifyrExtension { - - override val id: String = "connector-fhir-server" - - override def sourceConnectors: Seq[SourceConnector] = Seq( - new SourceConnector { - override val id: String = "fhir-server" - override val bindingClass: Class[_ <: MappingSourceBinding] = classOf[FhirServerSource] - override val settingsClass: Class[_ <: MappingJobSourceSettings] = classOf[FhirServerSourceSettings] - override def createReader(spark: SparkSession): BaseDataSourceReader[_, _] = new FhirServerDataSourceReader(spark) - } - ) -} diff --git a/ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerDataSourceReader.scala b/ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerDataSourceReader.scala deleted file mode 100644 index 1a9f4982..00000000 --- a/ignifyr-connector-fhir-server/src/main/scala/io/ignifyr/connector/fhirserver/FhirServerDataSourceReader.scala +++ /dev/null @@ -1,109 +0,0 @@ -package io.ignifyr.connector.fhirserver - -import io.onfhir.client.model.{ - BasicAuthenticationSettings, - BearerTokenAuthorizationSettings, - FixedTokenAuthenticationSettings -} -import io.onfhir.spark.connector.reader.FhirApiReader.OPTIONS -import io.ignifyr.engine.data.read.BaseDataSourceReader -import io.ignifyr.engine.model._ -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{DataFrame, SparkSession} - -import java.time.LocalDateTime - -/** - * Reader class for retrieving data from a FHIR server using a custom Spark data source. - * - * @param spark The SparkSession - */ -class FhirServerDataSourceReader(spark: SparkSession) - extends BaseDataSourceReader[FhirServerSource, FhirServerSourceSettings] { - - /** - * No validation required for this data source since the data comes form a Fhir Server indicating that it is valid - * */ - override val needCardinalityValidation: Boolean = false - - /** - * Reads data from the specified FHIR server source. - * - * @param mappingSourceBinding Configuration information for the mapping source. - * @param mappingJobSourceSettings Source settings of the mapping job for the FHIR server. - * @param schema Optional schema for the source data. - * @param timeRange Optional time range to filter the data. - * @param jobId Optional identifier of the mapping job executing the read operation. - * @return A DataFrame containing the source data from the FHIR server. - * @throws IllegalArgumentException If the path is not a directory for streaming jobs. - * @throws NotImplementedError If the specified source format is not implemented. - */ - override def read( - mappingSourceBinding: FhirServerSource, - mappingJobSourceSettings: FhirServerSourceSettings, - schema: Option[StructType], - timeRange: Option[(LocalDateTime, LocalDateTime)] = Option.empty, - jobId: Option[String] = Option.empty - ): DataFrame = { - // extract Spark option for the authentication from the given source settings - val authenticationOptions = extractAuthenticationOptions(mappingJobSourceSettings) - - /* - val fhirConfig = SparkFhirConfig.apply("R5") - val schemaUtil = new SparkSchemaUtil(fhirConfig) - //Get the schema of the resource type from the provided FHIR release (R4 or R5) - val resourceSchema = schemaUtil.getSparkSchemaForResourceType(mappingSource.resourceType).get - */ - - // The DataFrameReader -> FhirApiReaderWrapper conversion that supplies `.fhir(url)`. - // spark-on-fhir 2.0.0 moved it out of the FhirApiReader companion into this object. - import io.onfhir.spark.connector.SparkFhirApiConnector._ - implicit val implicitSpark: SparkSession = spark - implicitSpark.read - .fhir(mappingJobSourceSettings.serverUrl) - .on(mappingSourceBinding.resourceType) - .options(authenticationOptions) - .load( /*resourceSchema*/ ) - // If we use the schema of a resource type, spark processing takes too long, probably because of the huge size of the schema. - // Hence, we load without a schema; FhirApiReader internally utilizes spark.read.json to parse the retrieved FHIR resources. - } - - /** - * Extracts authentication options from the provided FhirServerSourceSettings. - * - * @param sourceSettings The FhirServerSourceSettings containing security settings. - * @return A map of authentication options extracted from the source settings. - */ - private def extractAuthenticationOptions(sourceSettings: FhirServerSourceSettings): Map[String, String] = { - sourceSettings.securitySettings - .map { - case BearerTokenAuthorizationSettings( - clientId, - clientSecret, - requiredScopes, - authzServerTokenEndpoint, - clientAuthenticationMethod - ) => - Map( - OPTIONS.AUTH -> OPTIONS.AUTH_TYPE.BEARER_TOKEN, - OPTIONS.AUTH_BEARER_TOKEN_PARAMETERS.CLIENT_ID -> clientId, - OPTIONS.AUTH_BEARER_TOKEN_PARAMETERS.CLIENT_SECRET -> clientSecret, - OPTIONS.AUTH_BEARER_TOKEN_PARAMETERS.REQUIRED_SCOPES -> requiredScopes.mkString(","), - OPTIONS.AUTH_BEARER_TOKEN_PARAMETERS.AUTHZ_SERVER_TOKEN_ENDPOINT -> authzServerTokenEndpoint, - OPTIONS.AUTH_BEARER_TOKEN_PARAMETERS.CLIENT_AUTHENTICATION_METHOD -> clientAuthenticationMethod - ) - case BasicAuthenticationSettings(username, password) => - Map( - OPTIONS.AUTH -> OPTIONS.AUTH_TYPE.BASIC, - OPTIONS.AUTH_BASIC_PARAMETERS.USERNAME -> username, - OPTIONS.AUTH_BASIC_PARAMETERS.PASSWORD -> password - ) - case FixedTokenAuthenticationSettings(token) => - Map( - OPTIONS.AUTH -> OPTIONS.AUTH_TYPE.FIXED_TOKEN, - OPTIONS.AUTH_FIXED_TOKEN_PARAMETERS.TOKEN -> token - ) - } - .getOrElse(Map.empty) - } -} diff --git a/ignifyr-connector-fhir-server/src/test/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtensionSpec.scala b/ignifyr-connector-fhir-server/src/test/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtensionSpec.scala deleted file mode 100644 index 23c123d1..00000000 --- a/ignifyr-connector-fhir-server/src/test/scala/io/ignifyr/connector/fhirserver/FhirServerConnectorExtensionSpec.scala +++ /dev/null @@ -1,19 +0,0 @@ -package io.ignifyr.connector.fhirserver - -import io.ignifyr.engine.model.{FhirServerSource, FhirServerSourceSettings} -import io.ignifyr.engine.spi.ExtensionRegistry -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Verifies the FHIR-server connector is discovered through ServiceLoader when this module is on the - * classpath (no Docker required — this only inspects the extension registry). - */ -class FhirServerConnectorExtensionSpec extends AnyFlatSpec with Matchers { - - "The FHIR-server connector extension" should "register a FhirServerSource connector through ServiceLoader" in { - val connector = ExtensionRegistry.sourceConnectors.get(classOf[FhirServerSource]) - connector.map(_.id) shouldBe Some("fhir-server") - connector.map(_.settingsClass) shouldBe Some(classOf[FhirServerSourceSettings]) - } -} diff --git a/ignifyr-connector-kafka/pom.xml b/ignifyr-connector-kafka/pom.xml deleted file mode 100644 index 378644fb..00000000 --- a/ignifyr-connector-kafka/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-connector-kafka_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - io.ignifyr.connector.kafka - - - - - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - - org.apache.spark - spark-sql-kafka-0-10_${scala.binary.version} - - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - - diff --git a/ignifyr-connector-kafka/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension b/ignifyr-connector-kafka/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension deleted file mode 100644 index 83399f3c..00000000 --- a/ignifyr-connector-kafka/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension +++ /dev/null @@ -1,2 +0,0 @@ -# Ignifyr Kafka source connector. -io.ignifyr.connector.kafka.KafkaConnectorExtension diff --git a/ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaConnectorExtension.scala b/ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaConnectorExtension.scala deleted file mode 100644 index faf75aa9..00000000 --- a/ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaConnectorExtension.scala +++ /dev/null @@ -1,80 +0,0 @@ -package io.ignifyr.connector.kafka - -import io.ignifyr.engine.data.read.BaseDataSourceReader -import io.ignifyr.engine.model.exception.FhirMappingException -import io.ignifyr.engine.model.{ - FhirMappingJobExecution, - FhirMappingTask, - KafkaSource, - KafkaSourceSettings, - MappingJobSourceSettings, - MappingSourceBinding -} -import io.ignifyr.engine.spi.{IgnifyrExtension, SourceConnector, SourceFailureDescriptor} -import org.apache.kafka.common.errors.UnknownTopicOrPartitionException -import org.apache.spark.sql.SparkSession - -import scala.annotation.tailrec - -/** - * Registers the Kafka source connector with the engine via ServiceLoader, plus a - * [[SourceFailureDescriptor]] that turns Kafka's `UnknownTopicOrPartitionException` into a - * message naming the missing topic(s) — logic that previously lived (with a hard Kafka import) in - * the engine's RunningJobRegistry and the server's ExecutionService. - */ -class KafkaConnectorExtension extends IgnifyrExtension { - - override val id: String = "connector-kafka" - - override def sourceConnectors: Seq[SourceConnector] = Seq( - new SourceConnector { - override val id: String = "kafka" - override val bindingClass: Class[_ <: MappingSourceBinding] = classOf[KafkaSource] - override val settingsClass: Class[_ <: MappingJobSourceSettings] = classOf[KafkaSourceSettings] - override def createReader(spark: SparkSession): BaseDataSourceReader[_, _] = new KafkaSourceReader(spark) - } - ) - - override def sourceFailureDescriptors: Seq[SourceFailureDescriptor] = Seq( - new SourceFailureDescriptor { - override def describeStreamingFailure( - error: Throwable, - execution: FhirMappingJobExecution, - mappingTaskName: String - ): Option[FhirMappingException] = - if (hasCause(error, classOf[UnknownTopicOrPartitionException])) { - val bindings = execution.mappingTasks - .find(_.name.contentEquals(mappingTaskName)) - .toSeq - .flatMap(_.sourceBinding.values) - Some(missingTopicsException(bindings)) - } else None - - override def describeBatchTaskFailure( - error: Throwable, - mappingTask: FhirMappingTask - ): Option[FhirMappingException] = - if (hasCause(error, classOf[UnknownTopicOrPartitionException])) - Some(missingTopicsException(mappingTask.sourceBinding.values.toSeq)) - else None - } - ) - - /** Builds the "missing topics" message from the Kafka bindings among the given source bindings. */ - private def missingTopicsException(bindings: Seq[MappingSourceBinding]): FhirMappingException = { - val topicNames = bindings.collect { case k: KafkaSource => k.topicName }.mkString(", ") - FhirMappingException( - s"The following Kafka topic(s) specified in the mapping task do not exist: $topicNames" - ) - } - - /** Walks the cause chain looking for an exception of the given type. */ - @tailrec - private def hasCause(error: Throwable, causeType: Class[_ <: Throwable]): Boolean = - error match { - case null => false - case e if causeType.isInstance(e) => true - case e if e.getCause eq e => false - case e => hasCause(e.getCause, causeType) - } -} diff --git a/ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaSourceReader.scala b/ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaSourceReader.scala deleted file mode 100644 index ba03105f..00000000 --- a/ignifyr-connector-kafka/src/main/scala/io/ignifyr/connector/kafka/KafkaSourceReader.scala +++ /dev/null @@ -1,199 +0,0 @@ -package io.ignifyr.connector.kafka - -import com.fasterxml.jackson.core.JsonParseException -import io.onfhir.api.Resource -import io.onfhir.util.JsonFormatter._ -import io.ignifyr.engine.data.read.BaseDataSourceReader -import io.ignifyr.engine.model.{KafkaSource, KafkaSourceSettings} -import org.apache.spark.sql.functions.{col, from_json, udf} -import org.apache.spark.sql.types._ -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.json4s.{JBool, JDouble, JInt, JLong, JNull} - -import java.time.LocalDateTime -import javax.ws.rs.InternalServerErrorException - -/** - * - * @param spark - */ -class KafkaSourceReader(spark: SparkSession) extends BaseDataSourceReader[KafkaSource, KafkaSourceSettings] { - - /** - * Read the source data for the given task from Kafka - * - * @param mappingSourceBinding Configuration information for the mapping source - * @param mappingJobSourceSettings Common settings for the source system - * @param schema Schema for the source data - * @param timeRange Time range for the data to read if given - * @param jobId The identifier of mapping job which executes the mapping - * @return - */ - override def read( - mappingSourceBinding: KafkaSource, - mappingJobSourceSettings: KafkaSourceSettings, - schema: Option[StructType] = Option.empty, - timeRange: Option[(LocalDateTime, LocalDateTime)] = Option.empty, - jobId: Option[String] = Option.empty - ): DataFrame = { - import spark.implicits._ - - if (schema.isEmpty) { - throw new IllegalArgumentException("Schema is required for streaming source") - } - - // Capture the schema itself, not this reader: the UDF is shipped to the executors, and closing over - // an instance method would drag the SparkSession along with it. - val sourceSchema = schema.get - // user-defined function (UDF) to process message (json) coming from the Kafka topic - val processDataUDF = udf((message: String) => KafkaSourceReader.coerceMessageToSchema(message, sourceSchema)) - - // Determine whether to use batch or streaming read mode based on the 'asStream' setting - if (mappingJobSourceSettings.asStream) { - spark.readStream // Use streaming mode for continuous ingestion of new messages from Kafka - .format("kafka") - .option("kafka.bootstrap.servers", mappingJobSourceSettings.bootstrapServers) - .option("subscribe", mappingSourceBinding.topicName) - .option("inferSchema", value = true) - .options(mappingSourceBinding.options) - .load() - .select($"value".cast(StringType)) // change the type of message from binary to string - .withColumn("value", processDataUDF(col("value"))) // replace 'value' column with the processed data - .select(from_json($"value", sourceSchema).as("record")) - .select("record.*") - } else { - // Filter out the 'startingOffsets' option as it always supposed to be "earliest" for batch kafka reads - val filteredOptions = mappingSourceBinding.options.filterNot(o => o._1 == "startingOffsets") - spark.read // Use read instead of readStream for a batch read - .format("kafka") - .option("kafka.bootstrap.servers", mappingJobSourceSettings.bootstrapServers) - .option("subscribe", mappingSourceBinding.topicName) - .option("inferSchema", value = true) - .options(filteredOptions) - .load() - .select($"value".cast(StringType)) // change the type of message from binary to string - .withColumn("value", processDataUDF(col("value"))) // replace 'value' column with the processed data - .select(from_json($"value", sourceSchema).as("record")) - .select("record.*") - } - } -} - -object KafkaSourceReader { - - /** - * Coerces a Kafka message (JSON) so that its values are acceptable for the schema's data types. REDCap - * and similar producers send every value as a string, so a field typed `double`/`integer`/`long`/ - * `boolean` in the schema has to be converted before `from_json` would accept it; an empty string means - * "no value" and becomes null. A value that cannot be converted fails the read rather than reaching the - * mapping as null. - * - * Lives on the companion object so the UDF that calls it carries no reference to the reader instance - * (and therefore none to the SparkSession) when it is serialized to the executors. - * - * @param message the raw Kafka message - * @param schema the schema of the source data - * @return the message with its values coerced to the schema's data types - */ - private[kafka] def coerceMessageToSchema(message: String, schema: StructType): String = { - val json: Resource = - try { // try-catch block needed to handle unparseable json - message.parseJson - } catch { - case e: JsonParseException => - throw new InternalServerErrorException("Kafka message is an unparseable JSON", e) - } - - // process each field so that string values are acceptable (if they are parseable of course) for some data types such as double and integer - json - .mapField(field => { - schema.fields - .find(p => p.name.contentEquals(field._1)) - .map(fieldType => { // get field type from the schema - fieldType.dataType match { - case _: DoubleType => - try { - field._1 -> { - // performs the following conversions: - // JString(v) => JDouble(v) - // JDouble(v) => JDouble(v) - // JString() => JNull - field._2.extract[String] match { - case str if str.nonEmpty => JDouble(str.toDouble) - case _ => JNull - } - } - } catch { - case e: NumberFormatException => - throw new InternalServerErrorException(s"${field._2} is not a parsable `Double`", e) - } - case _: IntegerType => - try { - field._1 -> { - // performs the following conversions: - // JString(v) => JInt(v) - // JInt(v) => JInt(v) - // JString() => JNull - field._2.extract[String] match { - case str if str.nonEmpty => JInt(str.toInt) - case _ => JNull - } - } - } catch { - case e: NumberFormatException => - throw new InternalServerErrorException(s"${field._2} is not a parsable `Integer`", e) - } - case _: LongType => - try { - field._1 -> { - // performs the following conversions: - // JString(v) => JLong(v) - // JLong(v) => JLong(v) - // JString() => JNull - field._2.extract[String] match { - case str if str.nonEmpty => JLong(str.toLong) - case _ => JNull - } - } - } catch { - case e: NumberFormatException => - throw new InternalServerErrorException(s"${field._2} is not a parsable `Long`", e) - } - case _: BooleanType => - try { - field._1 -> { - // performs the following conversions: - // JString(v) => JBool(v) - // JString("0") => JBool(false) - // JString("1") => JBool(true) - // JBool(v) => JBool(v) - // JString() => JNull - val bool = field._2.extractOpt[Boolean] // try to extract as boolean - bool match { - // matches JBool(v) - case Some(value) => JBool(value) - // try to extract as string - case None => - field._2.extractOpt[String] match { - // matches JString("0") or matches JString("1") - case Some(value) if value.contentEquals("0") || value.contentEquals("1") => - JBool(if (value.contentEquals("0")) false else true) - // matches JString(v) - case Some(value) if value.nonEmpty => JBool(value.toBoolean) - // matches JString() - case _ => JNull - } - } - } - } catch { - case e: IllegalArgumentException => - throw new InternalServerErrorException(s"${field._2} is not a parsable `Boolean`", e) - } - case _ => field - } - }) - .getOrElse(field) - }) - .toJson - } -} diff --git a/ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaConnectorExtensionSpec.scala b/ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaConnectorExtensionSpec.scala deleted file mode 100644 index 3607ef2f..00000000 --- a/ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaConnectorExtensionSpec.scala +++ /dev/null @@ -1,23 +0,0 @@ -package io.ignifyr.connector.kafka - -import io.ignifyr.engine.model.{KafkaSource, KafkaSourceSettings} -import io.ignifyr.engine.spi.ExtensionRegistry -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Verifies the Kafka connector is discovered through ServiceLoader when this module is on the - * classpath — both its source connector and its source-failure descriptor (no Docker required). - */ -class KafkaConnectorExtensionSpec extends AnyFlatSpec with Matchers { - - "The Kafka connector extension" should "register a KafkaSource connector through ServiceLoader" in { - val connector = ExtensionRegistry.sourceConnectors.get(classOf[KafkaSource]) - connector.map(_.id) shouldBe Some("kafka") - connector.map(_.settingsClass) shouldBe Some(classOf[KafkaSourceSettings]) - } - - it should "register a source-failure descriptor" in { - ExtensionRegistry.sourceFailureDescriptors should not be empty - } -} diff --git a/ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaSourceReaderTest.scala b/ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaSourceReaderTest.scala deleted file mode 100644 index 95effaa7..00000000 --- a/ignifyr-connector-kafka/src/test/scala/io/ignifyr/connector/kafka/KafkaSourceReaderTest.scala +++ /dev/null @@ -1,110 +0,0 @@ -package io.ignifyr.connector.kafka - -import org.apache.spark.sql.types._ -import org.json4s.jackson.JsonMethods -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import javax.ws.rs.InternalServerErrorException - -/** - * Kafka producers (REDCap among them) send every value as a JSON string, so the reader coerces each value - * to the type its schema declares before `from_json` sees it. An uncoerced value would be dropped as null - * by `from_json` instead of failing, which is why the conversion rules — and specifically which inputs are - * an error rather than a null — are worth pinning. The end-to-end read is covered by the long-tier - * `KafkaStreamingRedcapTest`. - */ -class KafkaSourceReaderTest extends AnyFlatSpec with Matchers { - - private def schemaOf(fieldType: DataType): StructType = StructType(Seq(StructField("field", fieldType))) - - private def coerce(json: String, fieldType: DataType): String = - KafkaSourceReader.coerceMessageToSchema(json, schemaOf(fieldType)) - - /** The coerced value of the single `field`, rendered back as compact JSON. */ - private def valueOf(json: String, fieldType: DataType): String = - JsonMethods.compact(JsonMethods.render(JsonMethods.parse(coerce(json, fieldType)) \ "field")) - - "coerceMessageToSchema" should "convert a stringified number to a double" in { - valueOf("""{"field":"1.5"}""", DoubleType) shouldBe "1.5" - } - - it should "leave an already numeric double alone" in { - valueOf("""{"field":1.5}""", DoubleType) shouldBe "1.5" - } - - it should "convert a stringified integer and long" in { - valueOf("""{"field":"42"}""", IntegerType) shouldBe "42" - valueOf("""{"field":"9999999999"}""", LongType) shouldBe "9999999999" - } - - // An empty string is REDCap's "not answered", and it has to become null rather than a parse failure. - it should "turn an empty string into null for every numeric type" in { - Seq[DataType](DoubleType, IntegerType, LongType).foreach { fieldType => - valueOf("""{"field":""}""", fieldType) shouldBe "null" - } - } - - it should "turn an empty string into null for a boolean" in { - valueOf("""{"field":""}""", BooleanType) shouldBe "null" - } - - it should "accept a real boolean unchanged" in { - valueOf("""{"field":true}""", BooleanType) shouldBe "true" - valueOf("""{"field":false}""", BooleanType) shouldBe "false" - } - - // REDCap encodes yes/no answers as "1"/"0", which `toBoolean` alone would reject. - it should "read the 1 and 0 strings as booleans" in { - valueOf("""{"field":"1"}""", BooleanType) shouldBe "true" - valueOf("""{"field":"0"}""", BooleanType) shouldBe "false" - } - - it should "read the true and false strings as booleans" in { - valueOf("""{"field":"true"}""", BooleanType) shouldBe "true" - valueOf("""{"field":"false"}""", BooleanType) shouldBe "false" - } - - it should "leave a field whose schema type needs no coercion untouched" in { - valueOf("""{"field":"plain text"}""", StringType) shouldBe "\"plain text\"" - } - - it should "leave a field that the schema does not declare untouched" in { - val coerced = KafkaSourceReader.coerceMessageToSchema("""{"known":"1","unknown":"1"}""", schemaOf(BooleanType)) - coerced should include("\"unknown\":\"1\"") - } - - it should "fail on a value that cannot be converted to the declared numeric type" in { - a[InternalServerErrorException] should be thrownBy coerce("""{"field":"not a number"}""", DoubleType) - a[InternalServerErrorException] should be thrownBy coerce("""{"field":"not a number"}""", IntegerType) - a[InternalServerErrorException] should be thrownBy coerce("""{"field":"not a number"}""", LongType) - } - - it should "fail on a value that cannot be converted to a boolean" in { - a[InternalServerErrorException] should be thrownBy coerce("""{"field":"maybe"}""", BooleanType) - } - - it should "fail on an unparseable message rather than passing it downstream" in { - val thrown = the[InternalServerErrorException] thrownBy coerce("{not json", StringType) - thrown.getMessage should include("unparseable JSON") - } - - it should "coerce every declared field of a multi-field message" in { - val schema = StructType( - Seq( - StructField("age", IntegerType), - StructField("weight", DoubleType), - StructField("consent", BooleanType), - StructField("name", StringType) - ) - ) - val coerced = - JsonMethods.parse( - KafkaSourceReader.coerceMessageToSchema("""{"age":"7","weight":"12.5","consent":"1","name":"p1"}""", schema) - ) - (coerced \ "age").values shouldBe 7 - (coerced \ "weight").values shouldBe 12.5 - (coerced \ "consent").values shouldBe true - (coerced \ "name").values shouldBe "p1" - } -} diff --git a/ignifyr-format-delta/pom.xml b/ignifyr-format-delta/pom.xml deleted file mode 100644 index a1110b32..00000000 --- a/ignifyr-format-delta/pom.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-format-delta_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - io.ignifyr.format.delta - - - - - - - - - io.ignifyr - ignifyr-sink-file_2.13 - - - - - io.delta - delta-spark_${scala.binary.version} - - - - - io.ignifyr - ignifyr-testkit_2.13 - test - - - - diff --git a/ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension b/ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension deleted file mode 100644 index 081a1fd1..00000000 --- a/ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension +++ /dev/null @@ -1,4 +0,0 @@ -# Enterprise Delta Lake sink format: contributes the Delta Spark-session wiring (spark.sql.extensions -# + the Delta catalog). The Delta FileSinkFormat handler is registered via the file connector's own -# format sub-SPI service file. -io.ignifyr.format.delta.DeltaFormatExtension diff --git a/ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.sink.file.format.FileSinkFormat b/ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.sink.file.format.FileSinkFormat deleted file mode 100644 index e87570ae..00000000 --- a/ignifyr-format-delta/src/main/resources/META-INF/services/io.ignifyr.sink.file.format.FileSinkFormat +++ /dev/null @@ -1,2 +0,0 @@ -# Enterprise Delta Lake file sink format handler. -io.ignifyr.format.delta.DeltaSinkFormat diff --git a/ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaFormatExtension.scala b/ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaFormatExtension.scala deleted file mode 100644 index 421c1e3c..00000000 --- a/ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaFormatExtension.scala +++ /dev/null @@ -1,23 +0,0 @@ -package io.ignifyr.format.delta - -import io.ignifyr.engine.spi.IgnifyrExtension - -/** - * [[IgnifyrExtension]] for the enterprise Delta Lake sink format. It contributes nothing to the - * engine's connector/sink registries directly — the Delta writer plugs into the file connector's - * format sub-SPI ([[DeltaSinkFormat]], via its own META-INF/services entry). Its role here is to - * contribute the Spark-session wiring Delta needs, so the community engine carries no Delta config: - * when this module is installed, the shared SparkSession is built with the Delta session extension - * and the Delta catalog. - */ -class DeltaFormatExtension extends IgnifyrExtension { - - override val id: String = "format-delta" - - override def sparkConfContributions: Map[String, String] = Map( - // Enable Delta Lake features by adding the DeltaSparkSessionExtension to the Spark session. - "spark.sql.extensions" -> "io.delta.sql.DeltaSparkSessionExtension", - // Use DeltaCatalog as the default catalog for managing Delta tables in Spark. - "spark.sql.catalog.spark_catalog" -> "org.apache.spark.sql.delta.catalog.DeltaCatalog" - ) -} diff --git a/ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaSinkFormat.scala b/ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaSinkFormat.scala deleted file mode 100644 index a9b4e182..00000000 --- a/ignifyr-format-delta/src/main/scala/io/ignifyr/format/delta/DeltaSinkFormat.scala +++ /dev/null @@ -1,39 +0,0 @@ -package io.ignifyr.format.delta - -import io.ignifyr.sink.file.format.{FileSinkFormat, FileSinkSupport} -import io.ignifyr.engine.model.{FhirMappingResult, FileSystemSinkSettings, SinkContentTypes} -import org.apache.spark.sql.{Dataset, SparkSession} - -/** - * Enterprise Delta Lake file *sink* format (optionally partitioned by FHIR resource type), - * contributed to the file sink's format sub-SPI via ServiceLoader. The Spark-session wiring - * Delta needs (`spark.sql.extensions` + the Delta catalog) is contributed by - * [[DeltaFormatExtension]] through IgnifyrExtension.sparkConfContributions, and the delta-spark - * dependency lives in this module — keeping Delta out of the community engine and jar. - */ -class DeltaSinkFormat extends FileSinkFormat { - - override val contentTypes: Seq[String] = Seq(SinkContentTypes.DELTA_LAKE) - - override def write( - spark: SparkSession, - df: Dataset[FhirMappingResult], - sinkSettings: FileSystemSinkSettings - ): Unit = { - import spark.implicits._ - if (sinkSettings.partitionByResourceType) { - FileSinkSupport.writePartitionedByResourceType( - spark, - df, - sinkSettings, - singleColumnJson = false, - writeGroup = (writer, outputPath) => writer.format(SinkContentTypes.DELTA_LAKE).save(outputPath) - ) - } else { - // Convert the mapped resource JSON strings into a frame and write it as Delta Lake. - val jsonDS = df.select("mappedFhirResource.mappedResource").as[String] - val jsonDF = spark.read.json(jsonDS) - FileSinkSupport.getWriter(jsonDF, sinkSettings).format(SinkContentTypes.DELTA_LAKE).save(sinkSettings.path) - } - } -} diff --git a/ignifyr-format-delta/src/test/resources/application.conf b/ignifyr-format-delta/src/test/resources/application.conf deleted file mode 100644 index 2dda04df..00000000 --- a/ignifyr-format-delta/src/test/resources/application.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Ignifyr configurations (format-delta suites). The Delta Spark-session extension + catalog are -# contributed by DeltaFormatExtension.sparkConfContributions, not configured here. -ignifyr { - - context-path = "testcontext" - - mappings = { - repository = { - folder-path = "/test-mappings" - } - schemas = { - repository = { - folder-path = "/test-schemas" - } - } - timeout = 5 seconds - } - - fhir-server-writer { - batch-group-size = 20 - } - - db = "db" -} - -# Spark configurations -spark = { - app.name = "Ignifyr format-delta TEST" - master = "local[4]" - checkpoint-dir = "checkpoint" -} - -akka = { - daemonic = "on" -} diff --git a/ignifyr-format-delta/src/test/scala/io/ignifyr/format/delta/DeltaSinkFormatTest.scala b/ignifyr-format-delta/src/test/scala/io/ignifyr/format/delta/DeltaSinkFormatTest.scala deleted file mode 100644 index 047d7d5e..00000000 --- a/ignifyr-format-delta/src/test/scala/io/ignifyr/format/delta/DeltaSinkFormatTest.scala +++ /dev/null @@ -1,78 +0,0 @@ -package io.ignifyr.format.delta - -import io.ignifyr.FhirMappingResultFixtures -import io.ignifyr.sink.file.FileSystemWriter -import io.ignifyr.sink.file.format.FileSinkFormatRegistry -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.model.{FhirMappingResult, FileSystemSinkSettings, SinkContentTypes} -import io.ignifyr.engine.util.FileUtils -import org.apache.spark.sql.functions.col -import org.apache.spark.sql.{Dataset, SparkSession} -import org.scalatest.BeforeAndAfterAll -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Tests the enterprise Delta Lake sink format: it is discovered through the file connector's format - * registry when this module is on the classpath, and the file writer writes Delta output (plain and - * partitioned by resource type) through it. These are the Delta cases relocated from the connector's - * FileSystemWriterTest; they rely on the Delta Spark-session wiring this module contributes via - * DeltaFormatExtension.sparkConfContributions (so the shared SparkSession is built with the Delta - * session extension + catalog). The shared test dataset comes from the testkit. - */ -class DeltaSinkFormatTest extends AnyFlatSpec with BeforeAndAfterAll with Matchers { - - val sparkSession: SparkSession = IgnifyrConfig.sparkSession - import sparkSession.implicits._ - - val df: Dataset[FhirMappingResult] = FhirMappingResultFixtures.sampleFhirMappingResults(sparkSession) - - "The Delta format" should "be discovered for the delta content type" in { - FileSinkFormatRegistry.sinkFormats.keySet should contain(SinkContentTypes.DELTA_LAKE) - } - - it should "write DataFrame into a Delta Lake file" in { - val outputFolderPath = s"${IgnifyrConfig.engineConfig.contextPath}/output-delta" - val fileSystemWriter = new FileSystemWriter(sinkSettings = - FileSystemSinkSettings(path = outputFolderPath, contentType = SinkContentTypes.DELTA_LAKE) - ) - fileSystemWriter.write(sparkSession, df, sparkSession.sparkContext.collectionAccumulator[FhirMappingResult]) - - val writtenDf = sparkSession.read - .format(SinkContentTypes.DELTA_LAKE) - .load(outputFolderPath) - writtenDf.count() shouldBe 15 - val resultDf = writtenDf.groupBy("resourceType").count() - val patientCount = resultDf.filter(col("resourceType") === "Patient").select("count").as[Long].head() - patientCount shouldBe 10 - val conditionCount = resultDf.filter(col("resourceType") === "Condition").select("count").as[Long].head() - conditionCount shouldBe 5 - } - - it should "write DataFrame as partitioned Delta Lake files based on resource type" in { - val outputFolderPath = s"${IgnifyrConfig.engineConfig.contextPath}/output-delta-by-resource" - val fileSystemWriter = new FileSystemWriter(sinkSettings = - FileSystemSinkSettings( - path = outputFolderPath, - contentType = SinkContentTypes.DELTA_LAKE, - partitionByResourceType = true - ) - ) - fileSystemWriter.write(sparkSession, df, sparkSession.sparkContext.collectionAccumulator[FhirMappingResult]) - - val conditionDf = sparkSession.read - .format(SinkContentTypes.DELTA_LAKE) - .load(s"$outputFolderPath/Condition") - conditionDf.count() shouldBe 5 - val patientDf = sparkSession.read - .format(SinkContentTypes.DELTA_LAKE) - .load(s"$outputFolderPath/Patient") - patientDf.count() shouldBe 10 - } - - override protected def afterAll(): Unit = { - super.afterAll() - // delete the context path used for the output folders - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath("").toFile) - } -} diff --git a/ignifyr-format-json/pom.xml b/ignifyr-format-json/pom.xml deleted file mode 100644 index 53b852ed..00000000 --- a/ignifyr-format-json/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-format-json_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - io.ignifyr.format.json - - - - - - - - - io.ignifyr - ignifyr-connector-file_2.13 - - - - - io.ignifyr - ignifyr-testkit_2.13 - test - - - - diff --git a/ignifyr-format-json/src/main/resources/META-INF/services/io.ignifyr.connector.file.format.FileSourceFormat b/ignifyr-format-json/src/main/resources/META-INF/services/io.ignifyr.connector.file.format.FileSourceFormat deleted file mode 100644 index 39535348..00000000 --- a/ignifyr-format-json/src/main/resources/META-INF/services/io.ignifyr.connector.file.format.FileSourceFormat +++ /dev/null @@ -1,2 +0,0 @@ -# Enterprise JSON/NDJSON file source format handler. -io.ignifyr.format.json.JsonSourceFormat diff --git a/ignifyr-format-json/src/main/scala/io/ignifyr/format/json/JsonSourceFormat.scala b/ignifyr-format-json/src/main/scala/io/ignifyr/format/json/JsonSourceFormat.scala deleted file mode 100644 index 819af321..00000000 --- a/ignifyr-format-json/src/main/scala/io/ignifyr/format/json/JsonSourceFormat.scala +++ /dev/null @@ -1,33 +0,0 @@ -package io.ignifyr.format.json - -import io.ignifyr.connector.file.format.{FileSourceFormat, FileSourceReadContext} -import io.ignifyr.engine.model.SourceContentTypes -import io.ignifyr.engine.util.SparkUtil -import org.apache.spark.sql.DataFrame - -/** - * Enterprise JSON / NDJSON file *source* format (each line of an NDJSON file is a separate JSON - * object). Contributed to the file connector's format sub-SPI via ServiceLoader; kept out of the - * community edition per the edition split (the community file sink still writes NDJSON). - */ -class JsonSourceFormat extends FileSourceFormat { - - override val contentTypes: Seq[String] = Seq(SourceContentTypes.JSON, SourceContentTypes.NDJSON) - - override def read(ctx: FileSourceReadContext): DataFrame = { - val binding = ctx.mappingSourceBinding - if (ctx.mappingJobSourceSettings.asStream) { - // schema cannot be inferred for streaming so let's infer it ourselves - val inferredSchema = ctx.spark.read.options(binding.options).json(ctx.finalPath).schema - ctx.spark.readStream - .options(binding.options) - .schema(inferredSchema) - .json(ctx.finalPath) - } else if (ctx.isZipFile) { - val unzippedFileContents = SparkUtil.readZip(ctx.finalPath, ctx.spark) - ctx.spark.read.options(binding.options).json(unzippedFileContents) - } else { - ctx.spark.read.options(binding.options).json(ctx.finalPath) - } - } -} diff --git a/ignifyr-format-json/src/test/resources/application.conf b/ignifyr-format-json/src/test/resources/application.conf deleted file mode 100644 index 5a3c63cb..00000000 --- a/ignifyr-format-json/src/test/resources/application.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Ignifyr configurations (format-json suites read their own file fixtures) -ignifyr { - - # A path to a file/directory from where any File system readings should use within the mappingjob definition. - context-path = "testcontext" - - mappings = { - repository = { - folder-path = "/test-mappings" - } - schemas = { - repository = { - folder-path = "/test-schemas" - } - } - timeout = 5 seconds - } - - fhir-server-writer { - batch-group-size = 20 - } - - db = "db" -} - -# Spark configurations -spark = { - app.name = "Ignifyr format-json TEST" - master = "local[4]" - checkpoint-dir = "checkpoint" -} - -akka = { - daemonic = "on" -} diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient1.json b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient1.json deleted file mode 100644 index 28e95146..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient1.json +++ /dev/null @@ -1,3 +0,0 @@ -{"pid": "p1", "gender": "male", "birthDate": "2000-05-10"}, -{"pid": "p2", "gender": "male", "birthDate": "1985-05-08"}, -{"pid": "p3", "gender": "male", "birthDate": "1997-02"} \ No newline at end of file diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient2.json b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient2.json deleted file mode 100644 index e9aeb610..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient2.json +++ /dev/null @@ -1,3 +0,0 @@ -{"pid": "p4", "gender": "male", "birthDate": "1999-06-05"}, -{"pid": "p5", "gender": "male", "birthDate": "1965-10-01"}, -{"pid": "p6", "gender": "female", "birthDate": "1991-03"} \ No newline at end of file diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient3.json b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient3.json deleted file mode 100644 index 6b5b9e9a..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/json/patient3.json +++ /dev/null @@ -1,3 +0,0 @@ -{"pid": "p7", "gender": "female", "birthDate": "1972-10-25"}, -{"pid": "p8", "gender": "female", "birthDate": "2010-01-10"}, -{"pid": "p9", "gender": "female", "birthDate": "1999-05-12"} \ No newline at end of file diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient1.txt b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient1.txt deleted file mode 100644 index 735f6d42..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient1.txt +++ /dev/null @@ -1,3 +0,0 @@ -{"pid": "p1", "gender": "male", "birthDate": "2000-05-10"} -{"pid": "p2", "gender": "male", "birthDate": "1985-05-08"} -{"pid": "p3", "gender": "male", "birthDate": "1997-02"} \ No newline at end of file diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient2.txt b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient2.txt deleted file mode 100644 index ef95e890..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient2.txt +++ /dev/null @@ -1,3 +0,0 @@ -{"pid": "p4", "gender": "male", "birthDate": "1999-06-05"} -{"pid": "p5", "gender": "male", "birthDate": "1965-10-01"} -{"pid": "p6", "gender": "female", "birthDate": "1991-03"} \ No newline at end of file diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient3.txt b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient3.txt deleted file mode 100644 index 258e6200..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/folder-test/txt-ndjson/patient3.txt +++ /dev/null @@ -1,3 +0,0 @@ -{"pid": "p7", "gender": "female", "birthDate": "1972-10-25"} -{"pid": "p8", "gender": "female", "birthDate": "2010-01-10"} -{"pid": "p9", "gender": "female", "birthDate": "1999-05-12"} \ No newline at end of file diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients-ndjson.txt b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients-ndjson.txt deleted file mode 100644 index 1a650ded..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients-ndjson.txt +++ /dev/null @@ -1,11 +0,0 @@ -{ "pid": "p1", "gender": "male", "birthDate": "2000-05-10", "deceasedDateTime": null, "homePostalCode": null } -{ "pid": "p2", "gender": "male", "birthDate": "1985-05-08", "deceasedDateTime": "2017-03-10", "homePostalCode": "G02547" } -{ "pid": "p3", "gender": "male", "birthDate": "1997-02", "deceasedDateTime": null, "homePostalCode": null } -{ "pid": "p4", "gender": "male", "birthDate": "1999-06-05", "deceasedDateTime": null, "homePostalCode": "H10564" } -{ "pid": "p5", "gender": "male", "birthDate": "1965-10-01", "deceasedDateTime": "2019-04-21", "homePostalCode": "G02547" } -{ "pid": "p6", "gender": "female", "birthDate": "1991-03", "deceasedDateTime": null, "homePostalCode": null } -/* This line is a comment to test spark options */ -{ "pid": "p7", "gender": "female", "birthDate": "1972-10-25", "deceasedDateTime": null, "homePostalCode": "V13135" } -{ "pid": "p8", "gender": "female", "birthDate": "2010-01-10", "deceasedDateTime": null, "homePostalCode": "Z54564" } -{ "pid": "p9", "gender": "female", "birthDate": "1999-05-12", "deceasedDateTime": null, "homePostalCode": null } -{ "pid": "p10", "gender": "female", "birthDate": "2003-11", "deceasedDateTime": null, "homePostalCode": null } diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients.json b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients.json deleted file mode 100644 index b3741068..00000000 --- a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/single-file-test/patients.json +++ /dev/null @@ -1,11 +0,0 @@ -{ "pid": "p1", "gender": "male", "birthDate": "2000-05-10", "deceasedDateTime": null, "homePostalCode": null }, -{ "pid": "p2", "gender": "male", "birthDate": "1985-05-08", "deceasedDateTime": "2017-03-10", "homePostalCode": "G02547" }, -{ "pid": "p3", "gender": "male", "birthDate": "1997-02", "deceasedDateTime": null, "homePostalCode": null }, -{ "pid": "p4", "gender": "male", "birthDate": "1999-06-05", "deceasedDateTime": null, "homePostalCode": "H10564" }, -{ "pid": "p5", "gender": "male", "birthDate": "1965-10-01", "deceasedDateTime": "2019-04-21", "homePostalCode": "G02547" }, -{ "pid": "p6", "gender": "female", "birthDate": "1991-03", "deceasedDateTime": null, "homePostalCode": null }, -/* This line is a comment to test spark options */ -{ "pid": "p7", "gender": "female", "birthDate": "1972-10-25", "deceasedDateTime": null, "homePostalCode": "V13135" }, -{ "pid": "p8", "gender": "female", "birthDate": "2010-01-10", "deceasedDateTime": null, "homePostalCode": "Z54564" }, -{ "pid": "p9", "gender": "female", "birthDate": "1999-05-12", "deceasedDateTime": null, "homePostalCode": null }, -{ "pid": "p10", "gender": "female", "birthDate": "2003-11", "deceasedDateTime": null, "homePostalCode": null } \ No newline at end of file diff --git a/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/zip-test/json.zip b/ignifyr-format-json/src/test/resources/file-data-source-reader-test-data/zip-test/json.zip deleted file mode 100644 index cb1d7d07cfefc2cb8225892c1d2f7e8aa9a3e0a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 690 zcmWIWW@Zs#U|`^2=#0>b6#BZcJsil}1;o5ST##6jnVMH(td~`spSL

x56>rl0^V zgYZj%0bc%Ry*+iAK3)zg7A_8QO!A!Rsw%11sk%}j(b>CLIo6puRylZSlx&<{OBwf~ zEz*(t<5c`kW7ZQ3rjWxLwkytXm~ptxY-yacx(pv@amU0{grf)FGZfbCa8 z*zc&!%*-6i91L^<#Qrp8B@^kWs2NQr(h&?@YrjOaAF@u;DTYbT$sD1HU!Aq2*f-r`E-;OOk_?H$W(9LpTMG%8N7B`$c;msvAk zNlCqsl34W0)cNY{XX!;@*vEY@dn%DZJJJ&T%4AKQ!&B!Fr zfGg-#fKC7b1u%&k5nNDFMg|Fnisvt4U$NX~M5obBM-5tJ)9ZjXK%*HP(Bzno8gR&_ c|3jD#4o2ckXJrF "true", - "distinct" -> "true" // 'distinct' option randomly changes the order of the rows in the result. - ) - - // Loop through each source binding configuration to run the test - val mappingJobSourceSettings = FileSystemSourceSettings( - name = s"JsonSourceFormatTest1", - sourceUri = "test-uri", - dataFolderPath = testDataFolderPath.concat(folderPath) - ) - sourceBindingConfigurations.foreach { case (fileName, contentType) => - // Define the source binding and settings for reading the file - val mappingSourceBinding = FileSystemSource(path = fileName, contentType = contentType, options = sparkOptions) - // Read the data from the specified file - val result: DataFrame = fileDataSourceReader.read(mappingSourceBinding, mappingJobSourceSettings, Option.empty) - - // Validate the result - result.count() shouldBe expectedRowNumber - result.columns shouldBe expectedColumns - // Check that the result contains the first and last row of the source data - result.collect().contains(expectedFirstRow) shouldBe true - result.collect().contains(expectedLastRow) shouldBe true - } - } - - it should "correctly read multiple files from JSON and NDJSON folders" in { - // Folder containing the test folders for JSON and NDJSON files - val folderPath = "/folder-test" - - // Expected values for validation - val expectedRowNumber = 9 - val expectedColumns = Array("birthDate", "gender", "pid") - // Expected rows for validation, one row from each file - val expectedRows = Set( - Row("2000-05-10", "male", "p1"), - Row("1999-06-05", "male", "p4"), - Row("1972-10-25", "female", "p7") - ) - - // A sequence of folder names and content type of the files to be selected - val sourceBindingConfigurations = Seq( - ("json", SourceContentTypes.JSON), - ("txt-ndjson", SourceContentTypes.NDJSON) - ) - - // Loop through each source binding configuration to run the test - val mappingJobSourceSettings = FileSystemSourceSettings( - name = "JsonSourceFormatTest2", - sourceUri = "test-uri", - dataFolderPath = testDataFolderPath.concat(folderPath) - ) - sourceBindingConfigurations.foreach { case (folderName, contentType) => - // Read the data using the reader and the defined settings - val mappingSourceBinding = FileSystemSource(path = folderName, contentType = contentType) - val result: DataFrame = fileDataSourceReader.read(mappingSourceBinding, mappingJobSourceSettings, Option.empty) - - // Validate the result - result.count() shouldBe expectedRowNumber - result.columns shouldBe expectedColumns - result.collect().toSet should contain allElementsOf expectedRows - } - } - - it should "correctly read from JSON and NDJSON files inside a zip archive" in { - - // Path to the zip file containing the test files - val folderPath = "/zip-test" - - // Expected values for validation - val expectedRowNumber = 9 - val expectedColumns = Array("birthDate", "gender", "pid") - // Expected rows for validation, one row from each file - val expectedRows = Set( - Row("2000-05-10", "male", "p1"), - Row("1999-06-05", "male", "p4"), - Row("1972-10-25", "female", "p7") - ) - - // Define the zip file names and their corresponding formats to be tested - val sourceBindingConfigurations = Seq( - ("json.zip", SourceContentTypes.JSON), // JSON inside zip - ("txt-ndjson.zip", SourceContentTypes.NDJSON) // Newline-delimited JSON inside zip - ) - - // Spark options for testing (e.g., allowing comments in the files) - val sparkOptions = Map( - "allowComments" -> "true" - ) - - // Loop through each zip file and perform the test - val mappingJobSourceSettings = FileSystemSourceSettings( - name = "JsonSourceFormatTest3", - sourceUri = "zip-uri", - dataFolderPath = testDataFolderPath.concat(folderPath) - ) - - sourceBindingConfigurations.foreach { case (zipFileName, contentType) => - // Define the source binding and read the data from the zip file - val mappingSourceBinding = FileSystemSource(path = zipFileName, contentType = contentType, options = sparkOptions) - val result: DataFrame = fileDataSourceReader.read(mappingSourceBinding, mappingJobSourceSettings, Option.empty) - - // Validate the result - result.count() shouldBe expectedRowNumber - result.columns shouldBe expectedColumns - result.collect().toSet should contain allElementsOf expectedRows - } - } -} diff --git a/ignifyr-observability/pom.xml b/ignifyr-observability/pom.xml deleted file mode 100644 index f668c983..00000000 --- a/ignifyr-observability/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-observability_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - io.ignifyr.observability - - - - - - - - - org.scala-lang - scala-library - - - - ch.qos.logback - logback-classic - - - ch.qos.logback - logback-core - - - - net.logstash.logback - logstash-logback-encoder - - - - com.sndyuk - logback-more-appenders - - - org.fluentd - fluent-logger - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - - diff --git a/ignifyr-observability/src/main/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoder.scala b/ignifyr-observability/src/main/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoder.scala deleted file mode 100644 index 8c43c1d6..00000000 --- a/ignifyr-observability/src/main/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoder.scala +++ /dev/null @@ -1,101 +0,0 @@ -package io.ignifyr.observability.logback - -import ch.qos.logback.classic.spi.{ILoggingEvent, IThrowableProxy, LoggerContextVO} -import ch.qos.logback.more.appenders.marker.MapMarker -import net.logstash.logback.encoder.LogstashEncoder -import net.logstash.logback.marker.LogstashMarker -import net.logstash.logback.marker.Markers.appendEntries - -import java.util - -/** - * An encoder for converting MapMarker objects to LogstashMarker objects. - */ -class MapMarkerToLogstashMarkerEncoder extends LogstashEncoder { - - /** - * Encodes the logging event by converting MapMarker to LogstashMarker and then calling super.encode(). - * - * @param event The logging event to encode. - * @return The encoded logging event as a byte array. - */ - override def encode(event: ILoggingEvent): Array[Byte] = { - val modifiedEvent = modifyEventWithLogstashMarker(event) - super.encode(modifiedEvent) - } - - /** - * Modifies the logging event with a LogstashMarker if it contains a MapMarker. - * - * @param event The logging event to modify. - * @return The modified logging event. - */ - private def modifyEventWithLogstashMarker(event: ILoggingEvent): ILoggingEvent = { - val map = getMapMarkerData(event.getMarker) - if (map.isDefined) { - val logstashMarker = toLogstashMarker(map.get) - new ModifiedLoggingEvent(event, logstashMarker) - } else { - event - } - } - - /** - * Extracts the data from a MapMarker. - * - * @param marker The marker to extract data from. - * @return An Optional containing the marker data if the marker is a MapMarker, otherwise empty. - */ - private def getMapMarkerData(marker: org.slf4j.Marker): Option[util.Map[String, _]] = { - marker match { - case m: MapMarker => Some(m.getMap) - case _ => None - } - } - - /** - * Converts a Map of marker data to a LogstashMarker. - * - * @param map The map of marker data. - * @return The corresponding LogstashMarker. - */ - private def toLogstashMarker(map: util.Map[String, _]): LogstashMarker = { - appendEntries(map) - } - - /** - * A wrapper class for the logging event that delegates to the original event but with a different marker. - */ - private class ModifiedLoggingEvent(originalEvent: ILoggingEvent, marker: LogstashMarker) extends ILoggingEvent { - override def getMarker: org.slf4j.Marker = marker - - // Implement other methods by delegating to the original event - override def getTimeStamp: Long = originalEvent.getTimeStamp - - override def getLevel: ch.qos.logback.classic.Level = originalEvent.getLevel - - override def getThreadName: String = originalEvent.getThreadName - - override def getMessage: String = originalEvent.getMessage - - override def getArgumentArray: Array[AnyRef] = originalEvent.getArgumentArray - - override def getFormattedMessage: String = originalEvent.getFormattedMessage - - override def getLoggerName: String = originalEvent.getLoggerName - - override def getLoggerContextVO: LoggerContextVO = originalEvent.getLoggerContextVO - - override def getThrowableProxy: IThrowableProxy = originalEvent.getThrowableProxy - - override def getCallerData: Array[StackTraceElement] = originalEvent.getCallerData - - override def hasCallerData: Boolean = originalEvent.hasCallerData - - override def getMDCPropertyMap: util.Map[String, String] = originalEvent.getMDCPropertyMap - - override def getMdc: util.Map[String, String] = originalEvent.getMdc - - override def prepareForDeferredProcessing(): Unit = originalEvent.prepareForDeferredProcessing() - } -} diff --git a/ignifyr-observability/src/test/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoderSpec.scala b/ignifyr-observability/src/test/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoderSpec.scala deleted file mode 100644 index 496e2a94..00000000 --- a/ignifyr-observability/src/test/scala/io/ignifyr/observability/logback/MapMarkerToLogstashMarkerEncoderSpec.scala +++ /dev/null @@ -1,96 +0,0 @@ -package io.ignifyr.observability.logback - -import ch.qos.logback.classic.spi.LoggingEvent -import ch.qos.logback.classic.{Level, LoggerContext} -import ch.qos.logback.more.appenders.marker.MapMarker -import net.logstash.logback.encoder.LogstashEncoder -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import java.nio.charset.StandardCharsets -import java.util - -/** - * This encoder is the whole of the observability module's logic and the only thing that gets the engine's - * structured execution markers into Elasticsearch: the engine logs a `MapMarker` (a plain map), which - * Logstash's encoder does not understand, so this class rewrites it into a `LogstashMarker` before - * delegating. If the rewrite silently stopped happening, the Kibana "Executions" dashboard would go blank - * while every log line still looked fine. - * - * The class is referenced by name from the server's logback.xml, so it is not reachable from Scala. - */ -class MapMarkerToLogstashMarkerEncoderSpec extends AnyFlatSpec with Matchers { - - private val loggerContext = new LoggerContext() - - private def startedEncoder: MapMarkerToLogstashMarkerEncoder = { - val encoder = new MapMarkerToLogstashMarkerEncoder - encoder.setContext(loggerContext) - encoder.start() - encoder - } - - /** A warn-level event carrying the given marker, shaped like the ones `SinkHandler` emits. */ - private def event(marker: org.slf4j.Marker): LoggingEvent = { - val loggingEvent = new LoggingEvent() - loggingEvent.setLoggerName("io.ignifyr.engine.data.write.SinkHandler") - loggingEvent.setLevel(Level.WARN) - loggingEvent.setMessage("Mapping failure") - loggingEvent.setTimeStamp(System.currentTimeMillis()) - loggingEvent.setLoggerContextRemoteView(loggerContext.getLoggerContextRemoteView) - loggingEvent.setMarker(marker) - loggingEvent - } - - private def mapMarker(entries: (String, Any)*): MapMarker = { - val map: util.Map[String, Any] = new util.HashMap[String, Any]() - entries.foreach { case (key, value) => map.put(key, value) } - new MapMarker("marker", map) - } - - private def encode(marker: org.slf4j.Marker): String = - new String(startedEncoder.encode(event(marker)), StandardCharsets.UTF_8) - - behavior of "MapMarkerToLogstashMarkerEncoder" - - it should "be a LogstashEncoder that instantiates without configuration" in { - new MapMarkerToLogstashMarkerEncoder shouldBe a[LogstashEncoder] - } - - // The load-bearing assertion: without the MapMarker -> LogstashMarker rewrite these keys would not be - // top-level fields of the emitted JSON, and the dashboard indexes them as top-level fields. - it should "lift the MapMarker entries into top-level JSON fields" in { - val json = encode(mapMarker("jobId" -> "job-1", "executionId" -> "exec-1", "errorCode" -> "INVALID_INPUT")) - json should include("\"jobId\":\"job-1\"") - json should include("\"executionId\":\"exec-1\"") - json should include("\"errorCode\":\"INVALID_INPUT\"") - } - - it should "keep the standard log fields alongside the marker entries" in { - val json = encode(mapMarker("jobId" -> "job-1")) - json should include("\"level\":\"WARN\"") - json should include("\"message\":\"Mapping failure\"") - json should include("\"logger_name\":\"io.ignifyr.engine.data.write.SinkHandler\"") - } - - it should "preserve the value types of the marker entries" in { - val json = encode(mapMarker("numOfWritten" -> 12, "isStreaming" -> true)) - json should include("\"numOfWritten\":12") - json should include("\"isStreaming\":true") - } - - it should "encode an event with no marker at all" in { - val json = new String(startedEncoder.encode(event(null)), StandardCharsets.UTF_8) - json should include("\"message\":\"Mapping failure\"") - } - - it should "leave a marker that is not a MapMarker to the delegate" in { - val json = encode(org.slf4j.MarkerFactory.getMarker("PLAIN_MARKER")) - json should include("\"message\":\"Mapping failure\"") - } - - it should "encode an empty MapMarker without emitting spurious fields" in { - val json = encode(mapMarker()) - json should include("\"message\":\"Mapping failure\"") - } -} diff --git a/ignifyr-redcap/CLAUDE.md b/ignifyr-redcap/CLAUDE.md deleted file mode 100644 index 4b8feb41..00000000 --- a/ignifyr-redcap/CLAUDE.md +++ /dev/null @@ -1,67 +0,0 @@ -# ignifyr-redcap — REDCap integration - -Enterprise module for REDCap support. Two things: (1) an engine CLI command that converts a REDCap -data-dictionary CSV into FHIR `StructureDefinition` schemas, and (2) server REST routes that proxy the -sibling **tofhir-redcap** service plus a per-project data-dictionary schema-import route. Package root -`io.ignifyr.redcap` (flat). Bundled into `ignifyr-server`, **not** `ignifyr-cli`. - -**Why a module:** dependency *direction*, not dependency isolation (its pom declares no third-party -compile dependency at all). It is the only plugin in the reactor depending on **both** -`ignifyr-engine` and `ignifyr-server-common`; living in the engine would force the engine to depend on -server code and invert the layering. It is also the only consumer of the `IgnifyrServerExtension` seam — -the module that justifies that SPI existing. Being enterprise, it does not opt into -`ban-enterprise-deps`. - -## Defining trait: it registers against BOTH SPIs -Two `META-INF/services` files, one per SPI — this is the module's whole point (REDCap plugs into both -the engine and the server with zero core changes): -- `io.ignifyr.engine.spi.IgnifyrExtension` → `RedCapExtension` — the engine side: contributes the - `extract-redcap-schemas` CLI command (formerly registered by the engine's `CoreExtension`). -- `io.ignifyr.server.common.spi.IgnifyrServerExtension` → `RedCapServerExtension` — the server side: - contributes routes and the `/metadata` version. Both use `id = "redcap"`. - -## Layout (`src/main/scala/io/ignifyr/redcap/`) -- `RedCapExtension` — engine `IgnifyrExtension`; its inline `CliCommandProvider.argsFromOptions` maps - Boot one-shot flags (`data-dictionary`, `definition-root-url`, `encoding`) to positional args. -- `RedCapServerExtension` — server `IgnifyrServerExtension`; `initialize` best-effort parses the - optional `ignifyr-redcap` HOCON block; `rootRoutes` contributes the `/redcap` proxy **only when that - block is present**; `schemaImportRoutes` **always** contributes the import route; - `externalComponentVersion()` GETs `/metadata` and returns the **entire unparsed response - body** as the version string, with no timeout of its own — the 1-second bound and the - swallow-on-failure live in the server's `MetadataService`, which files the result under - `Metadata.ignifyrRedcapVersion`. -- `ExtractRedCapSchemas` — the CLI `Command`: reads the dictionary CSV, calls `RedCapUtil`, writes each - `.StructureDefinition.json` into the schema repository folder (warns + overwrites on collision). -- `RedCapUtil` — the extraction object: one `SchemaDefinition` per REDCap Form; `getDataType` / - `getCardinality` map REDCap field & text-validation types to FHIR; injects a record-id field if - absent. `RedCapDataDictionaryColumns` / `RedCapDataTypes` / `RedCapTextValidationTypes` hold the - column/value constants. Only `extractSchemasAsSchemaDefinitions` is pure — `extractSchemas` also reads - `IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion`. - ⚠️ `recordIdField` defaults to `""`, and the CLI path takes that default — so CLI-extracted schemas get - a record-identifier field with an empty `id` and a `"."` path. Only the server import route - passes a real value (a query parameter). -- `RedCapEndpoint` + `RedCapService` + `RedCapServiceConfig` — the `/redcap` proxy (projects, - delete-with-reload, notification) to tofhir-redcap; `RedCapSchemaImportEndpoint` — the multipart - `/projects/{id}/schemas/redcap` import (persists via the server-provided `SchemaImportSink`); - `RedCapProjectConfig` — the proxied project payload. - -## Notes -- The proxy targets the sibling **tofhir-redcap** service — a **sanctioned legacy `tofhir` survivor**. - The default endpoint `http://localhost:8095/tofhir-redcap` and the `ignifyr-redcap` HOCON key keep - that value on purpose; do **not** rename them to `ignifyr`. -- BOM gotcha: REDCap exports the dictionary as UTF-8-with-BOM, so `RedCapUtil` reads the variable-name - column, then falls back to a BOM-prefixed column name before failing. -- The Kafka coupling (deleting a project's data from topics) lives in the external tofhir-redcap - service — there is **no** Kafka dependency in this module. - -## Tests -Two unit suites (`AnyFlatSpec`; no Docker, no testkit). `mvn test -pl ignifyr-redcap`. -- `RedCapExtensionSpec` — dual-SPI discovery, the CLI flag→positional mapping, config-gated `rootRoutes`, - and pure `RedCapUtil` extraction. -- `RedCapUtilDataTypeSpec` — the REDCap field type + text-validation type → FHIR data type table, driven - through `extractSchemasAsSchemaDefinitions`. **This is the widest lookup table in the repo** (35 - branches) and it decides the column types every downstream mapping reads, so a wrong entry yields a - schema that parses cleanly and types its data wrongly — never an error. Add a case here whenever you - touch the table. - -Endpoint-level import behaviour is covered by the server's long-tier `SchemaEndpointTest`. diff --git a/ignifyr-redcap/pom.xml b/ignifyr-redcap/pom.xml deleted file mode 100644 index 718811e9..00000000 --- a/ignifyr-redcap/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-redcap_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - io.ignifyr.redcap - - - - - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - io.ignifyr - ignifyr-server-common - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - - diff --git a/ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension b/ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension deleted file mode 100644 index 4d69ca47..00000000 --- a/ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension +++ /dev/null @@ -1,2 +0,0 @@ -# Ignifyr REDCap integration (engine-side contributions: the extract-redcap-schemas CLI command). -io.ignifyr.redcap.RedCapExtension diff --git a/ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.server.common.spi.IgnifyrServerExtension b/ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.server.common.spi.IgnifyrServerExtension deleted file mode 100644 index 4574583e..00000000 --- a/ignifyr-redcap/src/main/resources/META-INF/services/io.ignifyr.server.common.spi.IgnifyrServerExtension +++ /dev/null @@ -1,2 +0,0 @@ -# Ignifyr REDCap integration (server-side contributions: /redcap proxy + schema import routes). -io.ignifyr.redcap.RedCapServerExtension diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/ExtractRedCapSchemas.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/ExtractRedCapSchemas.scala deleted file mode 100644 index bad04a51..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/ExtractRedCapSchemas.scala +++ /dev/null @@ -1,64 +0,0 @@ -package io.ignifyr.redcap - -import io.onfhir.api.Resource -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.engine.cli.command.{Command, CommandExecutionContext} -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.util.{CsvUtil, FileUtils} -import org.json4s.jackson.Serialization.writePretty - -import java.io.{FileNotFoundException, FileWriter} - -/** - * Command to extract schemas from a REDCap data dictionary. - * */ -class ExtractRedCapSchemas extends Command { - - /** - * @param args The list of arguments. - * The first argument is the path of REDCap data dictionary file. - * The second argument is the definition root url. - * The third argument is the encoding of CSV file (OPTIONAL). - */ - override def execute(args: Seq[String], context: CommandExecutionContext): CommandExecutionContext = { - if (args.isEmpty) { - println( - "extract-redcap-schemas command requires the path of RedCap data dictionary to extract from and definition root url." - ) - context - } else if (args.length == 1) { - println("extract-redcap-schemas command requires definition root url.") - context - } else { - val filePath = args.head - try { - // read REDCap data dictionary - val content: Seq[Map[String, String]] = CsvUtil.readFromCSV(filePath, args.lift(2).getOrElse("UTF-8")) - // extract schemas - val fhirResources: Seq[Resource] = RedCapUtil.extractSchemas(content, args(1)) - // write each schema as a file - fhirResources.foreach(fhirResource => { - val resourceType = fhirResource.values("type") - val file = FileUtils - .getPath(IgnifyrConfig.engineConfig.schemaRepositoryFolderPath, resourceType + ".StructureDefinition.json") - .toFile - // when the schema already exists, warn user - if (file.exists()) { - println(s"Schema '$resourceType' already exists. Overriding it...") - } - // write schema to the file - val fw = new FileWriter(file) - try fw.write(writePretty(fhirResource)) - finally fw.close() - println(s"Schema '$resourceType' successfully created.") - }) - - context - } catch { - case _: FileNotFoundException => - println(s"The file cannot be found at the specified path:$filePath") - context - } - } - } -} diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapEndpoint.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapEndpoint.scala deleted file mode 100644 index 5c53e4d4..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapEndpoint.scala +++ /dev/null @@ -1,108 +0,0 @@ -package io.ignifyr.redcap - -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes} -import akka.http.scaladsl.server.Directives.{path, _} -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.redcap.RedCapEndpoint.{PARAMETER_RELOAD, SEGMENT_NOTIFICATION, SEGMENT_REDCAP} -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.onfhir.definitions.common.model.Json4sSupport._ - -/** - * This class defines routes for handling requests related to ignifyr-redcap service. - * - * @param redCapServiceConfig The configuration object for the ignifyr-redcap service. - */ -class RedCapEndpoint(redCapServiceConfig: RedCapServiceConfig) extends LazyLogging { - - // Create an instance of RedCapService using the provided configuration - val service: RedCapService = new RedCapService(redCapServiceConfig: RedCapServiceConfig) - - /** - * Defines the routes for handling requests to the ignifyr-redcap service. - * - * @param request The incoming REST call request. - * @return A Route representing the defined routes for the RedCap endpoint. - */ - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_REDCAP) { - pathEndOrSingleSlash { // Operations on Redcap projects - getProjects ~ saveProjects - } ~ path(Segment) { projectId => - deleteRedCapData(projectId) - } ~ pathPrefix(SEGMENT_NOTIFICATION) { - pathEndOrSingleSlash { // Operations on Redcap notification url - getNotificationUrl - } - } - } - } - - /** - * Retrieves the notification URL. - * - * @return A Route representing the route to retrieve the notification URL. - */ - private def getNotificationUrl: Route = { - get { - complete { - service.getNotificationUrl.map { notificationUrl => - HttpResponse(StatusCodes.OK, entity = HttpEntity(ContentTypes.`text/plain(UTF-8)`, notificationUrl)) - } - } - } - } - - /** - * Saves RedCap projects. - * - * @return A Route representing the route to save RedCap projects. - */ - private def saveProjects: Route = { - post { - entity(as[Seq[RedCapProjectConfig]]) { projects => - complete { - service.saveProjects(projects) - } - } - } - } - - /** - * Retrieves RedCap projects. - * - * @return A Route representing the route to retrieve RedCap projects. - */ - private def getProjects: Route = { - get { - complete { - service.getProjects - } - } - } - - /** - * Route to delete REDCap data from Kafka topics. - * - * @param projectId The identifier of the project whose data will be deleted. - * */ - private def deleteRedCapData(projectId: String): Route = { - delete { - parameter(PARAMETER_RELOAD) { reload => - complete { - service.deleteRedCapData(projectId = projectId, reload = reload.contentEquals("true")) - } - } - } - } -} - -/** - * Contains constants for segment names used in the RedCap endpoint. - */ -object RedCapEndpoint { - val SEGMENT_REDCAP = "redcap" - val SEGMENT_NOTIFICATION = "notification" - val PARAMETER_RELOAD = "reload" -} diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapExtension.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapExtension.scala deleted file mode 100644 index 94016d6e..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapExtension.scala +++ /dev/null @@ -1,29 +0,0 @@ -package io.ignifyr.redcap - -import io.ignifyr.engine.cli.command.Command -import io.ignifyr.engine.spi.{CliCommandProvider, IgnifyrExtension} - -/** - * Registers the REDCap engine-side contributions with the engine via ServiceLoader: the - * `extract-redcap-schemas` CLI command (previously registered by the engine's CoreExtension). - */ -class RedCapExtension extends IgnifyrExtension { - - override val id: String = "redcap" - - override def cliCommands: Seq[CliCommandProvider] = Seq( - new CliCommandProvider { - override val name: String = "extract-redcap-schemas" - - override def create(): Command = new ExtractRedCapSchemas() - - /** Boot's one-shot mode passes flags; translate them to the command's positional args. */ - override def argsFromOptions(options: Map[String, String]): Seq[String] = - Seq(options.get("data-dictionary"), options.get("definition-root-url"), options.get("encoding")).flatten - - override def helpText: Option[String] = Some( - "extract-redcap-schemas - Extracts schemas from the given REDCap data dictionary file. Schemas will be annotated with the given definition root url. If the encoding of CSV file is different from UTF-8, you should provide it." - ) - } - ) -} diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapProjectConfig.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapProjectConfig.scala deleted file mode 100644 index b9645a03..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapProjectConfig.scala +++ /dev/null @@ -1,10 +0,0 @@ -package io.ignifyr.redcap - -/** - * Represents the configuration details for a REDCap project. - * - * @param id The unique identifier of the REDCap project. - * @param token The API token used for accessing the REDCap project. - * @param recordIdField The field name used as the unique record identifier within the project. Defaults to "record_id". - */ -case class RedCapProjectConfig(id: String, token: String, recordIdField: String = "record_id") diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapSchemaImportEndpoint.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapSchemaImportEndpoint.scala deleted file mode 100644 index 10b2e2f4..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapSchemaImportEndpoint.scala +++ /dev/null @@ -1,53 +0,0 @@ -package io.ignifyr.redcap - -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.util.CsvUtil -import io.ignifyr.redcap.RedCapSchemaImportEndpoint.{ATTACHMENT, SEGMENT_REDCAP} -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.ignifyr.server.common.spi.SchemaImportSink -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.onfhir.definitions.common.model.SchemaDefinition - -/** - * The `/projects/{projectId}/schemas/redcap` route: imports a REDCap data dictionary file and - * creates a schema for each form defined in it. Contributed to the server's schemas endpoint via - * [[io.ignifyr.server.common.spi.IgnifyrServerExtension.schemaImportRoutes]]; the schemas are - * persisted through the server-provided [[SchemaImportSink]]. - */ -class RedCapSchemaImportEndpoint(schemaImportSink: SchemaImportSink) extends LazyLogging { - - def route(request: IgnifyrRestCall): Route = { - val projectId: String = request.projectId.get - pathPrefix(SEGMENT_REDCAP) { - importREDCapDataDictionary(projectId) - } - } - - /** - * Route to import a REDCap data dictionary file which will be used to create schemas. - * */ - private def importREDCapDataDictionary(projectId: String): Route = { - post { - fileUpload(ATTACHMENT) { case (_, byteSource) => - parameters("rootUrl", "recordIdField") { (rootUrl, recordIdField) => - complete { - CsvUtil.readFromCSVSource(byteSource).flatMap { rows => - // extract schema definitions from the data dictionary rows and save them - val definitions: Seq[SchemaDefinition] = - RedCapUtil.extractSchemasAsSchemaDefinitions(rows, rootUrl, recordIdField) - schemaImportSink.saveSchemas(projectId, definitions) - } - } - } - } - } - } -} - -object RedCapSchemaImportEndpoint { - val SEGMENT_REDCAP = "redcap" - private val ATTACHMENT = "attachment" -} diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServerExtension.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServerExtension.scala deleted file mode 100644 index 29983703..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServerExtension.scala +++ /dev/null @@ -1,54 +0,0 @@ -package io.ignifyr.redcap - -import akka.http.scaladsl.Http -import akka.http.scaladsl.model.headers.RawHeader -import akka.http.scaladsl.model.{HttpMethods, HttpRequest} -import akka.http.scaladsl.server.Route -import akka.http.scaladsl.unmarshalling.Unmarshal -import com.typesafe.config.Config -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.ignifyr.server.common.spi.{IgnifyrServerExtension, SchemaImportSink} - -import scala.concurrent.{ExecutionContext, Future} -import scala.util.Try - -/** - * Server-side REDCap contributions, discovered through ServiceLoader: - * - the `/redcap` proxy endpoints for the sibling tofhir-redcap service, enabled only when the - * `ignifyr-redcap` HOCON block is present (mirroring the previous optional wiring in - * IgnifyrServer); - * - the `/projects/{id}/schemas/redcap` data-dictionary import route, which needs no external - * service and is therefore always contributed; - * - the version of the connected tofhir-redcap service for the server's `/metadata`. - */ -class RedCapServerExtension extends IgnifyrServerExtension { - - override val id: String = "redcap" - - private var redCapServiceConfig: Option[RedCapServiceConfig] = None - - override def initialize(rootConfig: Config): Unit = - redCapServiceConfig = Try(new RedCapServiceConfig(rootConfig.getConfig("ignifyr-redcap"))).toOption - - override def rootRoutes: Seq[IgnifyrRestCall => Route] = - redCapServiceConfig.toSeq.map(config => new RedCapEndpoint(config).route _) - - override def schemaImportRoutes(schemaImportSink: SchemaImportSink): Seq[IgnifyrRestCall => Route] = - Seq(new RedCapSchemaImportEndpoint(schemaImportSink).route _) - - override def externalComponentVersion()(implicit ec: ExecutionContext): Future[Option[String]] = - redCapServiceConfig match { - case None => Future.successful(None) - case Some(config) => - import io.ignifyr.engine.Execution.actorSystem - val proxiedRequest = HttpRequest( - method = HttpMethods.GET, - uri = s"${config.endpoint}/metadata", - headers = RawHeader("Content-Type", "application/json") :: Nil - ) - Http() - .singleRequest(proxiedRequest) - .flatMap(resp => Unmarshal(resp.entity).to[String]) - .map(Some(_)) - } -} diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapService.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapService.scala deleted file mode 100644 index cac62f61..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapService.scala +++ /dev/null @@ -1,109 +0,0 @@ -package io.ignifyr.redcap - -import akka.http.scaladsl.Http -import akka.http.scaladsl.model.headers.RawHeader -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpMethods, HttpRequest} -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import org.json4s.DefaultFormats -import org.json4s.jackson.{JsonMethods, Serialization} - -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import scala.concurrent.Future -import scala.concurrent.duration.{DurationInt, FiniteDuration} - -/** - * Class for interacting with the ignifyr-redcap service. - * - * This class provides methods to retrieve notification URLs, save projects, and get projects from the ignifyr-redcap service. - * - * @param redCapServiceConfig The configuration object for the ignifyr-redcap service. - */ -class RedCapService(redCapServiceConfig: RedCapServiceConfig) extends LazyLogging { - - // Timeout duration for proxy requests to the ignifyr-redcap service - private val timeout: FiniteDuration = 20.seconds - - /** - * Retrieves the notification URL from the ignifyr-redcap service. - * - * @return A Future containing the notification URL as a String. - */ - def getNotificationUrl: Future[String] = { - val proxiedRequest = HttpRequest( - method = HttpMethods.GET, - uri = s"${redCapServiceConfig.notificationEndpoint}", - headers = RawHeader("Content-Type", "application/json") :: Nil - ) - - // Add timeout and transform response to a String - Http() - .singleRequest(proxiedRequest) - .flatMap { resp => resp.entity.toStrict(timeout) } - .map(strictEntity => strictEntity.data.utf8String) - } - - /** - * Saves the provided RedCap project configurations to the ignifyr-redcap service. - * - * @param requestBody The sequence of RedCapProjectConfig objects to be saved. - * @return A Future representing the completion of the save operation. - */ - def saveProjects(requestBody: Seq[RedCapProjectConfig]): Future[Unit] = { - implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats - - val proxiedRequest = HttpRequest( - method = HttpMethods.POST, - uri = s"${redCapServiceConfig.projectsEndpoint}", - headers = RawHeader("Content-Type", "application/json") :: Nil, - entity = HttpEntity(ContentTypes.`application/json`, Serialization.write(requestBody)) - ) - - // Add timeout - Http() - .singleRequest(proxiedRequest) - .flatMap { resp => resp.entity.toStrict(timeout) } - .map(_ => ()) - } - - /** - * Retrieves the projects from the ignifyr-redcap service. - * - * @return A Future containing the sequence of RedCapProjectConfig objects retrieved from the service. - */ - def getProjects: Future[Seq[RedCapProjectConfig]] = { - val proxiedRequest = HttpRequest( - method = HttpMethods.GET, - uri = s"${redCapServiceConfig.projectsEndpoint}", - headers = RawHeader("Content-Type", "application/json") :: Nil - ) - - // Add timeout and transform response to a sequence of RedCapProjectConfig objects - Http() - .singleRequest(proxiedRequest) - .flatMap { resp => resp.entity.toStrict(timeout) } - .map(strictEntity => JsonMethods.parse(strictEntity.data.utf8String).extract[Seq[RedCapProjectConfig]]) - } - - /** - * Deletes the REDCap data of the project from Kafka topics via ignifyr-redcap service. - * - * @param projectId The identifier of project - * @param reload Whether to reload REDCap data upon the deletion of Kafka topics - * */ - def deleteRedCapData(projectId: String, reload: Boolean): Future[Unit] = { - val proxiedRequest = HttpRequest( - method = HttpMethods.DELETE, - uri = - s"${redCapServiceConfig.projectsEndpoint}/${redCapServiceConfig.projectDataPath}/$projectId?${redCapServiceConfig.projectDataReloadParameter}=$reload", - headers = RawHeader("Content-Type", "application/json") :: Nil - ) - - // Add timeout - Http() - .singleRequest(proxiedRequest) - .flatMap { resp => resp.entity.toStrict(timeout) } - .map(_ => ()) - } -} diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServiceConfig.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServiceConfig.scala deleted file mode 100644 index bac38854..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapServiceConfig.scala +++ /dev/null @@ -1,51 +0,0 @@ -package io.ignifyr.redcap - -import com.typesafe.config.Config - -import scala.util.Try - -/** - * Configuration class for the ignifyr-redcap. - * - * @param redCapServiceConfig The configuration object containing the settings for the ignifyr-redcap service. - */ -class RedCapServiceConfig(redCapServiceConfig: Config) { - - /** - * The path to the notification endpoint. - */ - private lazy val notificationPath: String = - Try(redCapServiceConfig.getString("paths.notification")).getOrElse("notification") - - /** - * The path to the projects endpoint. - */ - private lazy val projectsPath: String = Try(redCapServiceConfig.getString("paths.projects")).getOrElse("projects") - - /** - * The path to the project's data endpoint - * */ - lazy val projectDataPath: String = Try(redCapServiceConfig.getString("paths.projectData")).getOrElse("data") - - /** - * Parameter to reload REDCap data upon recreation of Kafka topics. - * */ - lazy val projectDataReloadParameter: String = - Try(redCapServiceConfig.getString("parameters.reload")).getOrElse("reload") - - /** - * The base endpoint URL for the ignifyr-redcap service. - */ - lazy val endpoint: String = - Try(redCapServiceConfig.getString("endpoint")).getOrElse("http://localhost:8095/tofhir-redcap") - - /** - * Constructs the full URL for the notification endpoint by combining the base endpoint URL and the notification path. - */ - lazy val notificationEndpoint: String = s"$endpoint/$notificationPath" - - /** - * Constructs the full URL for the projects endpoint by combining the base endpoint URL and the projects path. - */ - lazy val projectsEndpoint: String = s"$endpoint/$projectsPath" -} diff --git a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapUtil.scala b/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapUtil.scala deleted file mode 100644 index 644605f1..00000000 --- a/ignifyr-redcap/src/main/scala/io/ignifyr/redcap/RedCapUtil.scala +++ /dev/null @@ -1,432 +0,0 @@ -package io.ignifyr.redcap - -import io.onfhir.api.{FHIR_DATA_TYPES, FHIR_FOUNDATION_RESOURCES, FHIR_ROOT_URL_FOR_DEFINITIONS, Resource} -import io.onfhir.definitions.common.model.{DataTypeWithProfiles, SchemaDefinition, SimpleStructureDefinition} -import io.ignifyr.common.util.SchemaUtil -import io.ignifyr.engine.config.IgnifyrConfig - -import javax.ws.rs.BadRequestException - -/** - * Util class providing methods to extract schemas from a REDCap data dictionary. - * */ -object RedCapUtil { - - /** - * Extracts the FHIR resources from the given REDCap data dictionary file. - * - * @param content The content of a REDCap data dictionary file - * @param definitionRootUrl The definition root url for the newly created schemas - * @param recordIdField The name of the field that represents the record ID. - * @return the list of schemas extracted from REDCap data dictionary - * */ - def extractSchemas( - content: Seq[Map[String, String]], - definitionRootUrl: String, - recordIdField: String = "" - ): Seq[Resource] = { - val schemaDefinitions: Seq[SchemaDefinition] = - extractSchemasAsSchemaDefinitions(content, definitionRootUrl, recordIdField) - // convert it to FHIR Resources - schemaDefinitions.map(definition => - SchemaUtil.convertToStructureDefinitionResource( - definition, - IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion - ) - ) - } - - /** - * Extracts Schema Definitions from the given REDCap data dictionary file. - * - * @param content The content of a REDCap data dictionary file - * @param definitionRootUrl The definition root url for the newly created schemas - * @param recordIdField The name of the field that represents the record ID in the output. - * @return the list of schemas extracted from REDCap data dictionary - * @throws BadRequestException when REDCap data dictionary file does not include {@link RedCapDataDictionaryColumns.VARIABLE_FIELD_NAME} - * */ - def extractSchemasAsSchemaDefinitions( - content: Seq[Map[String, String]], - definitionRootUrl: String, - recordIdField: String - ): Seq[SchemaDefinition] = { - // find forms - val forms: Map[String, Seq[Map[String, String]]] = - content.groupBy(row => row(RedCapDataDictionaryColumns.FORM_NAME)) - // create a schema for each form - forms - .map(form => { - val schemaName = form._1 - val schemaId = schemaName.capitalize - // get schema definitions - var definitions: Seq[SimpleStructureDefinition] = Seq.empty - // whether the record identifier field is included in the instrument or not - var includeRecordIdentifierField: Boolean = false - form._2.foreach(row => { - // read columns - // try to retrieve the variable name from the regular field name column. - // if it's not present, fallback to the field name column that includes BOM. - val variableName: Option[String] = row - .get(RedCapDataDictionaryColumns.VARIABLE_FIELD_NAME) - .orElse(row.get(RedCapDataDictionaryColumns.VARIABLE_FIELD_NAME_WITH_BOM)) - if (variableName.isEmpty) - throw new BadRequestException( - s"Invalid REDCap Data Dictionary file since it does not include the following column: ${RedCapDataDictionaryColumns.VARIABLE_FIELD_NAME}" - ) - val fieldType = row(RedCapDataDictionaryColumns.FIELD_TYPE) - val fieldLabel = row(RedCapDataDictionaryColumns.FIELD_LABEL).trim() - val fieldNotes = row.getOrElse(RedCapDataDictionaryColumns.FIELD_NOTES, fieldLabel).trim() - val required = row(RedCapDataDictionaryColumns.REQUIRED_FIELD) - val textValidationType = row.get(RedCapDataDictionaryColumns.TEXT_VALIDATION_TYPE) - - // discard the descriptive fields since they do not provide any data and they are not included in the export - // record response - if (!fieldType.contentEquals(RedCapDataTypes.DESCRIPTIVE)) { - // find applicable data type - val dataType = getDataType(fieldType, textValidationType) - // find cardinality - val cardinality = getCardinality(fieldType, required) - - definitions = definitions :+ SimpleStructureDefinition( - id = variableName.get, - path = s"$schemaId.${variableName.get}", - dataTypes = Some(Seq(dataType)), - isPrimitive = true, // every field of a REDCap schema is primitive - isChoiceRoot = false, - isArray = cardinality._2.isEmpty, - minCardinality = cardinality._1, - maxCardinality = cardinality._2, - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = Some(fieldLabel), - definition = Some(fieldNotes), - comment = None, - elements = None - ) - } - // check whether it is the record identifier field - if (!includeRecordIdentifierField && variableName.get.contentEquals(recordIdField)) { - includeRecordIdentifierField = true - } - }) - // add record identifier field if it does not exist in the form - if (!includeRecordIdentifierField) { - definitions = SimpleStructureDefinition( - id = recordIdField, - path = s"$schemaId.${recordIdField}", - dataTypes = Some(Seq(getDataType(RedCapDataTypes.TEXT))), - isPrimitive = true, // every field of a REDCap schema is primitive - isChoiceRoot = false, - isArray = false, // there is only one record identifier - minCardinality = 0, - maxCardinality = Some(1), - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = Some("Record Identifier"), - definition = Some("Unique identifier for individual record within a project"), - comment = None, - elements = None - ) +: definitions - } - - SchemaDefinition( - id = schemaId, - url = s"$definitionRootUrl/${FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION}/$schemaId", - version = SchemaDefinition.VERSION_LATEST, // Set the schema version to latest for the REDCap schemas - `type` = schemaId, - name = schemaName, - description = None, - rootDefinition = None, - fieldDefinitions = Some(definitions) - ) - }) - .toSeq - } - - /** - * Returns the minimum and maximum cardinality of a REDCap variable. - * - * @param fieldType the data type of REDCap variable. See {@link RedCapDataTypes} - * @param required whether the variable is required or not. 'y' is used to indicate that it is required. - * @return minimum and maximum cardinality as a tuple - * */ - private def getCardinality(fieldType: String, required: String): (Int, Option[Int]) = { - val minCardinality = required match { - case "y" => 1 - case _ => 0 - } - val maxCardinality = fieldType match { - case RedCapDataTypes.CHECKBOXES => None - case RedCapDataTypes.DESCRIPTIVE => - Some(0) // descriptive fields are used for decorative purposes, they do not provide any data - case _ => Some(1) - } - (minCardinality, maxCardinality) - } - - /** - * Returns the FHIR data type for a REDCap variable whose field and text validation types are given. - * - * @param fieldType the data type of REDCap variable. See {@link RedCapDataTypes} - * @param textValidationType the text validation type of REDCap variable. See {@link RedCapTextValidationTypes} - * @return corresponding FHIR data type - * @throws IllegalArgumentException when invalid data type or text validation types are given - * */ - private def getDataType(fieldType: String, textValidationType: Option[String] = None): DataTypeWithProfiles = { - fieldType match { - case RedCapDataTypes.TEXT | RedCapDataTypes.NOTES => - if (textValidationType.nonEmpty) { - textValidationType.get match { - case RedCapTextValidationTypes.DATE_DMY => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATE}")) - ) - case RedCapTextValidationTypes.DATE_MDY => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATE}")) - ) - case RedCapTextValidationTypes.DATE_YMD => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATE}")) - ) - case RedCapTextValidationTypes.DATETIME_DMY => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATETIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATETIME}")) - ) - case RedCapTextValidationTypes.DATETIME_MDY => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATETIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATETIME}")) - ) - case RedCapTextValidationTypes.DATETIME_YMD => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATETIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATETIME}")) - ) - case RedCapTextValidationTypes.DATETIME_SECOND_DMY => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATETIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATETIME}")) - ) - case RedCapTextValidationTypes.DATETIME_SECONDS_MDY => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATETIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATETIME}")) - ) - case RedCapTextValidationTypes.DATETIME_SECONDS_YMD => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DATETIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DATETIME}")) - ) - case RedCapTextValidationTypes.EMAIL => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.STRING, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.STRING}")) - ) - case RedCapTextValidationTypes.INTEGER => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.INTEGER, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.INTEGER}")) - ) - case RedCapTextValidationTypes.NUMBER => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DECIMAL, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DECIMAL}")) - ) - case RedCapTextValidationTypes.PHONE => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.STRING, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.STRING}")) - ) - case RedCapTextValidationTypes.TIME => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.TIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.TIME}")) - ) - case RedCapTextValidationTypes.ZIP_CODE => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.STRING, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.STRING}")) - ) - case RedCapTextValidationTypes.POSTAL_CODE_GERMANY => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.STRING, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.STRING}")) - ) - case RedCapTextValidationTypes.TIME_MM_SS => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.TIME, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.TIME}")) - ) - case RedCapTextValidationTypes.NUMBER_2DP => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DECIMAL, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DECIMAL}")) - ) - case "" => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.STRING, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.STRING}")) - ) - case _ => { - throw new IllegalArgumentException(s"Invalid text validation type for texts: ${textValidationType.get}") - } - } - } else { - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.STRING, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.STRING}")) - ) - } - case RedCapDataTypes.RADIO => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CODE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.CODE}")) - ) - case RedCapDataTypes.DROPDOWN => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CODE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.CODE}")) - ) - case RedCapDataTypes.CHECKBOXES => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CODE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.CODE}")) - ) - case RedCapDataTypes.CALC => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.DECIMAL, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.DECIMAL}")) - ) - case RedCapDataTypes.FILE => - if (textValidationType.nonEmpty) { - textValidationType.get match { - case RedCapTextValidationTypes.SIGNATURE => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.SIGNATURE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.SIGNATURE}")) - ) - case "" => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.BASE64BINARY, - profiles = - Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.BASE64BINARY}")) - ) - case _ => { - throw new IllegalArgumentException(s"Invalid text validation type for files: ${textValidationType.get}") - } - } - } else { - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.BASE64BINARY, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.BASE64BINARY}")) - ) - } - case RedCapDataTypes.YES_NO => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.BOOLEAN, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.BOOLEAN}")) - ) - case RedCapDataTypes.TRUE_FALSE => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.BOOLEAN, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.BOOLEAN}")) - ) - case RedCapDataTypes.DESCRIPTIVE => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.STRING, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.STRING}")) - ) - case RedCapDataTypes.SLIDER => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.INTEGER, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.INTEGER}")) - ) - case RedCapDataTypes.SQL => - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CODE, - profiles = Some(Seq(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/StructureDefinition/${FHIR_DATA_TYPES.CODE}")) - ) - case _ => { - throw new IllegalArgumentException(s"Invalid data type: $fieldType") - } - } - } -} - -/** - * Keeps the columns of a REDCap data dictionary csv - * */ -object RedCapDataDictionaryColumns { - val VARIABLE_FIELD_NAME = "Variable / Field Name" - // When exporting the Data Dictionary file from REDCap, it is encoded with 'UTF-8 with BOM'. - // The following variable is used to store the name including the BOM for 'Variable / Field Name' column. - val VARIABLE_FIELD_NAME_WITH_BOM = "\uFEFF\"Variable / Field Name\"" - val FORM_NAME = "Form Name" - val FIELD_TYPE = "Field Type" - val FIELD_LABEL = "Field Label" - val FIELD_NOTES = "Field Notes" - val REQUIRED_FIELD = "Required Field?" - val TEXT_VALIDATION_TYPE = "Text Validation Type OR Show Slider Number" -} - -/** - * Available values for {@link RedCapDataDictionaryColumns.FIELD_TYPE} - * */ -object RedCapDataTypes { - val TEXT = "text" // single-line text box (for text and numbers) - val NOTES = "notes" // large text box for lots of text - val RADIO = "radio" // radio buttons for multiple choice options - val DROPDOWN = "dropdown" // a dropdown menu for multiple choice options - val CHECKBOXES = "checkbox" // checkboxes for multiple choice options where more than one answer can be chosen - val CALC = "calc" // perform real-time calculations. Calculations can only result in numbers. - val FILE = "file" // upload a document - val YES_NO = "yesno" // radio buttons with yes and no options - val TRUE_FALSE = "truefalse" // radio buttons with true and false options - val DESCRIPTIVE = "descriptive" // text displayed with no data entry and optional image/file attachment - val SLIDER = "slider" // visual analogue scale; coded as 0-100 - val SQL = "sql" // select query statement to populate dropdown choices -} - -/** - * Available values for {@link RedCapDataDictionaryColumns.TEXT_VALIDATION_TYPE} - * */ -object RedCapTextValidationTypes { - val DATE_DMY = "date_dmy" // 31-12-2008 - val DATE_MDY = "date_mdy" // 12-31-2008 - val DATE_YMD = "date_ymd" // 2008-12-31 - val DATETIME_DMY = "datetime_dmy" // 16-02-2011 17:45 - val DATETIME_MDY = "datetime_mdy" // 02-16-2011 17:45 - val DATETIME_YMD = "datetime_ymd" // 2011-02-16 17:45 - val DATETIME_SECOND_DMY = "datetime_second_dmy" // 16-02-2011 17:45:23 - val DATETIME_SECONDS_MDY = "datetime_seconds_mdy" // 02-16-2011 17:45:23 - val DATETIME_SECONDS_YMD = "datetime_seconds_ymd" // 2011-02-16 17:45:23 - val EMAIL = "email" // john.doe@test - val INTEGER = "integer" // whole number with no decimal such as 1, 4, -10 - val NUMBER = "number" // (1.3, 22, -6.28) a general number - val PHONE = "phone" // 615-322-2222) - val TIME = "time" // (19:30, 04:15) - Military time - val ZIP_CODE = "zipcode" // (37212, 90210) 5-digit zipcode - val POSTAL_CODE_GERMANY = "postalcode_germany" // (37212, 90210) 5-digit zipcode - val TIME_MM_SS = "time_mm_ss" // (MM:SS) - val SIGNATURE = "signature" - val NUMBER_2DP = "number_2dp" // number with exactly two decimal places (125.34) -} diff --git a/ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapExtensionSpec.scala b/ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapExtensionSpec.scala deleted file mode 100644 index a4ffd77f..00000000 --- a/ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapExtensionSpec.scala +++ /dev/null @@ -1,63 +0,0 @@ -package io.ignifyr.redcap - -import com.typesafe.config.ConfigFactory -import io.ignifyr.engine.spi.ExtensionRegistry -import io.ignifyr.server.common.spi.IgnifyrServerExtensions -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Verifies the REDCap module is discovered through both extension SPIs when it is on the - * classpath (no Docker required), and exercises the pure data-dictionary extraction. The - * endpoint-level import behaviour stays covered by the server's SchemaEndpointTest. - */ -class RedCapExtensionSpec extends AnyFlatSpec with Matchers { - - "The RedCap engine extension" should "register the extract-redcap-schemas CLI command through ServiceLoader" in { - val provider = ExtensionRegistry.cliCommands.get("extract-redcap-schemas") - provider shouldBe defined - provider.get.helpText shouldBe defined - } - - it should "translate Boot's flag options into the command's positional arguments in order" in { - val provider = ExtensionRegistry.cliCommands("extract-redcap-schemas") - provider.argsFromOptions( - Map("encoding" -> "ISO-8859-9", "data-dictionary" -> "dict.csv", "definition-root-url" -> "http://test") - ) shouldBe Seq("dict.csv", "http://test", "ISO-8859-9") - provider.argsFromOptions(Map("data-dictionary" -> "dict.csv")) shouldBe Seq("dict.csv") - } - - "The RedCap server extension" should "be discovered through ServiceLoader and gate its proxy routes on configuration" in { - val withConfig = - IgnifyrServerExtensions.load( - ConfigFactory.parseString("""ignifyr-redcap { endpoint = "http://localhost:8095/tofhir-redcap" }""") - ) - val redcap = withConfig.find(_.id == "redcap") - redcap shouldBe defined - redcap.get.rootRoutes should have size 1 - - val withoutConfig = IgnifyrServerExtensions.load(ConfigFactory.empty()) - withoutConfig.find(_.id == "redcap").get.rootRoutes shouldBe empty - } - - "RedCapUtil" should "extract a schema per form and inject the record identifier field" in { - val rows = Seq( - Map( - RedCapDataDictionaryColumns.VARIABLE_FIELD_NAME -> "age", - RedCapDataDictionaryColumns.FORM_NAME -> "test_form", - RedCapDataDictionaryColumns.FIELD_TYPE -> RedCapDataTypes.TEXT, - RedCapDataDictionaryColumns.FIELD_LABEL -> "Age", - RedCapDataDictionaryColumns.REQUIRED_FIELD -> "y", - RedCapDataDictionaryColumns.TEXT_VALIDATION_TYPE -> RedCapTextValidationTypes.INTEGER - ) - ) - val definitions = RedCapUtil.extractSchemasAsSchemaDefinitions(rows, "http://test-schema", "record_id") - definitions should have size 1 - val schema = definitions.head - schema.id shouldBe "Test_form" - schema.url shouldBe "http://test-schema/StructureDefinition/Test_form" - val fields = schema.fieldDefinitions.get - fields.map(_.id) should contain theSameElementsInOrderAs Seq("record_id", "age") - fields.foreach(_.isPrimitive shouldBe true) - } -} diff --git a/ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapUtilDataTypeSpec.scala b/ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapUtilDataTypeSpec.scala deleted file mode 100644 index 72042a19..00000000 --- a/ignifyr-redcap/src/test/scala/io/ignifyr/redcap/RedCapUtilDataTypeSpec.scala +++ /dev/null @@ -1,256 +0,0 @@ -package io.ignifyr.redcap - -import io.onfhir.api.FHIR_DATA_TYPES -import io.onfhir.definitions.common.model.SimpleStructureDefinition -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import javax.ws.rs.BadRequestException - -/** - * The REDCap field type + text-validation type -> FHIR data type table, exercised through the pure - * entry point `extractSchemasAsSchemaDefinitions`. This is the widest lookup table in the repo and the - * only thing standing between a data dictionary and the column types every downstream mapping reads, so - * a wrong entry produces a schema that parses but types its data incorrectly — never an error. - */ -class RedCapUtilDataTypeSpec extends AnyFlatSpec with Matchers { - - private val definitionRootUrl = "https://ignifyr.io/fhir" - - /** One data-dictionary row, with only the columns the extractor actually reads. */ - private def row( - fieldType: String, - textValidationType: Option[String] = None, - variableName: String = "field1", - formName: String = "instrument", - required: String = "", - fieldLabel: String = "A label", - fieldNotes: Option[String] = None - ): Map[String, String] = - Map( - RedCapDataDictionaryColumns.FORM_NAME -> formName, - RedCapDataDictionaryColumns.VARIABLE_FIELD_NAME -> variableName, - RedCapDataDictionaryColumns.FIELD_TYPE -> fieldType, - RedCapDataDictionaryColumns.FIELD_LABEL -> fieldLabel, - RedCapDataDictionaryColumns.REQUIRED_FIELD -> required - ) ++ - textValidationType.map(RedCapDataDictionaryColumns.TEXT_VALIDATION_TYPE -> _) ++ - fieldNotes.map(RedCapDataDictionaryColumns.FIELD_NOTES -> _) - - /** The extracted field definitions of the single form, excluding the injected record-identifier field. */ - private def fieldsOf(rows: Map[String, String]*): Seq[SimpleStructureDefinition] = - RedCapUtil - .extractSchemasAsSchemaDefinitions(rows, definitionRootUrl, recordIdField = "record_id") - .head - .fieldDefinitions - .get - .filterNot(_.id == "record_id") - - private def fhirTypeOf(fieldType: String, textValidationType: Option[String] = None): String = - fieldsOf(row(fieldType, textValidationType)).head.dataTypes.get.head.dataType - - "getDataType" should "map every date text-validation type onto a FHIR date" in { - Seq( - RedCapTextValidationTypes.DATE_DMY, - RedCapTextValidationTypes.DATE_MDY, - RedCapTextValidationTypes.DATE_YMD - ).foreach(validation => fhirTypeOf(RedCapDataTypes.TEXT, Some(validation)) shouldBe FHIR_DATA_TYPES.DATE) - } - - it should "map every datetime text-validation type onto a FHIR dateTime" in { - Seq( - RedCapTextValidationTypes.DATETIME_DMY, - RedCapTextValidationTypes.DATETIME_MDY, - RedCapTextValidationTypes.DATETIME_YMD, - RedCapTextValidationTypes.DATETIME_SECOND_DMY, - RedCapTextValidationTypes.DATETIME_SECONDS_MDY, - RedCapTextValidationTypes.DATETIME_SECONDS_YMD - ).foreach(validation => fhirTypeOf(RedCapDataTypes.TEXT, Some(validation)) shouldBe FHIR_DATA_TYPES.DATETIME) - } - - it should "map both time text-validation types onto a FHIR time" in { - Seq(RedCapTextValidationTypes.TIME, RedCapTextValidationTypes.TIME_MM_SS) - .foreach(validation => fhirTypeOf(RedCapDataTypes.TEXT, Some(validation)) shouldBe FHIR_DATA_TYPES.TIME) - } - - it should "map the numeric text-validation types onto FHIR integer and decimal" in { - fhirTypeOf(RedCapDataTypes.TEXT, Some(RedCapTextValidationTypes.INTEGER)) shouldBe FHIR_DATA_TYPES.INTEGER - fhirTypeOf(RedCapDataTypes.TEXT, Some(RedCapTextValidationTypes.NUMBER)) shouldBe FHIR_DATA_TYPES.DECIMAL - fhirTypeOf(RedCapDataTypes.TEXT, Some(RedCapTextValidationTypes.NUMBER_2DP)) shouldBe FHIR_DATA_TYPES.DECIMAL - } - - it should "map the free-form text validations onto a FHIR string" in { - Seq( - RedCapTextValidationTypes.EMAIL, - RedCapTextValidationTypes.PHONE, - RedCapTextValidationTypes.ZIP_CODE, - RedCapTextValidationTypes.POSTAL_CODE_GERMANY - ).foreach(validation => fhirTypeOf(RedCapDataTypes.TEXT, Some(validation)) shouldBe FHIR_DATA_TYPES.STRING) - } - - // REDCap leaves the validation column empty for an unvalidated text field, and the column may be absent - // altogether in an older export. Both mean "plain string". - it should "map an empty and an absent text validation onto a FHIR string" in { - fhirTypeOf(RedCapDataTypes.TEXT, Some("")) shouldBe FHIR_DATA_TYPES.STRING - fhirTypeOf(RedCapDataTypes.TEXT, None) shouldBe FHIR_DATA_TYPES.STRING - } - - it should "treat a notes field like a text field" in { - fhirTypeOf(RedCapDataTypes.NOTES, None) shouldBe FHIR_DATA_TYPES.STRING - fhirTypeOf(RedCapDataTypes.NOTES, Some(RedCapTextValidationTypes.DATE_YMD)) shouldBe FHIR_DATA_TYPES.DATE - } - - it should "map the multiple-choice field types onto a FHIR code" in { - Seq(RedCapDataTypes.RADIO, RedCapDataTypes.DROPDOWN, RedCapDataTypes.CHECKBOXES, RedCapDataTypes.SQL) - .foreach(fieldType => fhirTypeOf(fieldType) shouldBe FHIR_DATA_TYPES.CODE) - } - - it should "map the boolean field types onto a FHIR boolean" in { - Seq(RedCapDataTypes.YES_NO, RedCapDataTypes.TRUE_FALSE) - .foreach(fieldType => fhirTypeOf(fieldType) shouldBe FHIR_DATA_TYPES.BOOLEAN) - } - - it should "map calc onto a decimal and slider onto an integer" in { - fhirTypeOf(RedCapDataTypes.CALC) shouldBe FHIR_DATA_TYPES.DECIMAL - fhirTypeOf(RedCapDataTypes.SLIDER) shouldBe FHIR_DATA_TYPES.INTEGER - } - - it should "map a file onto base64Binary, or a Signature when it is signed" in { - fhirTypeOf(RedCapDataTypes.FILE, None) shouldBe FHIR_DATA_TYPES.BASE64BINARY - fhirTypeOf(RedCapDataTypes.FILE, Some("")) shouldBe FHIR_DATA_TYPES.BASE64BINARY - fhirTypeOf(RedCapDataTypes.FILE, Some(RedCapTextValidationTypes.SIGNATURE)) shouldBe FHIR_DATA_TYPES.SIGNATURE - } - - it should "attach the canonical profile url of the mapped type" in { - fieldsOf(row(RedCapDataTypes.YES_NO)).head.dataTypes.get.head.profiles.get.head shouldBe - s"http://hl7.org/fhir/StructureDefinition/${FHIR_DATA_TYPES.BOOLEAN}" - } - - it should "reject an unknown field type" in { - val thrown = the[IllegalArgumentException] thrownBy fieldsOf(row("hologram")) - thrown.getMessage should include("Invalid data type: hologram") - } - - it should "reject an unknown text validation type" in { - val thrown = the[IllegalArgumentException] thrownBy fieldsOf(row(RedCapDataTypes.TEXT, Some("runes"))) - thrown.getMessage should include("Invalid text validation type for texts: runes") - } - - it should "reject an unknown text validation type on a file field" in { - val thrown = the[IllegalArgumentException] thrownBy fieldsOf(row(RedCapDataTypes.FILE, Some("runes"))) - thrown.getMessage should include("Invalid text validation type for files: runes") - } - - "getCardinality" should "make a checkbox field repeating and everything else single" in { - val checkbox = fieldsOf(row(RedCapDataTypes.CHECKBOXES)).head - checkbox.isArray shouldBe true - checkbox.maxCardinality shouldBe None // "*" - - val radio = fieldsOf(row(RedCapDataTypes.RADIO)).head - radio.isArray shouldBe false - radio.maxCardinality shouldBe Some(1) - } - - it should "require a field only when the dictionary marks it with y" in { - fieldsOf(row(RedCapDataTypes.TEXT, required = "y")).head.minCardinality shouldBe 1 - fieldsOf(row(RedCapDataTypes.TEXT, required = "")).head.minCardinality shouldBe 0 - fieldsOf(row(RedCapDataTypes.TEXT, required = "n")).head.minCardinality shouldBe 0 - } - - "extractSchemasAsSchemaDefinitions" should "produce one schema per form, named after it" in { - val schemas = RedCapUtil.extractSchemasAsSchemaDefinitions( - Seq( - row(RedCapDataTypes.TEXT, variableName = "a", formName = "demographics"), - row(RedCapDataTypes.TEXT, variableName = "b", formName = "vitals") - ), - definitionRootUrl, - recordIdField = "record_id" - ) - schemas.map(_.name) should contain theSameElementsAs Seq("demographics", "vitals") - schemas.map(_.id) should contain theSameElementsAs Seq("Demographics", "Vitals") - schemas.map(_.url) should contain(s"$definitionRootUrl/StructureDefinition/Vitals") - } - - // A descriptive field displays text and returns no data, so REDCap's export omits it and so must the schema. - it should "omit a descriptive field entirely" in { - val fields = fieldsOf( - row(RedCapDataTypes.DESCRIPTIVE, variableName = "banner"), - row(RedCapDataTypes.TEXT, variableName = "name") - ) - fields.map(_.id) shouldBe Seq("name") - } - - it should "inject the record identifier field when the form does not declare it" in { - val schema = RedCapUtil - .extractSchemasAsSchemaDefinitions( - Seq(row(RedCapDataTypes.TEXT, variableName = "name")), - definitionRootUrl, - "record_id" - ) - .head - val recordId = schema.fieldDefinitions.get.head - recordId.id shouldBe "record_id" - recordId.path shouldBe "Instrument.record_id" - recordId.short shouldBe Some("Record Identifier") - recordId.dataTypes.get.head.dataType shouldBe FHIR_DATA_TYPES.STRING - recordId.minCardinality shouldBe 0 - recordId.maxCardinality shouldBe Some(1) - } - - it should "not inject the record identifier field twice when the form already declares it" in { - val schema = RedCapUtil - .extractSchemasAsSchemaDefinitions( - Seq(row(RedCapDataTypes.TEXT, variableName = "record_id")), - definitionRootUrl, - "record_id" - ) - .head - schema.fieldDefinitions.get.count(_.id == "record_id") shouldBe 1 - } - - /* - * `recordIdField` defaults to "" and the CLI (`extract-redcap-schemas`) takes that default — only the - * server import route passes a real value from a query parameter. The result is an injected field with - * an empty id and a trailing-dot path. Pinned rather than changed: CLI-extracted schemas already on - * disk carry exactly this shape, so altering it is a migration decision, not a test fix. - */ - it should "inject an unnamed record identifier field when none is supplied (the CLI default)" in { - val schema = RedCapUtil - .extractSchemasAsSchemaDefinitions(Seq(row(RedCapDataTypes.TEXT, variableName = "name")), definitionRootUrl, "") - .head - val recordId = schema.fieldDefinitions.get.head - recordId.id shouldBe "" - recordId.path shouldBe "Instrument." - } - - // REDCap exports the dictionary as UTF-8-with-BOM, so the first column name arrives BOM-prefixed. - it should "fall back to the BOM-prefixed variable name column" in { - val bomRow = Map( - RedCapDataDictionaryColumns.FORM_NAME -> "instrument", - RedCapDataDictionaryColumns.VARIABLE_FIELD_NAME_WITH_BOM -> "name", - RedCapDataDictionaryColumns.FIELD_TYPE -> RedCapDataTypes.TEXT, - RedCapDataDictionaryColumns.FIELD_LABEL -> "Name", - RedCapDataDictionaryColumns.REQUIRED_FIELD -> "" - ) - fieldsOf(bomRow).map(_.id) shouldBe Seq("name") - } - - it should "reject a dictionary with no variable name column at all" in { - val noNameRow = Map( - RedCapDataDictionaryColumns.FORM_NAME -> "instrument", - RedCapDataDictionaryColumns.FIELD_TYPE -> RedCapDataTypes.TEXT, - RedCapDataDictionaryColumns.FIELD_LABEL -> "Name", - RedCapDataDictionaryColumns.REQUIRED_FIELD -> "" - ) - a[BadRequestException] should be thrownBy fieldsOf(noNameRow) - } - - it should "use the field label as the definition when there are no field notes" in { - val withNotes = fieldsOf(row(RedCapDataTypes.TEXT, fieldLabel = "Label", fieldNotes = Some("Notes"))).head - withNotes.short shouldBe Some("Label") - withNotes.definition shouldBe Some("Notes") - - val withoutNotes = fieldsOf(row(RedCapDataTypes.TEXT, fieldLabel = "Label")).head - withoutNotes.definition shouldBe Some("Label") - } -} diff --git a/ignifyr-runtime-scheduling/CLAUDE.md b/ignifyr-runtime-scheduling/CLAUDE.md deleted file mode 100644 index a9854bcc..00000000 --- a/ignifyr-runtime-scheduling/CLAUDE.md +++ /dev/null @@ -1,55 +0,0 @@ -# ignifyr-runtime-scheduling — cron scheduling capability - -Enterprise module providing cron-driven periodic batch execution as an installable capability behind -the engine's `SchedulerProvider` SPI, so the community engine carries **no** scheduling dependency. -Package root `io.ignifyr.runtime.scheduling`. Bundled into `ignifyr-server`, **not** `ignifyr-cli`. - -It owns the cron4j scheduler, per-job last-synchronization bookkeeping (for incremental source syncs), -and the registry of scheduled executions — logic physically moved out of the engine's -`FhirMappingJobManager` (`scheduleMappingJob`/`runnableMappingJob`/`getScheduledTimeRange`) and a -now-deleted `MappingJobScheduler` class (don't go looking for it — it exists only in git history). A job -carrying `schedulingSettings` with no provider installed parses fine and fails with -`MissingCapabilityException`. - -## Layout (`src/main/scala/io/ignifyr/runtime/scheduling/`) -- `SchedulingRuntimeExtension` — the `IgnifyrExtension` impl (`id = "runtime-scheduling"`); overrides - `schedulerProvider` to `Some(new Cron4jSchedulerProvider)`. Named in the sole - `META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension` file. Surfaced as `ExtensionRegistry.scheduler` - (single-capability — at most one). -- `Cron4jSchedulerProvider` — the `SchedulerProvider` impl. Validates the cron expression, requires a - non-empty `ignifyrDbFolderPath`, owns/starts the cron4j `Scheduler`, and on each fire runs an - incremental batch over `(lastSyncTime, now)`, persisting the new sync time. The provider now owns the - full start/register/deschedule dance — callers only schedule and deschedule. -- `ScheduledJobRegistry` — in-memory `jobId → executionId → (Scheduler, execution)` store (the - scheduling counterpart of the engine's `RunningJobRegistry`). Bridges each cron fire into - `RunningJobRegistry` (register on taskLaunching, `handleCompletedBatchJob` on success/failure) and - logs SCHEDULED / DESCHEDULED. - -## Notes -- **cron4j (`it.sauronsoftware.cron4j`) is enterprise-only** and banned from community modules by the - root `ban-enterprise-deps` enforcer gate. This module is its **sole declaration site** outside root - `dependencyManagement` — `ignifyr-server` never declares it and only gets it transitively through this - module. Note the gate is **opt-in, not a whitelist**: only the 8 poms declaring the - `ban-enterprise-deps` execution are constrained, so any other enterprise module could carry cron4j and - simply doesn't. -- Incremental-sync state is a filesystem convention: one append-only file per job at - `/scheduler/.txt` whose last line is the previous sync timestamp; - missing file → seed from `initialTime` (default epoch 0). -- Documented gotcha: `descheduleJobExecution` cannot actually stop an **in-flight** scheduled run - (Spark distributes tasks across threads; job-group cancellation doesn't apply) — see the TODO in code. - -## Tests -Tier-split. **Short** (`wildcardSuites=io.ignifyr.runtime.scheduling`, no Docker): -`SchedulingRuntimeExtensionSpec` (ServiceLoader discovery + empty-registry state) and -`ScheduledTimeRangeSpec` — the incremental-sync bookkeeping every scheduled fire depends on. Each fire -reads the previous instant from `/scheduler/.txt`, syncs `(lastSync, now)`, -then appends the new instant to that same file; getting the lower bound wrong is silent in **both** -directions (too early re-imports mapped data, too late skips a window forever). The long-tier -`SchedulingTest` only ever fires once against a fresh directory, so it reaches none of those states — -which is why `getScheduledTimeRange` is `private[scheduling]` rather than `private`. **Long** -(`membersOnlySuites=io.ignifyr.integrationtest`, gated on `${skipITs}`, Docker): `SchedulingTest` drives a -full cron/SQL→FHIR incremental sync against H2 + an onFHIR TestContainer (sleeps ~61s for one -every-minute fire, then deschedules) — `mvn -B verify -pl ignifyr-runtime-scheduling -DskipITs=false`. **Three** test-scope Ignifyr deps, all needed for ServiceLoader discovery on the test -classpath: `ignifyr-testkit` (harness + fixtures), `ignifyr-connector-sql` (the `SqlSource` reader) and -`ignifyr-sink-fhir` (the test writes to the onFHIR container via `FhirRepositorySinkSettings`, and since -the sink split that writer is no longer in the engine). diff --git a/ignifyr-runtime-scheduling/pom.xml b/ignifyr-runtime-scheduling/pom.xml deleted file mode 100644 index 54ee20dc..00000000 --- a/ignifyr-runtime-scheduling/pom.xml +++ /dev/null @@ -1,101 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-runtime-scheduling_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - - - test - - test - - - io.ignifyr.runtime.scheduling - - - - - integration-test - integration-test - - test - - - ${skipITs} - io.ignifyr.integrationtest - - - - - - - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - - it.sauronsoftware.cron4j - cron4j - - - - - io.ignifyr - ignifyr-testkit_2.13 - test - - - - io.ignifyr - ignifyr-connector-sql_2.13 - test - - - - - io.ignifyr - ignifyr-sink-fhir_2.13 - test - - - - diff --git a/ignifyr-runtime-scheduling/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension b/ignifyr-runtime-scheduling/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension deleted file mode 100644 index 5844eea4..00000000 --- a/ignifyr-runtime-scheduling/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension +++ /dev/null @@ -1,2 +0,0 @@ -# Ignifyr scheduling execution runtime. -io.ignifyr.runtime.scheduling.SchedulingRuntimeExtension diff --git a/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/Cron4jSchedulerProvider.scala b/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/Cron4jSchedulerProvider.scala deleted file mode 100644 index 033b9370..00000000 --- a/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/Cron4jSchedulerProvider.scala +++ /dev/null @@ -1,176 +0,0 @@ -package io.ignifyr.runtime.scheduling - -import com.typesafe.scalalogging.Logger -import io.ignifyr.engine.execution.RunningJobRegistry -import io.ignifyr.engine.mapping.job.IFhirMappingJobManager -import io.ignifyr.engine.model.{ - BaseSchedulingSettings, - FhirMappingJobExecution, - SinkSettings, - IdentityServiceSettings, - MappingJobSourceSettings, - SchedulingSettings, - SQLSchedulingSettings, - TerminologyServiceSettings -} -import io.ignifyr.engine.spi.SchedulerProvider -import it.sauronsoftware.cron4j.{Scheduler, SchedulingPattern} - -import java.io.{File, FileNotFoundException, FileWriter} -import java.net.URI -import java.nio.file.Paths -import java.time.{Instant, LocalDateTime, ZoneOffset} -import javax.ws.rs.BadRequestException -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, Future} -import scala.io.Source - -/** - * cron4j-backed [[SchedulerProvider]]: schedules batch mapping jobs on a cron expression, keeps a - * per-job last-synchronization time (under `/scheduler/.txt`) so each - * run only maps data added since the previous run, and tracks scheduled executions in a - * [[ScheduledJobRegistry]]. This is the body that previously lived in `FhirMappingJobManager` - * (`scheduleMappingJob`/`runnableMappingJob`/`getScheduledTimeRange`) plus the old `MappingJobScheduler`. - */ -class Cron4jSchedulerProvider extends SchedulerProvider { - - import io.ignifyr.engine.Execution.actorSystem.dispatcher - - private val logger: Logger = Logger(this.getClass) - - private val scheduledJobRegistry: ScheduledJobRegistry = new ScheduledJobRegistry - - override def scheduleMappingJob( - jobManager: IFhirMappingJobManager, - runningJobRegistry: RunningJobRegistry, - ignifyrDbFolderPath: String, - mappingJobExecution: FhirMappingJobExecution, - sourceSettings: Map[String, MappingJobSourceSettings], - sinkSettings: SinkSettings, - schedulingSettings: BaseSchedulingSettings, - terminologyServiceSettings: Option[TerminologyServiceSettings], - identityServiceSettings: Option[IdentityServiceSettings] - ): Unit = { - if (ignifyrDbFolderPath.isEmpty) { - throw new IllegalArgumentException( - "A scheduled mapping job was submitted, but ignifyr.db is not configured." - ) - } - // validate the cron expression - if (!SchedulingPattern.validate(schedulingSettings.cronExpression)) { - throw new BadRequestException(s"'${schedulingSettings.cronExpression}' is not a valid cron expression.") - } - // Folder that acts as the scheduler's state store for last-synchronization times. - val folderUri: URI = Paths.get(ignifyrDbFolderPath, "scheduler").toUri - // find the start time for SQL data sources - val startTime: LocalDateTime = schedulingSettings match { - case SQLSchedulingSettings(_, initialTime) => - if (initialTime.isEmpty) { - logger.info( - s"initialTime is not specified in the mappingJob. I will sync all the data from midnight, January 1, 1970 to the next run time." - ) - Instant.ofEpochMilli(0L).atOffset(ZoneOffset.UTC).toLocalDateTime - } else { - LocalDateTime.parse(initialTime.get) - } - case SchedulingSettings(_) => - Instant.ofEpochMilli(0L).atOffset(ZoneOffset.UTC).toLocalDateTime - } - // Schedule a task - val scheduler = new Scheduler() - scheduler.schedule( - schedulingSettings.cronExpression, - new Runnable() { - override def run(): Unit = { - val scheduledJob = runnableMappingJob( - jobManager, - folderUri, - mappingJobExecution, - startTime, - sourceSettings, - sinkSettings, - terminologyServiceSettings, - identityServiceSettings, - schedulingSettings - ) - Await.result(scheduledJob, Duration.Inf) - } - } - ) - // Register the scheduled execution (attaches the listener bridging cron fires into the running-job - // registry and logs SCHEDULED), then start the scheduler. - scheduledJobRegistry.registerSchedulingJob(runningJobRegistry, mappingJobExecution, scheduler) - scheduler.start() - } - - override def isScheduled(jobId: String, executionId: String): Boolean = - scheduledJobRegistry.isScheduled(jobId, executionId) - - override def getScheduledExecutions(jobId: String): Set[String] = - scheduledJobRegistry.getScheduledExecutions(jobId) - - override def descheduleJobExecution( - runningJobRegistry: RunningJobRegistry, - jobId: String, - executionId: String - ): Unit = - scheduledJobRegistry.descheduleJobExecution(runningJobRegistry, jobId, executionId) - - /** - * Runnable body for a scheduled periodic mapping job: computes the incremental time range from the - * last-sync file, executes the batch job over it, then records the new sync time. - */ - private def runnableMappingJob( - jobManager: IFhirMappingJobManager, - folderUri: URI, - mappingJobExecution: FhirMappingJobExecution, - startTime: LocalDateTime, - sourceSettings: Map[String, MappingJobSourceSettings], - sinkSettings: SinkSettings, - terminologyServiceSettings: Option[TerminologyServiceSettings], - identityServiceSettings: Option[IdentityServiceSettings], - schedulingSettings: BaseSchedulingSettings - ): Future[Unit] = { - val timeRange = getScheduledTimeRange(mappingJobExecution.jobId, folderUri, startTime) - logger.info(s"Running scheduled job with the expression: ${schedulingSettings.cronExpression}") - logger.info(s"Synchronizing data between ${timeRange._1} and ${timeRange._2}") - jobManager - .executeMappingJob( - mappingJobExecution, - sourceSettings, - sinkSettings, - terminologyServiceSettings, - identityServiceSettings, - Some(timeRange) - ) - .map(_ => { - val writer = - new FileWriter(s"${folderUri.getPath}/${mappingJobExecution.jobId}.txt", true) - try writer.write(timeRange._2.toString + "\n") - finally writer.close() // write last sync time to the file - }) - } - - /** - * Reads the latest synchronization time point for the job from its last-sync file, returning the - * time range (lastSyncTime, now) — or (startTime, now) when no sync has happened yet. - */ - private[scheduling] def getScheduledTimeRange( - mappingJobId: String, - folderUri: URI, - startTime: LocalDateTime - ): (LocalDateTime, LocalDateTime) = { - val file = new File(folderUri) - if (!file.exists || !file.isDirectory) { - file.mkdirs() - } - try { - val source = Source.fromFile(s"${folderUri.getPath}/$mappingJobId.txt") // read last sync time from file - val lines = source.getLines() - val lastLine = lines.foldLeft("") { case (_, line) => line } - (LocalDateTime.parse(lastLine), LocalDateTime.now()) // (lastSyncTime, currentTime)} - } catch { - case _: FileNotFoundException => (startTime, LocalDateTime.now()) - } - } -} diff --git a/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/ScheduledJobRegistry.scala b/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/ScheduledJobRegistry.scala deleted file mode 100644 index ed04c493..00000000 --- a/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/ScheduledJobRegistry.scala +++ /dev/null @@ -1,129 +0,0 @@ -package io.ignifyr.runtime.scheduling - -import com.typesafe.scalalogging.Logger -import io.ignifyr.engine.Execution.actorSystem -import io.ignifyr.engine.execution.RunningJobRegistry -import io.ignifyr.engine.execution.log.ExecutionLogger -import io.ignifyr.engine.model.{FhirMappingJobExecution, FhirMappingJobResult} -import it.sauronsoftware.cron4j.{Scheduler, SchedulerListener, TaskExecutor} - -/** - * In-memory registry of cron-scheduled mapping-job executions — the scheduling counterpart of the - * engine's [[RunningJobRegistry]] (which tracks running batch/streaming jobs). Lives in the - * enterprise scheduling runtime with the cron4j [[Scheduler]] so the community engine carries no - * scheduling dependency. - * - * On each cron fire, the attached [[SchedulerListener]] bridges the run into the engine's running-job - * registry: it registers the launching run as a batch job (so it is visible/stoppable) and, on - * completion, triggers the usual post-batch archiving + removal via - * [[RunningJobRegistry.handleCompletedBatchJob]]. - */ -class ScheduledJobRegistry { - - import actorSystem.dispatcher - - private val logger: Logger = Logger(this.getClass) - - // Keeps the scheduled jobs in the form of: jobId -> (executionId -> (Scheduler, execution)) - private val scheduledTasks - : collection.mutable.Map[String, collection.mutable.Map[String, (Scheduler, FhirMappingJobExecution)]] = - collection.mutable.Map[String, collection.mutable.Map[String, (Scheduler, FhirMappingJobExecution)]]() - - // When the actor system is terminated i.e. the system is shutdown, log every scheduled mapping job - // as 'DESCHEDULED' (the engine's RunningJobRegistry logs running jobs as 'STOPPED'). - actorSystem.whenTerminated - .map(_ => { - scheduledTasks.values - .flatMap(_.values.map(_._2)) - .foreach(execution => ExecutionLogger.logExecutionStatus(execution, FhirMappingJobResult.DESCHEDULED)) - }) - - /** - * Registers a scheduling job with the specified mapping job execution and scheduler, attaching a - * listener that drives each cron fire into the given running-job registry. - * - * @param runningJobRegistry Engine registry of running jobs, updated per cron fire. - * @param mappingJobExecution The mapping job execution. - * @param scheduler The cron4j scheduler associated with the job execution. - */ - def registerSchedulingJob( - runningJobRegistry: RunningJobRegistry, - mappingJobExecution: FhirMappingJobExecution, - scheduler: Scheduler - ): Unit = { - // add it to the scheduledTasks map - scheduledTasks - .getOrElseUpdate( - mappingJobExecution.jobId, - collection.mutable.Map[String, (Scheduler, FhirMappingJobExecution)]() - ) - .put(mappingJobExecution.id, (scheduler, mappingJobExecution)) - // log execution status as 'SCHEDULED' - ExecutionLogger.logExecutionStatus(mappingJobExecution, FhirMappingJobResult.SCHEDULED) - // add a scheduler listener to monitor task events - scheduler.addSchedulerListener(new SchedulerListener { - override def taskLaunching(executor: TaskExecutor): Unit = { - runningJobRegistry.registerBatchJob( - mappingJobExecution, - None, - s"Spark job for job: ${mappingJobExecution.jobId} mappingTasks: ${mappingJobExecution.mappingTasks.map(_.name).mkString(" ")}" - ) - } - - override def taskSucceeded(executor: TaskExecutor): Unit = { - runningJobRegistry.handleCompletedBatchJob(mappingJobExecution) - } - - override def taskFailed(executor: TaskExecutor, exception: Throwable): Unit = { - runningJobRegistry.handleCompletedBatchJob(mappingJobExecution) - } - }) - } - - /** - * Deschedules a job execution. - * - * @param runningJobRegistry Engine registry of running jobs (to stop any in-flight run). - * @param jobId The ID of the job. - * @param executionId The ID of the execution. - */ - def descheduleJobExecution(runningJobRegistry: RunningJobRegistry, jobId: String, executionId: String): Unit = { - // TODO: We call this function but it does not actually stop the execution of a scheduled mappings jobs - // due to the fact that Spark can distribute the tasks into several threads and our setJobGroup/cancelJobGroup - // logic cannot work properly in this case - // stop the job execution - runningJobRegistry.stopJobExecution(jobId, executionId) - // stop the scheduler for the specified job execution - scheduledTasks(jobId)(executionId)._1.stop() - logger.debug(s"Descheduled the mapping job with id: $jobId and execution: $executionId") - // log execution status as 'DESCHEDULED' - ExecutionLogger.logExecutionStatus(scheduledTasks(jobId)(executionId)._2, FhirMappingJobResult.DESCHEDULED) - // remove the execution from the scheduledTask Map - scheduledTasks(jobId).remove(executionId) - // if there are no executions left for the job, remove the job from the map - if (!scheduledTasks.contains(jobId)) { - scheduledTasks.remove(jobId) - } - } - - /** - * Checks if a job with the given execution ID is scheduled. - * - * @param jobId The ID of the job - * @param executionId The ID of the execution - * @return True if the given job execution is scheduled, otherwise false. - */ - def isScheduled(jobId: String, executionId: String): Boolean = { - scheduledTasks.contains(jobId) && scheduledTasks(jobId).contains(executionId) - } - - /** - * Gets scheduled executions for the given job. - * - * @param jobId Identifier of the job - * @return A set of execution ids - */ - def getScheduledExecutions(jobId: String): Set[String] = { - scheduledTasks.get(jobId).map(_.keySet).getOrElse(Set.empty).toSet - } -} diff --git a/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtension.scala b/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtension.scala deleted file mode 100644 index 5cff0e2a..00000000 --- a/ignifyr-runtime-scheduling/src/main/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtension.scala +++ /dev/null @@ -1,13 +0,0 @@ -package io.ignifyr.runtime.scheduling - -import io.ignifyr.engine.spi.{IgnifyrExtension, SchedulerProvider} - -/** - * Registers the scheduled (cron) execution capability with the engine via ServiceLoader. - */ -class SchedulingRuntimeExtension extends IgnifyrExtension { - - override val id: String = "runtime-scheduling" - - override def schedulerProvider: Option[SchedulerProvider] = Some(new Cron4jSchedulerProvider) -} diff --git a/ignifyr-runtime-scheduling/src/test/resources/application.conf b/ignifyr-runtime-scheduling/src/test/resources/application.conf deleted file mode 100644 index adda3038..00000000 --- a/ignifyr-runtime-scheduling/src/test/resources/application.conf +++ /dev/null @@ -1,39 +0,0 @@ -# Ignifyr configurations (runtime-scheduling suites; fixtures come from ignifyr-testkit) -ignifyr { - - # A path to a file/directory from where any File system readings should use within the mappingjob definition. - context-path = "testcontext" - - mappings = { - # The repository where the mapping definitions are kept (shipped by ignifyr-testkit). - repository = { - folder-path = "/test-mappings" - } - schemas = { - repository = { - folder-path = "/test-schemas" - } - } - # Timeout for each mapping execution on an individual input record - timeout = 5 seconds - } - - # Settings for FHIR repository writer - fhir-server-writer { - batch-group-size = 20 - } - - # Database folder of Ignifyr (the scheduling test passes its own temp path to the provider) - db = "db" -} - -# Spark configurations -spark = { - app.name = "Ignifyr runtime-scheduling TEST" - master = "local[4]" - checkpoint-dir = "checkpoint" -} - -akka = { - daemonic = "on" -} diff --git a/ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-drop.sql b/ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-drop.sql deleted file mode 100644 index 02e63d14..00000000 --- a/ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-drop.sql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE patients; - -DROP TABLE otherobservations; diff --git a/ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-populate.sql b/ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-populate.sql deleted file mode 100644 index 71b63ccc..00000000 --- a/ignifyr-runtime-scheduling/src/test/resources/sql/scheduling-populate.sql +++ /dev/null @@ -1,49 +0,0 @@ -CREATE TABLE patients -( - pid varchar(4) NOT NULL, - gender varchar(8) NULL, - birthDate date NULL, - deceasedDateTime timestamp NULL, - homePostalCode varchar(8) NULL -); - -CREATE TABLE otherobservations -( - pid varchar(255) NOT NULL, - "time" timestamp NULL, - encounterId varchar(255) NULL, - code varchar(255) NULL, - "value" varchar(255) NULL -); - - -INSERT INTO patients -(pid, gender, birthDate, deceasedDateTime, homePostalCode) -VALUES ('p1', 'male', TO_TIMESTAMP('2000-05-10', 'YYYY-MM-DD'), NULL, NULL), - ('p2', 'male', TO_TIMESTAMP('1985-05-08', 'YYYY-MM-DD'), TO_TIMESTAMP('2017-03-10', 'YYYY-MM-DD'), 'G02547'), - ('p3', 'male', TO_TIMESTAMP('1997-02', 'YYYY-MM'), NULL, 'G02547'), - ('p4', 'male', TO_TIMESTAMP('1999-06-05', 'YYYY-MM-DD'), TO_TIMESTAMP('2017-03-10', 'YYYY-MM-DD'), 'H10564'), - ('p5', 'male', TO_TIMESTAMP('1965-10-01', 'YYYY-MM-DD'), TO_TIMESTAMP('2019-04-21', 'YYYY-MM-DD'), 'G02547'), - ('p6', 'female', TO_TIMESTAMP('1991-03', 'YYYY-MM'), TO_TIMESTAMP('2017-03-10', 'YYYY-MM-DD'), 'G02547'), - ('p7', 'female', TO_TIMESTAMP('1972-10-25', 'YYYY-MM-DD'), TO_TIMESTAMP('2017-03-10', 'YYYY-MM-DD'), 'V13135'), - ('p8', 'female', TO_TIMESTAMP('2010-01-10', 'YYYY-MM-DD'), NULL, 'Z54564'), - ('p9', 'female', TO_TIMESTAMP('1999-05-12', 'YYYY-MM-DD'), TO_TIMESTAMP('2017-03-10', 'YYYY-MM-DD'), NULL), - ('p10', 'female', TO_TIMESTAMP('2003-11', 'YYYY-MM'), TO_TIMESTAMP('2017-03-10', 'YYYY-MM-DD'), 'G02547'); - - -INSERT INTO otherobservations -(pid, "time", encounterId, code, "value") -VALUES ('p1', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e1', '9110-8', '450'), - ('p2', TO_TIMESTAMP('2007-10-11T09:00:00+00:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e2', '1335-9', '420'), - ('p3', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e3', '1299-7', '30.5'), - ('p1', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e4', '1035-5', '45'), - ('p3', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e5', '1298-9', '43.2'), - ('p4', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e6', '38214-3', '10'), - ('p5', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e7', '9187-6', '4'), - ('p6', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e8', '445619006', '3'), - ('p7', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e9', '445597002', '5'), - ('p8', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e10', '9270-2', '5'), - ('p4', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e11', '313002', '25'), - ('p1', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e12', '35629', '10'), - ('p2', TO_TIMESTAMP('2007-10-11T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e13', '33834', '35'), - ('p4', TO_TIMESTAMP('2007-10-13T10:00:00+01:00', 'YYYY-MM-DDTHH24:MI:SS+TZH:TZM'), 'e14', '9269-2', '4-3-5-4'); diff --git a/ignifyr-runtime-scheduling/src/test/resources/test-schedule-mappingjob.json b/ignifyr-runtime-scheduling/src/test/resources/test-schedule-mappingjob.json deleted file mode 100644 index 2751df44..00000000 --- a/ignifyr-runtime-scheduling/src/test/resources/test-schedule-mappingjob.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "id": "test-schedule-mapping-job", - "schedulingSettings": { - "jsonClass": "SQLSchedulingSettings", - "cronExpression": "* * * * *", - "initialTime": "2007-10-12T10:00:00" - }, - "sourceSettings": { - "source": { - "jsonClass": "SqlSourceSettings", - "name": "test-source", - "sourceUri": "https://aiccelerate.eu/data-integration-suite/test-data", - "databaseUrl": "jdbc:h2:mem:inputDb;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE", - "username": "", - "password": "" - } - }, - "sinkSettings": { - "jsonClass": "FhirRepositorySinkSettings", - "fhirRepoUrl": "http://localhost:8081/fhir" - }, - "mappings": [ - { - "name": "patient-mapping", - "mappingRef": "https://aiccelerate.eu/fhir/mappings/patient-mapping", - "sourceBinding": { - "source": { - "jsonClass": "SqlSource", - "query": "select * from patients" - } - } - }, - { - "name": "other-observation-mapping", - "mappingRef": "https://aiccelerate.eu/fhir/mappings/other-observation-mapping", - "sourceBinding": { - "source": { - "jsonClass": "SqlSource", - "query": "select * from otherobservations o WHERE o.time > $fromTs AND o.time < $toTs" - } - } - } - ], - "dataProcessingSettings": { - "saveErroneousRecords": false, - "archiveMode": "off" - } -} diff --git a/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/integrationtest/SchedulingTest.scala b/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/integrationtest/SchedulingTest.scala deleted file mode 100644 index 4a425389..00000000 --- a/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/integrationtest/SchedulingTest.scala +++ /dev/null @@ -1,145 +0,0 @@ -package io.ignifyr.integrationtest - -import io.onfhir.api.model.HttpStatus -import io.onfhir.api.client.FhirBatchTransactionRequestBuilder -import io.onfhir.api.util.FHIRUtil -import io.onfhir.path.FhirPathUtilFunctionsFactory -import io.onfhir.util.JsonFormatter.formats -import io.ignifyr.{OnFhirTestContainer, IgnifyrTestSpec} -import io.ignifyr.engine.mapping.context.MappingContextLoader -import io.ignifyr.engine.mapping.job.FhirMappingJobManager -import io.ignifyr.engine.model.{FhirMappingJob, FhirMappingJobExecution, FhirRepositorySinkSettings} -import io.ignifyr.engine.util.{FhirMappingJobFormatter, FhirMappingUtility} -import io.ignifyr.runtime.scheduling.Cron4jSchedulerProvider -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.{Assertion, BeforeAndAfterAll} - -import java.nio.file.Files -import java.sql.{Connection, DriverManager, Statement} -import scala.concurrent.duration.Duration -import scala.concurrent.Await -import scala.io.{BufferedSource, Source} -import scala.util.{Failure, Success, Using} - -/** - * Re-homed from the engine (the parked `SchedulingTest`) and rewritten onto the extracted - * [[Cron4jSchedulerProvider]] SPI: the provider now owns and starts the cron4j scheduler internally, - * so the test only schedules the job and tears it down via `descheduleJobExecution`. Drives a - * cron-scheduled SQL->FHIR incremental sync end to end (H2 source + onFHIR container), so it needs - * the SQL connector on its test classpath for the SqlSource reader to be ServiceLoader-discovered. - */ -class SchedulingTest extends AnyFlatSpec with BeforeAndAfterAll with IgnifyrTestSpec with OnFhirTestContainer { - - import io.ignifyr.engine.Execution.actorSystem.dispatcher - - val DATABASE_URL = "jdbc:h2:mem:inputDb;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE" - - // The provider owns and starts cron4j; the test schedules and, in teardown, deschedules. Its - // per-job last-synchronization files are written under an isolated temp directory. - private val schedulerProvider = new Cron4jSchedulerProvider - private val ignifyrDbFolderPath: String = Files.createTempDirectory("ignifyr-scheduling-test").toString - - override protected def beforeAll(): Unit = { - super.beforeAll() - val sql = readFileContent("/sql/scheduling-populate.sql") - runSQL(sql) - } - - override protected def afterAll(): Unit = { - deleteResources() - val sql = readFileContent("/sql/scheduling-drop.sql") - runSQL(sql) - super.afterAll() - } - - private def readFileContent(fileName: String): String = { - val source: BufferedSource = Source.fromInputStream(getClass.getResourceAsStream(fileName)) - try source.mkString - finally source.close() - } - - private def runSQL(sql: String): Boolean = { - Using.Manager { use => - val con: Connection = use(DriverManager.getConnection(DATABASE_URL)) - val stm: Statement = use(con.createStatement) - stm.execute(sql) - } match { - case Success(value) => value - case Failure(e) => throw e - } - } - - private def deleteResources(): Assertion = { - var batchRequest: FhirBatchTransactionRequestBuilder = onFhirClient.batch() - // Delete all patients between p1-p10 and related observation - (1 to 10).foreach(i => { - batchRequest = batchRequest.entry(_.delete("Patient", FhirMappingUtility.getHashedId("Patient", "p" + i))) - }) - val f = onFhirClient - .search("Observation") - .where("subject", "Patient/" + FhirMappingUtility.getHashedId("Patient", "p4")) flatMap { observationBundle => - observationBundle.searchResults.foreach(obs => { - batchRequest = batchRequest.entry(_.delete("Observation", (obs \ "id").extract[String])) - }) - batchRequest.returnMinimal().asInstanceOf[FhirBatchTransactionRequestBuilder].execute() map { res => - res.httpStatus shouldBe HttpStatus.OK - } - } - Await.result(f, Duration.Inf) - } - - val fhirSinkSettings: FhirRepositorySinkSettings = FhirRepositorySinkSettings(fhirRepoUrl = onFhirClient.getBaseUrl()) - - val testScheduleMappingJobFilePath: String = getClass.getResource("/test-schedule-mappingjob.json").toURI.getPath - - it should "schedule a FhirMappingJob with cron and sink settings restored from a file" in { - val lMappingJob: FhirMappingJob = FhirMappingJobFormatter.readMappingJobFromFile(testScheduleMappingJobFilePath) - - val fhirMappingJobManager = new FhirMappingJobManager( - mappingRepository, - new MappingContextLoader, - schemaRepository, - Map(FhirPathUtilFunctionsFactory.defaultPrefix -> FhirPathUtilFunctionsFactory), - sparkSession - ) - val mappingJobExecution = FhirMappingJobExecution(mappingTasks = lMappingJob.mappings, job = lMappingJob) - // The provider validates the cron expression, registers the execution and starts the scheduler. - schedulerProvider.scheduleMappingJob( - jobManager = fhirMappingJobManager, - runningJobRegistry = runningJobRegistry, - ignifyrDbFolderPath = ignifyrDbFolderPath, - mappingJobExecution = mappingJobExecution, - sourceSettings = lMappingJob.sourceSettings, - sinkSettings = - lMappingJob.sinkSettings.asInstanceOf[FhirRepositorySinkSettings].copy(fhirRepoUrl = onFhirClient.getBaseUrl()), - schedulingSettings = lMappingJob.schedulingSettings.get, - terminologyServiceSettings = None, - identityServiceSettings = None - ) - Thread.sleep(61000) // wait for the job to be executed once (cron runs every minute) - schedulerProvider.descheduleJobExecution(runningJobRegistry, mappingJobExecution.jobId, mappingJobExecution.id) - - val searchTest = - onFhirClient.read("Patient", FhirMappingUtility.getHashedId("Patient", "p8")).executeAndReturnResource() flatMap { - p1Resource => - FHIRUtil.extractIdFromResource(p1Resource) shouldBe FhirMappingUtility.getHashedId("Patient", "p8") - FHIRUtil.extractValue[String](p1Resource, "gender") shouldBe "female" - FHIRUtil.extractValue[String](p1Resource, "birthDate") shouldBe "2010-01-10" - - onFhirClient.search("Observation").where("code", "9269-2").executeAndReturnBundle() flatMap { - observationBundle => - // the Observation with the code 9269-2 matches our time range, others should not - observationBundle.searchResults.length shouldBe 1 - (observationBundle.searchResults.head \ "subject" \ "reference").extract[String] shouldBe - FhirMappingUtility.getHashedReference("Patient", "p4") - // the Observation with the code 445619006, as an example, does not match our time range - onFhirClient.search("Observation").where("code", "445619006").executeAndReturnBundle() map { - emptyObservationBundle => - emptyObservationBundle.searchResults shouldBe empty - } - } - } - Await.result(searchTest, Duration.Inf) - } - -} diff --git a/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/ScheduledTimeRangeSpec.scala b/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/ScheduledTimeRangeSpec.scala deleted file mode 100644 index 1e82cf1d..00000000 --- a/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/ScheduledTimeRangeSpec.scala +++ /dev/null @@ -1,102 +0,0 @@ -package io.ignifyr.runtime.scheduling - -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import java.io.{File, FileWriter} -import java.nio.file.{Files, Paths} -import java.time.LocalDateTime -import java.time.format.DateTimeParseException - -/** - * The incremental-sync bookkeeping behind every scheduled run. Each fire reads the previous - * synchronisation instant from `/scheduler/.txt` and syncs `(lastSync, now)`; - * afterwards it appends the new instant to that same file. - * - * Getting the read wrong is silent in both directions — too early a lower bound re-imports data that was - * already mapped, too late a one skips a window forever — and the long-tier `SchedulingTest` only ever - * exercises a single fire against a fresh directory, so none of the states below are reached there. - */ -class ScheduledTimeRangeSpec extends AnyFlatSpec with Matchers { - - private val provider = new Cron4jSchedulerProvider - private val epoch = LocalDateTime.of(1970, 1, 1, 0, 0) - - /** A fresh, empty scheduler state directory. */ - private def schedulerFolder(): File = - Files.createTempDirectory("ignifyr-scheduler-state").toFile - - /** Appends lines the way `runnableMappingJob` does: the instant's `toString`, one per line. */ - private def writeSyncFile(folder: File, jobId: String, instants: LocalDateTime*): Unit = { - val writer = new FileWriter(s"${folder.toURI.getPath}/$jobId.txt", true) - try instants.foreach(instant => writer.write(instant.toString + "\n")) - finally writer.close() - } - - "getScheduledTimeRange" should "start from the job's initial time when no sync has happened yet" in { - val (from, to) = provider.getScheduledTimeRange("job-1", schedulerFolder().toURI, epoch) - from shouldBe epoch - to should be > from - } - - it should "resume from the recorded synchronisation instant" in { - val folder = schedulerFolder() - val lastSync = LocalDateTime.of(2026, 3, 1, 12, 0) - writeSyncFile(folder, "job-1", lastSync) - - provider.getScheduledTimeRange("job-1", folder.toURI, epoch)._1 shouldBe lastSync - } - - // The file is append-only: every completed run adds a line, so only the last one is the current state. - it should "take the last line of an append-only file" in { - val folder = schedulerFolder() - writeSyncFile( - folder, - "job-1", - LocalDateTime.of(2026, 3, 1, 12, 0), - LocalDateTime.of(2026, 3, 1, 13, 0), - LocalDateTime.of(2026, 3, 1, 14, 0) - ) - - provider.getScheduledTimeRange("job-1", folder.toURI, epoch)._1 shouldBe LocalDateTime.of(2026, 3, 1, 14, 0) - } - - it should "keep each job's synchronisation state separate" in { - val folder = schedulerFolder() - writeSyncFile(folder, "job-1", LocalDateTime.of(2026, 3, 1, 12, 0)) - writeSyncFile(folder, "job-2", LocalDateTime.of(2026, 3, 2, 12, 0)) - - provider.getScheduledTimeRange("job-1", folder.toURI, epoch)._1 shouldBe LocalDateTime.of(2026, 3, 1, 12, 0) - provider.getScheduledTimeRange("job-2", folder.toURI, epoch)._1 shouldBe LocalDateTime.of(2026, 3, 2, 12, 0) - provider.getScheduledTimeRange("job-3", folder.toURI, epoch)._1 shouldBe epoch - } - - it should "create the state directory when it does not exist yet" in { - val missing = Paths.get(schedulerFolder().getAbsolutePath, "scheduler").toFile - missing should not(exist) - - provider.getScheduledTimeRange("job-1", missing.toURI, epoch)._1 shouldBe epoch - missing should exist - } - - it should "end the range at the current time" in { - val before = LocalDateTime.now() - val (_, to) = provider.getScheduledTimeRange("job-1", schedulerFolder().toURI, epoch) - to should be >= before - to should be <= LocalDateTime.now() - } - - /* - * Pinned rather than changed: only FileNotFoundException is caught, so a sync file that exists but has - * no parsable last line fails the run instead of falling back to `startTime`. A crash between opening - * the writer and writing the line leaves exactly such a zero-byte file, and from then on the job stops - * syncing. Changing that is a scheduling-semantics decision (silently re-syncing from `initialTime` - * could re-import a lot of data), so it is recorded here rather than fixed in passing. - */ - it should "fail rather than fall back when the sync file holds no parsable instant" in { - val folder = schedulerFolder() - Files.createFile(Paths.get(folder.getAbsolutePath, "job-1.txt")) - - a[DateTimeParseException] should be thrownBy provider.getScheduledTimeRange("job-1", folder.toURI, epoch) - } -} diff --git a/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtensionSpec.scala b/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtensionSpec.scala deleted file mode 100644 index 395d6e83..00000000 --- a/ignifyr-runtime-scheduling/src/test/scala/io/ignifyr/runtime/scheduling/SchedulingRuntimeExtensionSpec.scala +++ /dev/null @@ -1,26 +0,0 @@ -package io.ignifyr.runtime.scheduling - -import io.ignifyr.engine.spi.ExtensionRegistry -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Lightweight, no-Docker smoke for the scheduling runtime: the [[SchedulingRuntimeExtension]] is - * discovered through ServiceLoader so the engine exposes a scheduling capability, and a fresh - * provider reports no scheduled executions. The full cron/SQL/onFHIR behavioural test (the parked - * `SchedulingTest`) is re-homed here with the shared testkit, once fixtures are classpath-portable. - */ -class SchedulingRuntimeExtensionSpec extends AnyFlatSpec with Matchers { - - behavior of "SchedulingRuntimeExtension" - - it should "register a scheduling execution provider with the engine through ServiceLoader" in { - ExtensionRegistry.scheduler shouldBe defined - } - - it should "report no scheduled executions for an unknown job on a fresh registry" in { - val provider = new Cron4jSchedulerProvider - provider.isScheduled("no-such-job", "no-such-execution") shouldBe false - provider.getScheduledExecutions("no-such-job") shouldBe empty - } -} diff --git a/ignifyr-runtime-streaming/CLAUDE.md b/ignifyr-runtime-streaming/CLAUDE.md deleted file mode 100644 index 8f0bf2b8..00000000 --- a/ignifyr-runtime-streaming/CLAUDE.md +++ /dev/null @@ -1,80 +0,0 @@ -# ignifyr-runtime-streaming — streaming execution capability - -Enterprise module providing Spark structured-streaming execution as an installable capability behind -the engine's `StreamingExecutionProvider` SPI. Package root `io.ignifyr.runtime.streaming`. Bundled -into `ignifyr-server`, **not** `ignifyr-cli`. Its only production dependency is `ignifyr-engine` -(Spark streaming comes transitively), so it carries no enterprise-only libs. - -The community engine can build a streaming source dataset, but **starting and writing** the streaming -queries lives here. Without this module a job with `asStream=true` (or streaming source settings) -parses fine and fails at launch with `MissingCapabilityException` naming -`com.pontegra.ignifyr:ignifyr-runtime-streaming`. This is a carve-out of code that previously lived in -`FhirMappingJobManager.startMappingJobStream` — the "one-folder move between editions" rule in action. - -## Layout (`src/main/scala/io/ignifyr/runtime/streaming/`) -- `StreamingRuntimeExtension` — the `IgnifyrExtension` impl (`id = "runtime-streaming"`); overrides - `streamingProvider` to `Some(new StreamingJobExecutor)`, contributes nothing else. Named in the sole - `META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension` file. -- `StreamingJobExecutor` — implements `StreamingExecutionProvider.startMappingJobStream`: builds the - writer, runs each task through the engine's `MappingTaskPipeline.runMappingTask`, writes via - `StreamingSinkHandler`, logs STARTED/FAILURE, and returns `Map[taskName, Future[StreamingQuery]]`. -- `StreamingSinkHandler` (object) — `writeStream` wraps the engine's batch `SinkHandler.writeMappingResult` - inside a Spark `foreachBatch`, so each micro-batch is written exactly like a batch chunk, with an - explicit per-job/per-mapping-task `checkpointLocation`. - -## Notes -- The engine's `ExtensionRegistry.streaming` is a single-capability `Option` — **at most one** provider - may be installed. A second one now fails at **engine startup**, because `ExtensionRegistry.init()` - force-materializes `streaming`; it previously surfaced only at first job launch (or incidentally, via - `IgnifyrEngine` consulting it for the archive timer). -- Two deliberate error strategies coexist: `StreamingSinkHandler` **catches per-micro-batch exceptions - and only logs** (the stream survives bad chunks), while `StreamingJobExecutor`'s task-level `.recover` - logs FAILURE and **rethrows**. - - That log line is the *only* record a failed micro-batch leaves, so it must carry the `Throwable` - itself — **never `e.getMessage`**. Passing a `String` selects scala-logging's - `error(String, Any*)` overload, whose varargs fill `{}` placeholders; since the message is already - fully interpolated there is no placeholder, and both the argument and the stack trace are silently - discarded. -- Checkpoint dirs are per-job **and** per-mapping-task so distinct streams never mix Spark offsets. - -## Tests -Tier-split, one execution each. - -**Short** (`wildcardSuites=io.ignifyr.runtime.streaming`, no Docker) — `StreamingSinkHandlerTest` -(`AnyFlatSpec`): drives a **`MemoryStream`** through `StreamingSinkHandler.writeStream` to exercise the -catch-and-continue behaviour, pushing one chunk at a time and calling `processAllAvailable()` between -them. It asserts both halves of that contract: the writer's chunk counter reaching 2 proves it was handed -a *further* chunk after one threw, and `isActive`/`exception.isEmpty` prove the exception never escaped -the `foreachBatch`. Keep the first — without it the suite would still pass if the stream produced only -the failing chunk and then nothing at all, which is indistinguishable from swallow-and-continue from the -query's side. -- ⚠️ **Drive the batches; never wait on a clock.** This suite used a `rate` source plus - `awaitTermination(5000)`, and the pass depended on that window: Spark's streaming startup dominates it - and varies by machine (the second chunk landed ~0.5s *after* the window on one developer box — only - `stop()` blocking on the in-flight batch saved the assertion — and needed 16–42s on a slower one). - Shortening the window to 2s reproduces the failure on demand. `processAllAvailable()` removes the - dependency instead of widening it. Note the old `awaitTermination` never rethrew anything either — - `StreamingSinkHandler` swallows per-micro-batch exceptions, so the query never fails and that half of - the contract went unasserted; the second swallowed error it used to log was an `InterruptedException` - from the `stop()` that cut the batch short. -- ⚠️ **The mock job's source settings must carry `asStream = true`.** `FhirMappingJobExecution` derives - `isStreamingJob` from them, and that flag is what makes `SinkHandler.logMappingJobResult` call - `ExecutionLogger.logExecutionResultForStreamingMappingTask` (stateless) instead of - `logExecutionResultForChunk`, whose per-execution cache is only ever seeded by a **batch** `STARTED` - log. Leave it at the default and every chunk throws `NoSuchElementException: key not found: ` - after the write — swallowed by the handler, so the suite still passes while failing for a reason it - never meant to test. - -**Long** (`membersOnlySuites=io.ignifyr.integrationtest`, gated on `${skipITs}`, Docker) — two -end-to-end suites that exercise the real capability seam rather than the mechanics: -- `StreamingFolderWatchTest` — a CSV dropped into a watched directory *after* the query starts must flow - through the testkit's `patient-mapping` and land as FHIR Patients in the onFHIR container. Uses - `archiveMode = off` so it does not depend on the archiver timer (which only `IgnifyrEngine` starts). -- `KafkaStreamingRedcapTest` — a `KafkaContainer` stands in for the external `tofhir-redcap` service: - REDCap-shaped JSON records are published to a topic and consumed through an ordinary `KafkaSource`. - -Both mix in `IgnifyrTestSpec` + `OnFhirTestContainer`, so the module test-depends on `ignifyr-testkit`, -`ignifyr-connector-file`, `ignifyr-connector-kafka` and `ignifyr-sink-fhir` — the connectors and the sink -must be on the test classpath to be ServiceLoader-discovered, which is the same seam the production -edition boundary uses. Run them with `mvn -B verify -pl ignifyr-runtime-streaming -DskipITs=false`, and -clear `checkpoint/` + `logs/` first or the streaming suites trip `CONCURRENT_STREAM_LOG_UPDATE`. diff --git a/ignifyr-runtime-streaming/pom.xml b/ignifyr-runtime-streaming/pom.xml deleted file mode 100644 index a982cb67..00000000 --- a/ignifyr-runtime-streaming/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-runtime-streaming_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - - - test - - test - - - io.ignifyr.runtime.streaming - - - - - integration-test - integration-test - - test - - - ${skipITs} - io.ignifyr.integrationtest - - - - - - - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.mockito - mockito-scala_2.13 - test - - - - - io.ignifyr - ignifyr-testkit_2.13 - test - - - - io.ignifyr - ignifyr-connector-file_2.13 - test - - - - io.ignifyr - ignifyr-connector-kafka_2.13 - test - - - - io.ignifyr - ignifyr-sink-fhir_2.13 - test - - - - org.testcontainers - kafka - test - - - - diff --git a/ignifyr-runtime-streaming/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension b/ignifyr-runtime-streaming/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension deleted file mode 100644 index b7d9864e..00000000 --- a/ignifyr-runtime-streaming/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension +++ /dev/null @@ -1,2 +0,0 @@ -# Ignifyr streaming execution runtime. -io.ignifyr.runtime.streaming.StreamingRuntimeExtension diff --git a/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingJobExecutor.scala b/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingJobExecutor.scala deleted file mode 100644 index 2ceeabfd..00000000 --- a/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingJobExecutor.scala +++ /dev/null @@ -1,71 +0,0 @@ -package io.ignifyr.runtime.streaming - -import com.typesafe.scalalogging.Logger -import io.ignifyr.engine.data.write.SinkWriterFactory -import io.ignifyr.engine.execution.log.ExecutionLogger -import io.ignifyr.engine.model.{ - FhirMappingJobExecution, - FhirMappingJobResult, - SinkSettings, - IdentityServiceSettings, - MappingJobSourceSettings, - TerminologyServiceSettings -} -import io.ignifyr.engine.spi.{MappingTaskPipeline, StreamingExecutionProvider} -import org.apache.spark.sql.streaming.StreamingQuery - -import scala.concurrent.{ExecutionContext, Future} - -/** - * Streaming execution provider: starts a Spark structured-streaming query per mapping task, reading - * each task through the engine's [[MappingTaskPipeline]] and writing results with - * [[StreamingSinkHandler]]. This is the body that previously lived in - * `FhirMappingJobManager.startMappingJobStream`. - */ -class StreamingJobExecutor extends StreamingExecutionProvider { - - private val logger: Logger = Logger(this.getClass) - - override def startMappingJobStream( - pipeline: MappingTaskPipeline, - mappingJobExecution: FhirMappingJobExecution, - sourceSettings: Map[String, MappingJobSourceSettings], - sinkSettings: SinkSettings, - terminologyServiceSettings: Option[TerminologyServiceSettings], - identityServiceSettings: Option[IdentityServiceSettings] - )(implicit ec: ExecutionContext): Map[String, Future[StreamingQuery]] = { - val sinkWriter = SinkWriterFactory.apply(sinkSettings) - sinkWriter.validate() - mappingJobExecution.mappingTasks - .map(t => { - logger.debug( - s"Streaming mapping job ${mappingJobExecution.jobId}, mapping name ${t.name} is started and waiting for the data..." - ) - // log the start of the FHIR mapping task execution - ExecutionLogger - .logExecutionStatus(mappingJobExecution, FhirMappingJobResult.STARTED, Some(t.name), isChunkResult = false) - // Construct a tuple of (mapping name, Future[StreamingQuery]) - t.name -> - pipeline - .runMappingTask( - jobId = mappingJobExecution.jobId, - task = t, - sourceSettings = sourceSettings, - terminologyServiceSettings = terminologyServiceSettings, - identityServiceSettings = identityServiceSettings, - executionId = Some(mappingJobExecution.id), - projectId = Some(mappingJobExecution.projectId) - ) - .map(ts => - StreamingSinkHandler.writeStream(pipeline.sparkSession, mappingJobExecution, ts, sinkWriter, t.name) - ) - .recover { case e: Throwable => - // log the execution status as "FAILURE" - ExecutionLogger - .logExecutionStatus(mappingJobExecution, FhirMappingJobResult.FAILURE, Some(t.name), Some(e)) - throw e - } - }) - .toMap - } -} diff --git a/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingRuntimeExtension.scala b/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingRuntimeExtension.scala deleted file mode 100644 index 7c81a0e6..00000000 --- a/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingRuntimeExtension.scala +++ /dev/null @@ -1,13 +0,0 @@ -package io.ignifyr.runtime.streaming - -import io.ignifyr.engine.spi.{IgnifyrExtension, StreamingExecutionProvider} - -/** - * Registers the streaming execution capability with the engine via ServiceLoader. - */ -class StreamingRuntimeExtension extends IgnifyrExtension { - - override val id: String = "runtime-streaming" - - override def streamingProvider: Option[StreamingExecutionProvider] = Some(new StreamingJobExecutor) -} diff --git a/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingSinkHandler.scala b/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingSinkHandler.scala deleted file mode 100644 index 10d07d17..00000000 --- a/ignifyr-runtime-streaming/src/main/scala/io/ignifyr/runtime/streaming/StreamingSinkHandler.scala +++ /dev/null @@ -1,52 +0,0 @@ -package io.ignifyr.runtime.streaming - -import com.typesafe.scalalogging.Logger -import io.ignifyr.engine.data.write.{BaseSinkWriter, SinkHandler} -import io.ignifyr.engine.model.{FhirMappingJobExecution, FhirMappingResult} -import org.apache.spark.sql.streaming.StreamingQuery -import org.apache.spark.sql.{Dataset, SparkSession} - -/** - * Streaming sink: wraps the engine's batch [[SinkHandler.writeMappingResult]] in a Spark - * structured-streaming `foreachBatch`, so each micro-batch is written exactly as a batch chunk. - * Moved out of the engine as part of the streaming capability. - */ -object StreamingSinkHandler { - - private val logger: Logger = Logger(this.getClass) - - /** - * Writes streaming FHIR mapping results to the given writer via per-micro-batch delegation to the - * engine's batch writer. Uses a per-job, per-mapping-task checkpoint directory so distinct streams - * do not share offsets. - */ - def writeStream( - spark: SparkSession, - mappingJobExecution: FhirMappingJobExecution, - df: Dataset[FhirMappingResult], - resourceWriter: BaseSinkWriter, - mappingTaskName: String - ): StreamingQuery = { - val datasetWrite = (dataset: Dataset[FhirMappingResult], _: Long) => - try { - SinkHandler.writeMappingResult(spark, mappingJobExecution, mappingTaskName, dataset, resourceWriter) - } catch { - case e: Throwable => - // Pass the throwable, NOT e.getMessage: a String selects scala-logging's - // `error(String, Any*)` overload, whose args fill `{}` placeholders — and this message is - // fully interpolated, so the argument was silently discarded along with the stack trace. - // This is the deliberate swallow path (the stream survives bad chunks), so this line is the - // only record a failed micro-batch ever leaves. - logger.error( - s"Streaming chunk resulted in error for project: ${mappingJobExecution.projectId}, job: ${mappingJobExecution.jobId}, execution: ${mappingJobExecution.id}, mappingTask: $mappingTaskName", - e - ) - } - - df.writeStream - // Explicit per-job, per-mapping-task checkpoint dir so distinct streams don't mix up offsets. - .option("checkpointLocation", mappingJobExecution.getCheckpointDirectory(mappingTaskName)) - .foreachBatch(datasetWrite) - .start() - } -} diff --git a/ignifyr-runtime-streaming/src/test/resources/application.conf b/ignifyr-runtime-streaming/src/test/resources/application.conf deleted file mode 100644 index 6e9b123d..00000000 --- a/ignifyr-runtime-streaming/src/test/resources/application.conf +++ /dev/null @@ -1,51 +0,0 @@ -# Ignifyr configurations (runtime-streaming suites; mappings/schemas come from ignifyr-testkit). -# Without this, IgnifyrTestSpec resolves the default "mappings"/"schemas" folders, which are not on -# the test classpath, and the integration suites abort at construction. -ignifyr { - - # Base for relative file-system readings within a mapping job. - context-path = "testcontext" - - mappings = { - # The repository where the mapping definitions are kept (shipped by ignifyr-testkit). - repository = { - folder-path = "/test-mappings" - } - - # Configuration of the schemas used in the mapping definitions (shipped by ignifyr-testkit). - schemas = { - repository = { - folder-path = "/test-schemas" - } - } - - # Timeout for each mapping execution on an individual input record - timeout = 5 seconds - } - - archiving = { - erroneous-records-folder = "erroneous-records-folder" - archive-folder = "archive-folder" - stream-archiving-frequency = 5000 - } - - # Settings for FHIR repository writer - fhir-server-writer { - batch-group-size = 20 - } - - # Database folder of Ignifyr - db = "db" -} - -# Spark configurations -spark = { - app.name = "Ignifyr runtime-streaming TEST" - master = "local[4]" - # Streaming query checkpoints (StreamingFolderWatchTest / KafkaStreamingRedcapTest isolate per jobId). - checkpoint-dir = "checkpoint" -} - -akka = { - daemonic = "on" -} diff --git a/ignifyr-runtime-streaming/src/test/resources/streaming-folder-watch/patients.csv b/ignifyr-runtime-streaming/src/test/resources/streaming-folder-watch/patients.csv deleted file mode 100644 index 0c02f6a3..00000000 --- a/ignifyr-runtime-streaming/src/test/resources/streaming-folder-watch/patients.csv +++ /dev/null @@ -1,11 +0,0 @@ -pid,gender,birthDate,deceasedDateTime,homePostalCode -p1,male,2000-05-10,, -p2,male,1985-05-08,2017-03-10,G02547 -p3,male,1997-02,, -p4,male,1999-06-05,,H10564 -p5,male,1965-10-01,2019-04-21,G02547 -p6,female,1991-03,, -p7,female,1972-10-25,,V13135 -p8,female,2010-01-10,,Z54564 -p9,female,1999-05-12,, -p10,female,2003-11,, diff --git a/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/KafkaStreamingRedcapTest.scala b/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/KafkaStreamingRedcapTest.scala deleted file mode 100644 index 8498fb29..00000000 --- a/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/KafkaStreamingRedcapTest.scala +++ /dev/null @@ -1,167 +0,0 @@ -package io.ignifyr.integrationtest - -import io.onfhir.api.client.FhirBatchTransactionRequestBuilder -import io.onfhir.api.util.FHIRUtil -import io.onfhir.util.JsonFormatter.formats -import io.ignifyr.{IgnifyrTestSpec, OnFhirTestContainer} -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.mapping.context.MappingContextLoader -import io.ignifyr.engine.mapping.job.FhirMappingJobManager -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FhirMappingUtility -import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} -import org.apache.kafka.common.serialization.StringSerializer -import org.apache.spark.sql.streaming.StreamingQuery -import org.scalatest.BeforeAndAfterAll -import org.scalatest.flatspec.AnyFlatSpec -import org.testcontainers.containers.KafkaContainer -import org.testcontainers.utility.DockerImageName - -import java.io.File -import java.util.Properties -import scala.concurrent.duration._ -import scala.concurrent.{Await, Future} - -/** - * End-to-end Kafka streaming test, simulating the REDCap ingestion path WITHOUT the external - * `tofhir-redcap` service. In production, `tofhir-redcap` subscribes to REDCap and publishes each - * record as a JSON message to a Kafka topic; Ignifyr consumes it with an ordinary `KafkaSource` - * (`asRedCap` was removed as dead config — commit 912d15cb). Here a `KafkaContainer` stands in and - * we publish REDCap-shaped patient records directly to the topic, exercising the exact Kafka - * streaming path: `ignifyr-connector-kafka` builds the source dataset, this module's - * `StreamingJobExecutor` runs the query, and records flow through `patient-mapping` into onFHIR. - * - * Integration phase (Docker): MongoDB + srdc/onfhir:r5 (OnFhirTestContainer) + a Kafka container. - * Uses `rp*` patient ids so it never collides with the folder-watch suite's `p*` ids. - */ -class KafkaStreamingRedcapTest - extends AnyFlatSpec - with BeforeAndAfterAll - with IgnifyrTestSpec - with OnFhirTestContainer { - - import io.ignifyr.engine.Execution.actorSystem.dispatcher - - private val jobId = "kafka-streaming-redcap-test" - private val topic = "redcap-patients" - - private val kafka: KafkaContainer = - new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.4.0")).withReuse(true) - - private var query: Option[StreamingQuery] = None - - private val fhirMappingJobManager = new FhirMappingJobManager( - mappingRepository, - new MappingContextLoader, - schemaRepository, - Map.empty, - sparkSession - ) - - // REDCap-shaped records — a JSON object per row keyed by the `Ext-patient` schema fields, exactly - // what tofhir-redcap would push to the topic. - private val redcapRecords: Seq[String] = Seq( - """{"pid":"rp1","gender":"male","birthDate":"1980-01-01","deceasedDateTime":"","homePostalCode":"K01000"}""", - """{"pid":"rp2","gender":"female","birthDate":"1992-07-15","deceasedDateTime":"","homePostalCode":"K02000"}""" - ) - - override protected def beforeAll(): Unit = { - super.beforeAll() - kafka.start() - org.apache.commons.io.FileUtils.deleteQuietly(new File(IgnifyrConfig.sparkCheckpointDirectory, jobId)) - publishRecords() - } - - override protected def afterAll(): Unit = { - query.foreach(q => if (q.isActive) q.stop()) - deleteResources() - org.apache.commons.io.FileUtils.deleteQuietly(new File(IgnifyrConfig.sparkCheckpointDirectory, jobId)) - if (kafka.isRunning) kafka.stop() - super.afterAll() - } - - private def publishRecords(): Unit = { - val props = new Properties() - props.put("bootstrap.servers", kafka.getBootstrapServers) - props.put("key.serializer", classOf[StringSerializer].getName) - props.put("value.serializer", classOf[StringSerializer].getName) - val producer = new KafkaProducer[String, String](props) - try redcapRecords.foreach(r => producer.send(new ProducerRecord[String, String](topic, r)).get()) - finally { - producer.flush() - producer.close() - } - } - - private def deleteResources(): Unit = { - var batchRequest: FhirBatchTransactionRequestBuilder = onFhirClient.batch() - Seq("rp1", "rp2").foreach { id => - batchRequest = batchRequest.entry(_.delete("Patient", FhirMappingUtility.getHashedId("Patient", id))) - } - Await.result( - batchRequest.returnMinimal().asInstanceOf[FhirBatchTransactionRequestBuilder].execute(), - 60.seconds - ) - } - - private def awaitPatient(rawId: String, timeout: FiniteDuration = 90.seconds): Unit = { - val hashedId = FhirMappingUtility.getHashedId("Patient", rawId) - val deadline = timeout.fromNow - var found = false - while (!found && deadline.hasTimeLeft()) { - val attempt = onFhirClient.read("Patient", hashedId).executeAndReturnResource().map { resource => - FHIRUtil.extractIdFromResource(resource) == hashedId - } - found = - try Await.result(attempt, 10.seconds) - catch { case _: Throwable => false } - if (!found) Thread.sleep(2000) - } - assert( - found, - s"Patient '$hashedId' did not appear in onFHIR within $timeout (Kafka streaming did not consume the topic)" - ) - } - - // lazy: `kafka.getBootstrapServers` is only valid after `kafka.start()` in beforeAll, so this must - // not be evaluated at construction time. - private lazy val streamingJob: FhirMappingJob = FhirMappingJob( - id = jobId, - sourceSettings = Map( - "source" -> KafkaSourceSettings( - name = "redcap-kafka-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/redcap-kafka", - bootstrapServers = kafka.getBootstrapServers, - asStream = true - ) - ), - sinkSettings = FhirRepositorySinkSettings(fhirRepoUrl = onFhirClient.getBaseUrl()), - mappings = Seq( - FhirMappingTask( - name = "patient-mapping", - mappingRef = "https://aiccelerate.eu/fhir/mappings/patient-mapping", - sourceBinding = Map("source" -> KafkaSource(topicName = topic, options = Map("startingOffsets" -> "earliest"))) - ) - ), - dataProcessingSettings = DataProcessingSettings(archiveMode = ArchiveModes.OFF, saveErroneousRecords = false) - ) - - it should "consume REDCap-shaped records from Kafka and write FHIR Patients (Kafka streaming)" in { - val streams: Map[String, Future[StreamingQuery]] = fhirMappingJobManager.startMappingJobStream( - mappingJobExecution = FhirMappingJobExecution(mappingTasks = streamingJob.mappings, job = streamingJob), - sourceSettings = streamingJob.sourceSettings, - sinkSettings = streamingJob.sinkSettings - ) - query = streams.values.headOption.map(f => Await.result(f, 60.seconds)) - query shouldBe defined - - awaitPatient("rp1") - awaitPatient("rp2") - - val rp2 = Await.result( - onFhirClient.read("Patient", FhirMappingUtility.getHashedId("Patient", "rp2")).executeAndReturnResource(), - 15.seconds - ) - FHIRUtil.extractValue[String](rp2, "gender") shouldBe "female" - } -} diff --git a/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/StreamingFolderWatchTest.scala b/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/StreamingFolderWatchTest.scala deleted file mode 100644 index fb38913d..00000000 --- a/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/integrationtest/StreamingFolderWatchTest.scala +++ /dev/null @@ -1,159 +0,0 @@ -package io.ignifyr.integrationtest - -import io.onfhir.api.client.FhirBatchTransactionRequestBuilder -import io.onfhir.api.util.FHIRUtil -import io.onfhir.util.JsonFormatter.formats -import io.ignifyr.{IgnifyrTestSpec, OnFhirTestContainer} -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.mapping.context.MappingContextLoader -import io.ignifyr.engine.mapping.job.FhirMappingJobManager -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FhirMappingUtility -import org.apache.spark.sql.streaming.StreamingQuery -import org.scalatest.BeforeAndAfterAll -import org.scalatest.flatspec.AnyFlatSpec - -import java.io.File -import java.nio.file.{Files, Path, Paths, StandardCopyOption} -import scala.concurrent.duration._ -import scala.concurrent.{Await, Future} - -/** - * End-to-end folder-watch streaming test — fills the gap left by the mechanics-only - * `StreamingSinkHandlerTest` (rate source) and the fabricated `FileStreamInputArchiverTest`. - * - * It exercises the real capability seam: the community engine builds the file-stream source dataset - * (via `ignifyr-connector-file`, on the test classpath) and this module's `StreamingJobExecutor` - * (discovered through the `IgnifyrExtension` ServiceLoader SPI) starts and writes the Spark - * structured-streaming query. A CSV dropped into the watched directory after the query starts must - * flow through the `patient-mapping` (testkit fixture, schema `Ext-patient`) and land as FHIR - * Patient resources in the onFHIR/repofyr container. - * - * Runs in the integration phase (needs Docker: MongoDB + srdc/onfhir:r5 via `OnFhirTestContainer`). - * Archiving of consumed inputs is covered separately by `FileStreamInputArchiverTest`; here - * `archiveMode = off` so the streaming archiver timer (started only by `IgnifyrEngine`) is not - * required. - */ -class StreamingFolderWatchTest - extends AnyFlatSpec - with BeforeAndAfterAll - with IgnifyrTestSpec - with OnFhirTestContainer { - - import io.ignifyr.engine.Execution.actorSystem.dispatcher - - // A unique job id keeps this job's Spark checkpoint dir (`//`) - // isolated from any other suite so streaming offsets never collide. - private val jobId = "streaming-folder-watch-test" - - // The directory Spark's file-stream source polls: /. - private val watchRoot: Path = Files.createTempDirectory("ignifyr-folder-watch-") - private val watchedDir: Path = watchRoot.resolve("patients") - - private var queries: Seq[StreamingQuery] = Seq.empty - - private val fhirMappingJobManager = new FhirMappingJobManager( - mappingRepository, - new MappingContextLoader, - schemaRepository, - Map.empty, - sparkSession - ) - - private val patientMappingTask: FhirMappingTask = FhirMappingTask( - name = "patient-mapping", - mappingRef = "https://aiccelerate.eu/fhir/mappings/patient-mapping", - sourceBinding = Map("source" -> FileSystemSource(path = "patients", contentType = SourceContentTypes.CSV)) - ) - - private val streamingJob: FhirMappingJob = FhirMappingJob( - id = jobId, - sourceSettings = Map( - "source" -> FileSystemSourceSettings( - name = "folder-watch-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/streaming-test-data", - dataFolderPath = watchRoot.toAbsolutePath.toString, - asStream = true - ) - ), - sinkSettings = FhirRepositorySinkSettings(fhirRepoUrl = onFhirClient.getBaseUrl()), - mappings = Seq(patientMappingTask), - dataProcessingSettings = DataProcessingSettings(archiveMode = ArchiveModes.OFF, saveErroneousRecords = false) - ) - - override protected def beforeAll(): Unit = { - super.beforeAll() - Files.createDirectories(watchedDir) - // Fresh checkpoints so the stream reprocesses from scratch on each run. - org.apache.commons.io.FileUtils.deleteQuietly(new File(IgnifyrConfig.sparkCheckpointDirectory, jobId)) - } - - override protected def afterAll(): Unit = { - queries.foreach(q => if (q.isActive) q.stop()) - deleteResources() - org.apache.commons.io.FileUtils.deleteQuietly(watchRoot.toFile) - org.apache.commons.io.FileUtils.deleteQuietly(new File(IgnifyrConfig.sparkCheckpointDirectory, jobId)) - super.afterAll() - } - - private def deleteResources(): Unit = { - var batchRequest: FhirBatchTransactionRequestBuilder = onFhirClient.batch() - (1 to 10).foreach { i => - batchRequest = batchRequest.entry(_.delete("Patient", FhirMappingUtility.getHashedId("Patient", "p" + i))) - } - Await.result( - batchRequest.returnMinimal().asInstanceOf[FhirBatchTransactionRequestBuilder].execute(), - 60.seconds - ) - } - - /** Copy the CSV fixture into the watched directory so Spark's file-stream source picks it up. */ - private def dropPatientsCsv(): Unit = { - val in = getClass.getResourceAsStream("/streaming-folder-watch/patients.csv") - try Files.copy(in, watchedDir.resolve("patients.csv"), StandardCopyOption.REPLACE_EXISTING) - finally in.close() - } - - /** Poll onFHIR until the given hashed Patient id is readable, or fail after the timeout. */ - private def awaitPatient(rawId: String, timeout: FiniteDuration = 90.seconds): Unit = { - val hashedId = FhirMappingUtility.getHashedId("Patient", rawId) - val deadline = timeout.fromNow - var found = false - while (!found && deadline.hasTimeLeft()) { - val attempt = onFhirClient.read("Patient", hashedId).executeAndReturnResource().map { resource => - FHIRUtil.extractIdFromResource(resource) == hashedId - } - found = - try Await.result(attempt, 10.seconds) - catch { case _: Throwable => false } - if (!found) Thread.sleep(2000) - } - assert( - found, - s"Patient '$hashedId' did not appear in onFHIR within $timeout (streaming folder-watch did not process the dropped file)" - ) - } - - it should "process a CSV dropped into the watched folder and write FHIR Patients (folder-watch streaming)" in { - val streams: Map[String, Future[StreamingQuery]] = fhirMappingJobManager.startMappingJobStream( - mappingJobExecution = FhirMappingJobExecution(mappingTasks = streamingJob.mappings, job = streamingJob), - sourceSettings = streamingJob.sourceSettings, - sinkSettings = streamingJob.sinkSettings - ) - queries = streams.values.map(f => Await.result(f, 60.seconds)).toSeq - queries should not be empty - - // Drop the input AFTER the query is live so the file-stream source detects it as a new file. - dropPatientsCsv() - - // Assert a representative slice of the batch landed (p1 male / p8 female per the fixture). - awaitPatient("p1") - awaitPatient("p8") - - val p8 = Await.result( - onFhirClient.read("Patient", FhirMappingUtility.getHashedId("Patient", "p8")).executeAndReturnResource(), - 15.seconds - ) - FHIRUtil.extractValue[String](p8, "gender") shouldBe "female" - } -} diff --git a/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/runtime/streaming/StreamingSinkHandlerTest.scala b/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/runtime/streaming/StreamingSinkHandlerTest.scala deleted file mode 100644 index 07946ac8..00000000 --- a/ignifyr-runtime-streaming/src/test/scala/io/ignifyr/runtime/streaming/StreamingSinkHandlerTest.scala +++ /dev/null @@ -1,119 +0,0 @@ -package io.ignifyr.runtime.streaming - -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.data.write.BaseSinkWriter -import io.ignifyr.engine.model.{ - DataProcessingSettings, - FhirMappingJob, - FhirMappingJobExecution, - FhirMappingResult, - FileSystemSourceSettings, - SinkSettings -} -import org.apache.commons.io.FileUtils -import org.apache.spark.sql.execution.streaming.MemoryStream -import org.apache.spark.sql.{Dataset, SQLContext, SparkSession} -import org.apache.spark.util.CollectionAccumulator -import org.mockito.MockitoSugar._ -import org.scalatest.BeforeAndAfterAll -import org.scalatest.flatspec.AnyFlatSpec - -import java.io.File -import java.sql.Timestamp -import java.time.Instant -import java.util.concurrent.atomic.AtomicInteger - -/** - * Sink writer that counts the chunks handed to it and throws on the first one. - * - * "Only the first chunk fails" is a property of the writer here rather than of a stateful mockito - * argument matcher flipping a captured `var` — mockito re-evaluates matchers at times of its own - * choosing, so making the *stub* carry the ordering is one less thing to reason about. Declared top - * level, not as an inner class of the suite, so it carries no reference to the spec instance when Spark - * closes over it. - */ -private class CountingSinkWriter extends BaseSinkWriter(new SinkSettings {}) { - - val chunks = new AtomicInteger(0) - - override def write( - sparkSession: SparkSession, - df: Dataset[FhirMappingResult], - problemsAccumulator: CollectionAccumulator[FhirMappingResult] - ): Unit = - if (chunks.incrementAndGet() == 1) throw new RuntimeException("the first chunk fails on purpose") - - override def validate(): Unit = () -} - -class StreamingSinkHandlerTest extends AnyFlatSpec with BeforeAndAfterAll { - - val sparkSession: SparkSession = IgnifyrConfig.sparkSession - - /** - * The streaming query below writes Spark offset/commit logs under a fixed checkpoint path - * (jobId / mapping-task hash). Leftover logs from a previous run make the next run fail with - * CONCURRENT_STREAM_LOG_UPDATE, so wipe the checkpoint directory around the suite — beforeAll - * guards against a prior interrupted run, afterAll keeps the working tree clean. - */ - private def clearCheckpointDirectory(): Unit = - FileUtils.deleteQuietly(new File(IgnifyrConfig.sparkCheckpointDirectory)) - - override def beforeAll(): Unit = clearCheckpointDirectory() - - override def afterAll(): Unit = clearCheckpointDirectory() - - "StreamingSinkHandler" should "continue processing subsequent chunks for streaming queries after a chunk throws an exception" in { - val mockJob: FhirMappingJob = mock[FhirMappingJob] - when(mockJob.id).thenReturn("jobId") - when(mockJob.dataProcessingSettings).thenReturn(DataProcessingSettings.apply()) - // `asStream = true` matters: FhirMappingJobExecution derives isStreamingJob from it, and that flag is - // what makes SinkHandler.logMappingJobResult take the streaming path. With the default (false) the - // handler under test runs against an execution that claims to be a batch one, so after each write it - // reaches ExecutionLogger.logExecutionResultForChunk, whose per-execution cache only a batch STARTED - // log ever seeds — it throws NoSuchElementException, which this handler then swallows as a second - // "chunk resulted in error". The assertions still held, but every chunk failed for a reason the suite - // was not testing. - when(mockJob.sourceSettings).thenReturn( - Map("0" -> FileSystemSourceSettings.apply("name", "sourceUri", "dataFolderPath", asStream = true)) - ) - - val execution: FhirMappingJobExecution = FhirMappingJobExecution("executionId", "projectId", mockJob) - - // A MemoryStream, not a `rate` source: it lets the test decide when a micro-batch exists, so the - // assertions below are driven by what has actually been processed rather than by a wall-clock - // window. A fixed window here was fragile in both directions -- Spark's streaming startup is what - // dominates it, and it varies by machine (a 5s window left ~0.5s of margin on one developer box and - // was far too short on a slower one). - import sparkSession.implicits._ - implicit val sqlContext: SQLContext = sparkSession.sqlContext - val source = MemoryStream[FhirMappingResult] - - val sinkWriter = new CountingSinkWriter - val streamingQuery = - StreamingSinkHandler.writeStream(sparkSession, execution, source.toDS(), sinkWriter, "someMappingTaskName") - - try { - // First chunk: the writer throws. `processAllAvailable` returns only once the micro-batch is done, - // so no waiting is guessed at. - source.addData(mappingResult("first")) - streamingQuery.processAllAvailable() - assert(sinkWriter.chunks.get() == 1, "the first chunk should have reached the sink writer") - - // Second chunk: this is the actual contract -- the throwing chunk was swallowed inside - // `foreachBatch`, so the query keeps consuming and the writer is handed the next one. Without this - // the suite would still pass if the stream produced only the failing chunk and then nothing at all, - // which is indistinguishable from swallow-and-continue from the query's side. - source.addData(mappingResult("second")) - streamingQuery.processAllAvailable() - assert(sinkWriter.chunks.get() == 2, "the sink writer should have been handed a chunk after one threw") - - // The other half: the exception never escaped the micro-batch, so the query itself is unharmed. - assert(streamingQuery.isActive, "the streaming query should have survived the throwing chunk") - assert(streamingQuery.exception.isEmpty, "the streaming query should not have recorded a failure") - } finally streamingQuery.stop() - } - - private def mappingResult(source: String): FhirMappingResult = - FhirMappingResult("jobId", "someMappingTaskName", Timestamp.from(Instant.now()), source) -} diff --git a/ignifyr-rxnorm/CLAUDE.md b/ignifyr-rxnorm/CLAUDE.md deleted file mode 100644 index 98b84d5e..00000000 --- a/ignifyr-rxnorm/CLAUDE.md +++ /dev/null @@ -1,63 +0,0 @@ -# ignifyr-rxnorm — RxNorm terminology client - -Standalone module: a client for the public RxNorm REST API plus a FHIRPath function library so -mappings can call RxNorm lookups. Package root `io.ignifyr.rxnorm`. It is **not** an `IgnifyrExtension` -SPI plugin and is **not** ServiceLoader-discovered — it plugs in through onFHIR's FHIRPath -function-library config (note the `ignifyr` parent block; the key is -`ignifyr.functionLibraries.rxn.className`): - -```hocon -ignifyr { - functionLibraries { - rxn { - className = "io.ignifyr.rxnorm.RxNormApiFunctionLibraryFactory" - args = ["https://rxnav.nlm.nih.gov", 2] - } - } -} -``` - -That block is active in `ignifyr-server`'s `application.conf` and a commented example in -`ignifyr-engine`'s. Bundled into `ignifyr-server` only, **not** `ignifyr-cli`. - -**Why a module:** it is an artifact boundary, not a code dependency. **Nothing in the repo compiles -against it** — no file outside the module imports `io.ignifyr.rxnorm`, and the module itself imports no -Ignifyr code at all (only onFHIR). Attaching or removing the `rxn:` functions is a config string plus a -jar on the classpath. Keeping it separate also keeps a blocking live-network HTTP client (global -singleton `ActorSystem`, `Await.result` per call) and `opencsv` out of the engine. - -Two edition oddities worth knowing: its declared `ignifyr-common` dependency is **unused by main code** -(its only effect is dragging in `onfhir-definition-commons`, which a *test* needs), and although it -inherits the community `io.ignifyr` groupId it does **not** opt into the `ban-enterprise-deps` enforcer -gate the community modules all declare, while only the enterprise server bundles it. Its edition -placement is genuinely unsettled rather than principled. - -## Contents (3 files in `src/main/scala/io/ignifyr/rxnorm/`) -- `RxNormApiClient` — HTTP client for RxNorm endpoints. -- `RxNormApiFunctionLibrary` — the `rxn:`-prefixed FHIRPath functions backed by the client - (`findRxConceptIdsByNdc`, `getMedicationDetails`, `findIngredientsOfDrug`, `getATC`; keep them - `@FhirPathFunction`-annotated). `RxNormApiFunctionLibraryFactory` — the onFHIR - `IFhirPathFunctionLibraryFactory` that is the actual registration entry point (constructed with - `rxNormApiRootUrl`, `timeoutInSec`) — is declared at the **bottom of this same file**, not in one of - its own. -- `PullRxNormNdcMedDetails` — utility/runner for bulk NDC → medication-detail pulls. - -It pins `com.opencsv:opencsv` to 5.5.1 inline, overriding the root-managed `opencsv.version`. - -## Tests -**Short tier** (`wildcardSuites=io.ignifyr.rxnorm`, no Docker, **no network**): -`RxNormApiClientTest`, `RxNormApiFunctionLibraryTest`. `mvn test -pl ignifyr-rxnorm`. - -Both run against `RxNormApiStub`, a local akka-http server bound on an ephemeral port that answers a -suite-declared map of canned bodies and **404s anything unlisted** — so an unexpected call fails loudly -instead of passing silently. The whole seam is the client's constructor: it takes its root url, so -pointing it at the stub needs no HTTP interception and no extra dependency (the stub uses the same -akka-http the client itself calls through). This is also the only way the not-found and non-200 branches -are reachable at all. - -⚠️ Two things changed here on 2026-08-10 (`899faba7`) — older notes claiming otherwise are stale. The -suites used to sit in the **default package** and the module declared **no `scalatest-maven-plugin`**, so -they compiled, looked like coverage, and ran nowhere under Maven; they also called the live -`rxnav.nlm.nih.gov`, making a green build depend on a third party's uptime. `check-test-tiers.sh` now -carries an invariant (2b) that fails the build for any module with test sources and no plugin, so the -gap cannot reopen. diff --git a/ignifyr-rxnorm/pom.xml b/ignifyr-rxnorm/pom.xml deleted file mode 100644 index a389e3de..00000000 --- a/ignifyr-rxnorm/pom.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-rxnorm_2.13 - jar - - - src/main/scala - src/test/scala - - - - net.alchim31.maven - scala-maven-plugin - - ${scala.binary.version} - UTF-8 - - -deprecation - -feature - - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - org.scalatest - scalatest-maven-plugin - - - test - - test - - - io.ignifyr.rxnorm - - - - - - - - - - - io.onfhir - onfhir-common_${scala.binary.version} - - - io.onfhir - onfhir-path_${scala.binary.version} - - - - com.typesafe.akka - akka-http_${scala.binary.version} - - - com.typesafe.akka - akka-stream_${scala.binary.version} - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - - com.opencsv - opencsv - 5.5.1 - - - - io.ignifyr - ignifyr-common - - - diff --git a/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/PullRxNormNdcMedDetails.scala b/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/PullRxNormNdcMedDetails.scala deleted file mode 100644 index b9d8cfc9..00000000 --- a/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/PullRxNormNdcMedDetails.scala +++ /dev/null @@ -1,196 +0,0 @@ -package io.ignifyr.rxnorm - -import java.io.{FileReader, FileWriter} -import com.opencsv.{CSVReaderBuilder, CSVWriter} -import io.onfhir.api.util.FHIRUtil -import org.json4s.JsonAST.{JArray, JDouble, JObject, JString} -import org.json4s._ -import io.onfhir.util.JsonFormatter._ - -/** - * Handles the extraction of NDC (National Drug Codes) from a given CSV file (prescriptions.csv) and retrieves - * corresponding drug information from the RxNorm API. The extracted data is then compiled into a new CSV file - * named ndcToMedDetails.csv. This approach prevents HTTP 429 errors (Too Many Requests) by caching necessary - * drug information locally, facilitating efficient data mapping without direct RxNorm API calls for each request. - * The ndcToMedDetails.csv serves as a resource for the prescriptions-mapping-without-rxn mapping within - * the Mimic dataset. - * - * Inputs: - * - rxNormApiClient: Configured client for RxNorm API requests, including endpoint and timeout settings. - * - csvFilePath: File path to the prescription data CSV, containing NDC codes for querying the RxNorm API. - * - * Output: - * - Creates ndcToMedDetails.csv: A CSV file containing drug details mapped from NDC codes, with columns for: - * NDC, dose form RxNorm code (doseFormRxcui), dose form name (doseFormName), active ingredient RxNorm code - * (activeIngredientRxcui), active ingredient name (activeIngredientName), numerator unit, numerator value, - * denominator unit, and denominator value. These columns are used by the prescriptions-mapping-without-rxn - * mapping in the Mimic dataset. - */ - -object PullRxNormNdcMedDetails extends App { - // RxNorm Api client for calls - val rxNormApiClient = RxNormApiClient("https://rxnav.nlm.nih.gov", 10) - - // Read ncd codes from prescription table - val csvFilePath = "mimic-iv-data/hosp/prescriptions.csv" - - // Open the CSV file reader - val fileReader = new FileReader(csvFilePath) - val csvReader = new CSVReaderBuilder(fileReader).withSkipLines(1).build() // Skip header line - - // For unique ndc codes - var ndcSet = scala.collection.mutable.Set[String]() - - try { - // Iterate over each row in the CSV file - var record = csvReader.readNext() - - while (record != null) { - // Extract the "ndc" field - val ndcField = record(11) - - // Add it into the set - ndcSet += ndcField - - // Read the next record - record = csvReader.readNext() - } - // Convert it into immutable set - val ndcSetImmutable = ndcSet.toSet - println(s"number of ndc codes: ${ndcSetImmutable.size}") - - // Create FileWriter object - val fileWriter = new FileWriter("ndcToMedDetails.csv") - - // Create CSVWriter object - val csvWriter = new CSVWriter(fileWriter) - - try { - // Set headers for the csv file - val header = Array( - "ndc", - "doseFormRxcui", - "doseFormName", - "activeIngredientRxcui", - "activeIngredientName", - "numeratorUnit", - "numeratorValue", - "denominatorUnit", - "denominatorValue" - ) - csvWriter.writeNext(header) - - ndcSetImmutable.foreach(ndc => { - if (ndc.nonEmpty) { - // Get conceptIds from RxNorm Api - try { - val conceptIds = rxNormApiClient.findRxConceptIdByNdc(ndc); - - conceptIds.iterator - .filter(_.nonEmpty) - .flatMap(rxcui => rxNormApiClient.getRxcuiHistoryStatus(rxcui)) - .map(response => { - // Get the ingredient details - val ingredients = - FHIRUtil - .extractValueOptionByPath[Seq[JObject]]( - response, - "rxcuiStatusHistory.definitionalFeatures.ingredientAndStrength" - ) - .getOrElse(Nil) - val ingredientObjs = - ingredients - .map(i => { - val requiredFields = - Seq( - "activeIngredientRxcui", - "activeIngredientName", - "numeratorValue", - "numeratorUnit", - "denominatorValue", - "denominatorUnit" - ) - .map(f => - FHIRUtil - .extractValueOption[String](i, f) - .filter(_ != "") // Should filter empty string values - .map(v => - if (f == "numeratorValue" || f == "denominatorValue") - f -> JDouble(v.toDouble) - else - f -> JString(v) - ) - ) - if (requiredFields.forall(_.isDefined)) - Some(JObject(requiredFields.map(_.get).toList)) - else - None - }) - - val doseFormObj = - FHIRUtil - .extractValueOptionByPath[Seq[JObject]]( - response, - "rxcuiStatusHistory.definitionalFeatures.doseFormConcept" - ) - .getOrElse(Nil) - .headOption - if (ingredientObjs.nonEmpty && ingredientObjs.forall(_.isDefined)) { - Some( - JObject( - List( - "ingredientAndStrength" -> JArray(ingredientObjs.map(_.get).toList) - ) ++ - doseFormObj - .map(d => "doseFormConcept" -> d) - .toSeq - ) - ) - } else - None - }) - .find(_.isDefined) - .foreach(r => { - - val doseFormRxcui = (r.get \ "doseFormConcept" \ "doseFormRxcui").extract[String] - val doseFormName = (r.get \ "doseFormConcept" \ "doseFormName").extract[String] - val activeIngredientRxcui = - ((r.get \ "ingredientAndStrength")(0) \ "activeIngredientRxcui").extract[String] - val activeIngredientName = - ((r.get \ "ingredientAndStrength")(0) \ "activeIngredientName").extract[String] - val numeratorUnit = ((r.get \ "ingredientAndStrength")(0) \ "numeratorUnit").extract[String] - val numeratorValue = ((r.get \ "ingredientAndStrength")(0) \ "numeratorValue").extract[String] - val denominatorUnit = ((r.get \ "ingredientAndStrength")(0) \ "denominatorUnit").extract[String] - val denominatorValue = ((r.get \ "ingredientAndStrength")(0) \ "denominatorValue").extract[String] - csvWriter.writeNext( - Array( - ndc, - doseFormRxcui, - doseFormName, - activeIngredientRxcui, - activeIngredientName, - numeratorUnit, - numeratorValue, - denominatorUnit, - denominatorValue - ) - ) - - }) - } catch { - case e: Throwable => println(e) - } - } - }) - } finally { - // Close the CSV write - fileWriter.close() - csvWriter.close() - } - - } finally { - // Close the CSV reader - fileReader.close() - csvReader.close() - } -} diff --git a/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiClient.scala b/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiClient.scala deleted file mode 100644 index 013518b8..00000000 --- a/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiClient.scala +++ /dev/null @@ -1,205 +0,0 @@ -package io.ignifyr.rxnorm - -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.model._ -import akka.http.scaladsl.unmarshalling.{Unmarshal, Unmarshaller} -import io.onfhir.api.Resource -import io.onfhir.api.util.FHIRUtil -import io.onfhir.path.FhirPathException -import io.onfhir.util.JsonFormatter._ -import org.json4s.JsonAST.{JDouble, JObject, JString} - -import java.util.concurrent.TimeUnit -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext} - -case class RxNormApiClient(rxNormApiRootUrl: String, timeoutInSec: Int) { - - /** - * Find out RxNorm Concept Id (rxcui) for given NDC code (active or historic) - * See https://lhncbc.nlm.nih.gov/RxNav/APIs/api-RxNorm.getNDCProperties.html - * @param ndc NDC (National drug classification) code - * @return Corresponding RxNorm concept id - */ - def findRxConceptIdByNdc(ndc: String): Seq[String] = { - val uri = Uri.apply(s"$rxNormApiRootUrl/REST/ndcproperties.json") - val request = - HttpRequest - .apply( - HttpMethods.GET, - uri = uri.withQuery(Uri.Query.apply(Map("id" -> ndc, "ndcstatus" -> "ALL"))) - ) - - executeRequest(request) - .flatMap(response => - FHIRUtil - .extractValueOptionByPath[Seq[String]](response, "ndcPropertyList.ndcProperty.rxcui") - .map(_.filter(r => r != "" && r != "/")) - ) - .getOrElse(Nil) - } - - /** - * Find out RxNorm Concept Id (rxcui) for given drug name - * See https://lhncbc.nlm.nih.gov/RxNav/APIs/api-RxNorm.findRxcuiByString.html - * @param drug - * @return - */ - def findRxConceptIdByName(drug: String): Option[String] = { - val request = - HttpRequest - .apply( - HttpMethods.GET, - uri = Uri.apply(s"$rxNormApiRootUrl/REST/rxcui.json?name=${drug}&search=2") - ) - - executeRequest(request) - .flatMap(response => - FHIRUtil - .extractValueOptionByPath[Seq[String]](response, "idGroup.rxnormId") - .flatMap(_.headOption) - ) - } - - /** - * RxNorm API call for getRxcuiHistoryStatus - * If concept is unknown, return None - * See https://lhncbc.nlm.nih.gov/RxNav/APIs/api-RxNorm.getRxcuiHistoryStatus.html - * @param rxcui RxNorm Concept Identifier - * @return - */ - def getRxcuiHistoryStatus(rxcui: String): Option[Resource] = { - val request = - HttpRequest - .apply( - HttpMethods.GET, - uri = Uri.apply(s"$rxNormApiRootUrl/REST/rxcui/$rxcui/historystatus.json") - ) - - executeRequest(request) - .filter(response => { - // Check if this is concept is known (either active or obsolete) - val status = FHIRUtil - .extractValueOptionByPath[String](response, "rxcuiStatusHistory.metaData.status") - status.exists(_ != "UNKNOWN") - }) - } - - /** - * Find out active ingredients and their basic properties for given RxNorm Clinical Drug Pack or component - * See https://lhncbc.nlm.nih.gov/RxNav/APIs/api-RxNorm.getAllProperties.html - * @param rxcui - * @return - */ - def getIngredientProperties(rxcui: String): Seq[Resource] = { - val request = - HttpRequest - .apply( - HttpMethods.GET, - uri = Uri.apply(s"$rxNormApiRootUrl/REST/rxcui/${rxcui}/allProperties.json?prop=attributes") - ) - - executeRequest(request) - .map(response => - FHIRUtil - .extractValueOptionByPath[Seq[JObject]](response, "propConceptGroup.propConcept") - .map(attrs => { - val parsedProps = - attrs - .map(attr => - FHIRUtil.extractValue[String](attr, "propName") -> - FHIRUtil.extractValue[String](attr, "propValue") - ) - - val ingredientCodes = parsedProps.filter(_._1 == "Active_ingredient_RxCUI").map(_._2) - val ingredientNames = parsedProps.filter(_._1 == "Active_ingredient_name").map(_._2) - val numeratorValues = parsedProps.filter(_._1 == "Numerator_Value").map(_._2) - val numeratorUnits = parsedProps.filter(_._1 == "Numerator_Units").map(_._2) - val denominatorValues = parsedProps.filter(_._1 == "Denominator_Value").map(_._2) - val denominatorUnits = parsedProps.filter(_._1 == "Denominator_Units").map(_._2) - - ingredientCodes.indices.map(i => - JObject( - List( - "Active_ingredient_RxCUI" -> JString(ingredientCodes.apply(i)), - "Active_ingredient_name" -> JString(ingredientNames.apply(i)), - "Numerator_Value" -> JDouble(numeratorValues.apply(i).toDouble), - "Numerator_Units" -> JString(numeratorUnits.apply(i)), - "Denominator_Value" -> JDouble(denominatorValues.apply(i).toDouble), - "Denominator_Units" -> JString(denominatorUnits.apply(i)) - ) - ) - ) - }) - .getOrElse(Nil) - ) - .getOrElse(Nil) - } - - /** - * Get corresponding ATC codes for a RxNorm ingredient concept - * See https://lhncbc.nlm.nih.gov/RxNav/APIs/api-RxNorm.getRxProperty.html - * @param rxcui Concept id - * @return - */ - def getAtcCode(rxcui: String): Seq[String] = { - val uri: String = s"$rxNormApiRootUrl/REST/rxcui/$rxcui/property.json?propName=ATC" - val request = - HttpRequest - .apply( - HttpMethods.GET, - uri = Uri.apply(uri) - ) - executeRequest(request) - .flatMap(response => - FHIRUtil - .extractValueOptionByPath[Seq[String]](response, "propConceptGroup.propConcept.propValue") - .map(_.filter(r => r != "" && r != "/")) - ) - .getOrElse(Nil) - } - - /** - * Call the RxNorm Rest Service with JSON format and return the response JSON content, if success - * - * @param request - * @return - */ - private def executeRequest(request: HttpRequest): Option[Resource] = { - implicit val actorSystem: ActorSystem = RxNormApiClient.actorSystem - implicit val ec: ExecutionContext = actorSystem.dispatcher - import RxNormApiClient.rxnormApiJsonUnmarshaller - val call = - Http() - .singleRequest(request) - .flatMap { - case HttpResponse(StatusCodes.OK, _, entity, _) => Unmarshal.apply(entity).to[Option[Resource]] - case HttpResponse(oth, _, entity, _) => - entity.discardBytes() - throw new FhirPathException( - s"Problem while accessing RxNorm API given with root URL $rxNormApiRootUrl! Rest call to ${request.uri.toString()} returns $oth!" - ) - } - Await.result(call, Duration.apply(timeoutInSec, TimeUnit.SECONDS)) - } -} - -object RxNormApiClient { - implicit val actorSystem: ActorSystem = ActorSystem("RxNormApiClient") - - /** - * Unmarshaller for - */ - implicit val rxnormApiJsonUnmarshaller: Unmarshaller[HttpEntity, Option[Resource]] = - Unmarshaller.stringUnmarshaller - .forContentTypes(ContentTypes.`application/json`) - .mapWithInput { case (entity: HttpEntity, data: String) => - if (entity.isKnownEmpty() || data.isEmpty) { - None - } else { - val resource = data.parseJson - if (resource.obj.isEmpty) None else Some(resource) - } - } -} diff --git a/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibrary.scala b/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibrary.scala deleted file mode 100644 index 180f744f..00000000 --- a/ignifyr-rxnorm/src/main/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibrary.scala +++ /dev/null @@ -1,307 +0,0 @@ -package io.ignifyr.rxnorm - -import io.onfhir.api.FHIR_DATA_TYPES -import io.onfhir.api.util.FHIRUtil -import io.onfhir.path.annotation.{ - FhirPathFunction, - FhirPathFunctionDocumentation, - FhirPathFunctionParameter, - FhirPathFunctionReturn -} -import io.onfhir.path.grammar.FhirPathExprParser.ExpressionContext -import io.onfhir.path.{ - AbstractFhirPathFunctionLibrary, - FhirPathComplex, - FhirPathEnvironment, - FhirPathException, - FhirPathExpressionEvaluator, - FhirPathNumber, - FhirPathResult, - FhirPathString, - IFhirPathFunctionLibraryFactory -} -import org.json4s.JsonAST.{JArray, JDouble, JObject, JString} - -/** - * FHIR Path Function library to access RxNorm API functionalities using them for mappings for medication concepts - * @param rxNormApiRootUrl Root URL for RxNorm API - * @param timeoutInSec Timeout for API calls - * @param context FHIR Path context - * @param current Current FHIR Path input context - * @param actorSystem Akka Actor System - */ -class RxNormApiFunctionLibrary( - rxNormApiClient: RxNormApiClient, - context: FhirPathEnvironment, - current: Seq[FhirPathResult] -) extends AbstractFhirPathFunctionLibrary - with Serializable { - val evaluator = new FhirPathExpressionEvaluator(context, current) - - /** - * - * @param ndcExpr - * @return - */ - @FhirPathFunction( - documentation = FhirPathFunctionDocumentation( - detail = "Finds out RxNorm Concept Ids (rxcui) for the given NDC code.", - usageWarnings = None, - parameters = Some( - Seq( - FhirPathFunctionParameter( - name = "ndcExpr", - detail = "The NDC code expression.", - examples = None - ) - ) - ), - returnValue = FhirPathFunctionReturn( - detail = None, - examples = Seq( - """["104922", "105784"]""" - ) - ), - examples = Seq( - "rxn:findRxConceptIdsByNdc('12345678901')" - ) - ), - insertText = "rxn:findRxConceptIdsByNdc()", - detail = "rxn", - label = "rxn:findRxConceptIdByNdc", - kind = "Function", - returnType = Seq(FHIR_DATA_TYPES.STRING), - inputType = Seq(FHIR_DATA_TYPES.STRING) - ) - def findRxConceptIdsByNdc(ndcExpr: ExpressionContext): Seq[FhirPathResult] = { - val ndc: Option[String] = - evaluator.visit(ndcExpr) match { - case Seq(FhirPathString(s)) => Some(s) - case Seq(FhirPathNumber(v)) => - Some((0 until (11 - v.toLong.toString().length)).map(_ => "0").mkString + v.toLong.toString()) - case Nil => None - case _ => throw new FhirPathException("Invalid parameter ndc! It should evaluate a single string") - } - val result = - ndc - .map(rxNormApiClient.findRxConceptIdByNdc) - .getOrElse(Nil) - .map(FhirPathString) - result - } - - /** - * - * @param rxcuiExpr - * @return - */ - @FhirPathFunction( - documentation = FhirPathFunctionDocumentation( - detail = "Finds out Ingredients and their properties for the given RxNorm Drug.", - usageWarnings = None, - parameters = Some( - Seq( - FhirPathFunctionParameter( - name = "rxcuiExpr", - detail = "The RxNorm Concept Id expression for the drug.", - examples = None - ) - ) - ), - returnValue = FhirPathFunctionReturn( - detail = None, - examples = Seq( - """{"ingredientAndStrength": [{"activeIngredientRxcui": "104922","activeIngredientName": "Acetaminophen","numeratorValue": 500.0,"numeratorUnit": "mg","denominatorValue": 1.0,"denominatorUnit": "tablet"},{"activeIngredientRxcui": "105784","activeIngredientName": "Ibuprofen","numeratorValue": 200.0,"numeratorUnit": "mg","denominatorValue": 1.0,"denominatorUnit": "tablet"}],"doseFormConcept": {"rxcui": "106366","doseFormName": "Tablet"}}""" - ) - ), - examples = Seq( - "rxn:getMedicationDetails('12345')" - ) - ), - insertText = "rxn:getMedicationDetails()", - detail = "rxn", - label = "rxn:getMedicationDetails", - kind = "Function", - returnType = Seq("complex"), - inputType = Seq(FHIR_DATA_TYPES.STRING) - ) - def getMedicationDetails(rxcuiExpr: ExpressionContext): Seq[FhirPathResult] = { - val conceptIds = evaluator.visit(rxcuiExpr) - if (!conceptIds.forall(_.isInstanceOf[FhirPathString])) - throw new FhirPathException("Invalid parameter rxcui! It should evaluate to 0 or more string") - val rxcuis = conceptIds.map(_.asInstanceOf[FhirPathString].s) - // Go over all the concepts and find the first one that have information we are looking for - rxcuis.iterator - .flatMap(rxcui => rxNormApiClient.getRxcuiHistoryStatus(rxcui)) - .map(response => { - // Get the ingredient details - val ingredients = - FHIRUtil - .extractValueOptionByPath[Seq[JObject]]( - response, - "rxcuiStatusHistory.definitionalFeatures.ingredientAndStrength" - ) - .getOrElse(Nil) - val ingredientObjs = - ingredients - .map(i => { - val requiredFields = - Seq( - "activeIngredientRxcui", - "activeIngredientName", - "numeratorValue", - "numeratorUnit", - "denominatorValue", - "denominatorUnit" - ) - .map(f => - FHIRUtil - .extractValueOption[String](i, f) - .filter(_ != "") // Should filter empty string values - .map(v => - if (f == "numeratorValue" || f == "denominatorValue") - f -> JDouble(v.toDouble) - else - f -> JString(v) - ) - ) - if (requiredFields.forall(_.isDefined)) - Some(JObject(requiredFields.map(_.get).toList)) - else - None - }) - - val doseFormObj = - FHIRUtil - .extractValueOptionByPath[Seq[JObject]](response, "rxcuiStatusHistory.definitionalFeatures.doseFormConcept") - .getOrElse(Nil) - .headOption - if (ingredientObjs.nonEmpty && ingredientObjs.forall(_.isDefined)) { - Some( - JObject( - List( - "ingredientAndStrength" -> JArray(ingredientObjs.map(_.get).toList) - ) ++ - doseFormObj - .map(d => "doseFormConcept" -> d) - .toSeq - ) - ) - } else - None - }) - .find(_.isDefined) - .map(r => FhirPathComplex(r.get)) - .toList - } - - /** - * - * @param rxcuiExpr - * @return - */ - @FhirPathFunction( - documentation = FhirPathFunctionDocumentation( - detail = "Finds out Ingredients and their properties for the given RxNorm Drug.", - usageWarnings = None, - parameters = Some( - Seq( - FhirPathFunctionParameter( - name = "rxcuiExpr", - detail = "The RxNorm Concept Id expression for the drug.", - examples = None - ) - ) - ), - returnValue = FhirPathFunctionReturn( - detail = None, - examples = Seq( - """[{"ingredient": "Acetaminophen","strength": "500 mg"}, {"ingredient": "Ibuprofen","strength": "200 mg"}]""" - ) - ), - examples = Seq( - "rxn:findIngredientsOfDrug('12345')" - ) - ), - insertText = "rxn:findIngredientsOfDrug()", - detail = "rxn", - label = "rxn:findIngredientsOfDrug", - kind = "Function", - returnType = Seq("complex"), - inputType = Seq(FHIR_DATA_TYPES.STRING) - ) - def findIngredientsOfDrug(rxcuiExpr: ExpressionContext): Seq[FhirPathResult] = { - val rxcui: Option[String] = - evaluator.visit(rxcuiExpr) match { - case Seq(FhirPathString(s)) => Some(s) - case Nil => None - case _ => throw new FhirPathException("Invalid parameter rxcui! It should evaluate to a single string") - } - rxcui - .map(rxNormApiClient.getIngredientProperties) - .map(_.map(FhirPathComplex)) - .getOrElse(Nil) - } - - /** - * Get the corresponding ATC code for given RxNorm ingredient concept - * @param rxcuiExpr Concept id - * @return - */ - @FhirPathFunction( - documentation = FhirPathFunctionDocumentation( - detail = "Finds the corresponding ATC code for the given RxNorm ingredient concept.", - usageWarnings = None, - parameters = Some( - Seq( - FhirPathFunctionParameter( - name = "rxcuiExpr", - detail = "The RxNorm Concept Id expression for the drug.", - examples = None - ) - ) - ), - returnValue = FhirPathFunctionReturn( - detail = None, - examples = Seq( - "\"N02BE01\"" - ) - ), - examples = Seq( - "rxn:getATC('12345')" - ) - ), - insertText = "rxn:getATC()", - detail = "rxn", - label = "rxn:getATC", - kind = "Function", - returnType = Seq(FHIR_DATA_TYPES.STRING), - inputType = Seq(FHIR_DATA_TYPES.STRING) - ) - def getATC(rxcuiExpr: ExpressionContext): Seq[FhirPathResult] = { - val rxcui: String = - evaluator.visit(rxcuiExpr) match { - case Seq(FhirPathString(s)) => s - case _ => throw new FhirPathException("Invalid parameter rxcui! It should evaluate to a single string") - } - rxNormApiClient - .getAtcCode(rxcui) - .map(FhirPathString) - } -} - -/** - * Factory class to generate function library - * @param rxNormApiRootUrl Root URL for RxNorm API e.g. https://rxnav.nlm.nih.gov - * @param timeoutInSec Timeout in seconds for API calls - * @param actorSystem Akka actor system for HTTP calls - */ -class RxNormApiFunctionLibraryFactory(rxNormApiRootUrl: String, timeoutInSec: Int /*config:Config*/ ) - extends IFhirPathFunctionLibraryFactory - with Serializable { - // val rxNormApiRootUrl = config.getString("api.root-url") - // val timeoutInSec = config.getInt("api.timeout") - val rxNormApiClient = RxNormApiClient(rxNormApiRootUrl, timeoutInSec) - override def getLibrary(context: FhirPathEnvironment, current: Seq[FhirPathResult]): AbstractFhirPathFunctionLibrary = - new RxNormApiFunctionLibrary(rxNormApiClient, context, current) -} diff --git a/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiClientTest.scala b/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiClientTest.scala deleted file mode 100644 index 4d86e2aa..00000000 --- a/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiClientTest.scala +++ /dev/null @@ -1,104 +0,0 @@ -package io.ignifyr.rxnorm - -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.onfhir.path.FhirPathException -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Covers how the client reads RxNorm's responses: which JSON path each call picks its answer out of, - * and what an absent or malformed answer turns into. The bodies below are trimmed real RxNorm - * responses served by [[RxNormApiStub]]. - */ -class RxNormApiClientTest extends AnyFlatSpec with Matchers with RxNormApiStub { - - override protected val cannedResponses: Map[String, String] = Map( - "/REST/ndcproperties.json?id=63739054410&ndcstatus=ALL" -> - """{"ndcPropertyList":{"ndcProperty":[{"ndcItem":"63739-0544-10","rxcui":"313096"}]}}""", - // RxNorm answers an unknown NDC with an empty object rather than a 404. - "/REST/ndcproperties.json?id=123&ndcstatus=ALL" -> "{}", - // A concept whose rxcui entries are placeholders; the client filters "" and "/" out. - "/REST/ndcproperties.json?id=00000000000&ndcstatus=ALL" -> - """{"ndcPropertyList":{"ndcProperty":[{"rxcui":""},{"rxcui":"/"},{"rxcui":"111"}]}}""", - "/REST/rxcui.json?name=aspirin&search=2" -> """{"idGroup":{"name":"aspirin","rxnormId":["1191"]}}""", - "/REST/rxcui.json?name=nothing&search=2" -> """{"idGroup":{"name":"nothing"}}""", - "/REST/rxcui/603748/historystatus.json" -> - """{"rxcuiStatusHistory":{"metaData":{"status":"Active"},"attributes":{"rxcui":"603748"}}}""", - "/REST/rxcui/999999999/historystatus.json" -> """{"rxcuiStatusHistory":{"metaData":{"status":"UNKNOWN"}}}""", - "/REST/rxcui/476556/allProperties.json?prop=attributes" -> - """{"propConceptGroup":{"propConcept":[ - |{"propCategory":"ATTRIBUTES","propName":"Active_ingredient_RxCUI","propValue":"276237"}, - |{"propCategory":"ATTRIBUTES","propName":"Active_ingredient_RxCUI","propValue":"282401"}, - |{"propCategory":"ATTRIBUTES","propName":"Active_ingredient_name","propValue":"emtricitabine"}, - |{"propCategory":"ATTRIBUTES","propName":"Active_ingredient_name","propValue":"tenofovir disoproxil fumarate"}, - |{"propCategory":"ATTRIBUTES","propName":"Numerator_Value","propValue":"200"}, - |{"propCategory":"ATTRIBUTES","propName":"Numerator_Value","propValue":"300"}, - |{"propCategory":"ATTRIBUTES","propName":"Numerator_Units","propValue":"MG"}, - |{"propCategory":"ATTRIBUTES","propName":"Numerator_Units","propValue":"MG"}, - |{"propCategory":"ATTRIBUTES","propName":"Denominator_Value","propValue":"1"}, - |{"propCategory":"ATTRIBUTES","propName":"Denominator_Value","propValue":"1"}, - |{"propCategory":"ATTRIBUTES","propName":"Denominator_Units","propValue":"EA"}, - |{"propCategory":"ATTRIBUTES","propName":"Denominator_Units","propValue":"EA"} - |]}}""".stripMargin, - "/REST/rxcui/476/allProperties.json?prop=attributes" -> "{}", - "/REST/rxcui/276237/property.json?propName=ATC" -> - """{"propConceptGroup":{"propConcept":[{"propName":"ATC","propValue":"J05AF09"}]}}""" - ) - - private lazy val client = RxNormApiClient(rxNormRootUrl, timeoutInSec = 10) - - "RxNormApiClient" should "get the corresponding RxNorm CUI for a given NDC" in { - client.findRxConceptIdByNdc("63739054410") shouldBe Seq("313096") - } - - it should "return nothing for a non-existent NDC" in { - client.findRxConceptIdByNdc("123") shouldBe Nil - } - - it should "drop the empty and slash placeholder concept ids" in { - client.findRxConceptIdByNdc("00000000000") shouldBe Seq("111") - } - - it should "find a concept id by drug name" in { - client.findRxConceptIdByName("aspirin") shouldBe Some("1191") - } - - it should "return nothing when a drug name resolves to no concept" in { - client.findRxConceptIdByName("nothing") shouldBe None - } - - it should "get the history status of a known concept" in { - client.getRxcuiHistoryStatus("603748").isDefined shouldBe true - } - - // RxNorm answers for an unknown concept too, with status UNKNOWN; the client must not treat that as a hit. - it should "report no history status for a concept RxNorm does not know" in { - client.getRxcuiHistoryStatus("999999999") shouldBe None - } - - it should "get the ingredients of a drug, pairing each property by position" in { - val result = client.getIngredientProperties("476556") - result.length shouldBe 2 - (result.head \ "Active_ingredient_RxCUI").extract[String] shouldBe "276237" - (result.head \ "Active_ingredient_name").extract[String] shouldBe "emtricitabine" - (result.head \ "Numerator_Value").extract[Int] shouldBe 200 - (result.head \ "Numerator_Units").extract[String] shouldBe "MG" - (result(1) \ "Active_ingredient_name").extract[String] shouldBe "tenofovir disoproxil fumarate" - } - - it should "return no ingredients for a concept that has none" in { - client.getIngredientProperties("476") shouldBe empty - } - - it should "get the ATC code of a concept" in { - client.getAtcCode("276237") shouldBe Seq("J05AF09") - } - - // Any status other than 200 is turned into a FhirPathException naming the root url, because these - // calls run inside FHIRPath evaluation where that message is all the mapping author will see. - it should "raise a FhirPathException naming the root url when RxNorm does not answer with 200" in { - val thrown = the[FhirPathException] thrownBy client.getAtcCode("no-such-concept") - thrown.getMessage should include(rxNormRootUrl) - thrown.getMessage should include("404") - } -} diff --git a/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibraryTest.scala b/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibraryTest.scala deleted file mode 100644 index 2e6c4bbf..00000000 --- a/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiFunctionLibraryTest.scala +++ /dev/null @@ -1,44 +0,0 @@ -package io.ignifyr.rxnorm - -import io.onfhir.path.FhirPathEvaluator -import org.json4s.JsonAST.JNull -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * The `rxn:` FHIRPath functions, resolved against [[RxNormApiStub]] rather than the live RxNorm API. - * This is the layer a mapping author actually writes against, and it is attached by configuration only - * (`ignifyr.functionLibraries.rxn.className`), so nothing in Scala would notice if a function stopped - * resolving. - */ -class RxNormApiFunctionLibraryTest extends AnyFlatSpec with Matchers with RxNormApiStub { - - override protected val cannedResponses: Map[String, String] = Map( - "/REST/ndcproperties.json?id=63739054410&ndcstatus=ALL" -> - """{"ndcPropertyList":{"ndcProperty":[{"rxcui":"313096"}]}}""", - // The library normalises an NDC to 11 digits before calling, so the stub is keyed on the padded form. - "/REST/ndcproperties.json?id=00000000123&ndcstatus=ALL" -> "{}", - "/REST/rxcui/276237/property.json?propName=ATC" -> - """{"propConceptGroup":{"propConcept":[{"propName":"ATC","propValue":"J05AF09"}]}}""" - ) - - private lazy val evaluator = - FhirPathEvaluator - .apply() - .withDefaultFunctionLibraries() - .withFunctionLibrary("rxn", new RxNormApiFunctionLibraryFactory(rxNormRootUrl, 10)) - - "RxNormApiFunctionLibrary" should "resolve findRxConceptIdsByNdc" in { - evaluator.evaluateOptionalString("rxn:findRxConceptIdsByNdc(63739054410)", JNull) shouldBe Some("313096") - } - - it should "return nothing for an NDC RxNorm does not know" in { - evaluator.evaluateOptionalString("rxn:findRxConceptIdsByNdc(123)", JNull) shouldBe None - } - - // Unlike findRxConceptIdsByNdc, getATC rejects a numeric literal: its rxcui parameter must be a string. - it should "resolve getATC" in { - evaluator.evaluateOptionalString("rxn:getATC('276237')", JNull) shouldBe Some("J05AF09") - } - -} diff --git a/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiStub.scala b/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiStub.scala deleted file mode 100644 index 8a84bded..00000000 --- a/ignifyr-rxnorm/src/test/scala/io/ignifyr/rxnorm/RxNormApiStub.scala +++ /dev/null @@ -1,63 +0,0 @@ -package io.ignifyr.rxnorm - -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.model._ -import org.scalatest.{BeforeAndAfterAll, Suite} - -import scala.concurrent.Await -import scala.concurrent.duration.DurationInt - -/** - * A local stand-in for the RxNorm REST API, bound on an ephemeral port for the lifetime of a suite. - * - * The client under test takes its root url as a constructor argument, so pointing it here is the whole - * of the seam — no HTTP interception and no extra dependency (the stub uses the same akka-http the - * client itself calls through). Testing against rxnav.nlm.nih.gov instead would make a green build - * depend on a third party's uptime, and would give no way at all to exercise the not-found and - * non-200 branches. - * - * A suite declares the canned bodies it needs in [[cannedResponses]], keyed by request path plus its - * query parameters sorted by name; anything unlisted is answered with 404, which is what makes an - * unexpected call visible instead of silent. - */ -trait RxNormApiStub extends BeforeAndAfterAll { this: Suite => - - // The client keeps its own singleton ActorSystem; reusing it avoids standing up a second one. - private implicit val actorSystem: ActorSystem = RxNormApiClient.actorSystem - - private var binding: Http.ServerBinding = _ - - /** Root url to hand to the code under test. Valid between `beforeAll` and `afterAll`. */ - protected def rxNormRootUrl: String = s"http://localhost:${binding.localAddress.getPort}" - - /** Canned JSON bodies, keyed as `` or `?` with the names sorted. */ - protected def cannedResponses: Map[String, String] - - override protected def beforeAll(): Unit = { - super.beforeAll() - binding = Await.result( - Http() - .newServerAt("localhost", 0) - .bindSync { request => - cannedResponses.get(keyOf(request)) match { - case Some(body) => HttpResponse(entity = HttpEntity(ContentTypes.`application/json`, body)) - case None => HttpResponse(StatusCodes.NotFound) - } - }, - 10.seconds - ) - } - - override protected def afterAll(): Unit = { - try Await.result(binding.unbind(), 10.seconds) - finally super.afterAll() - } - - /** The client builds some uris with an ordered query map and others by hand, so sort before matching. */ - private def keyOf(request: HttpRequest): String = { - val path = request.uri.path.toString - val parameters = request.uri.query().toMap.toSeq.sorted.map { case (name, value) => s"$name=$value" } - if (parameters.isEmpty) path else s"$path?${parameters.mkString("&")}" - } -} diff --git a/ignifyr-server-common/CLAUDE.md b/ignifyr-server-common/CLAUDE.md deleted file mode 100644 index 41a4c48a..00000000 --- a/ignifyr-server-common/CLAUDE.md +++ /dev/null @@ -1,51 +0,0 @@ -# ignifyr-server-common — shared server infrastructure + the server extension SPI - -Shared Akka-HTTP plumbing used by `ignifyr-server` (and any other server build) **and** the home of the -server-side extension SPI the edition split relies on. Package root `io.ignifyr.server.common`. Five -source files; changes ripple to every server endpoint and every server plugin, so keep them -backward-compatible. - -**Why it exists: to break a cycle.** A server-side plugin can never depend on `ignifyr-server` — that is -the distribution that shades it — yet both halves must compile against the same seam. So the seam lives -here, in a library that declares **no Ignifyr dependency at all** (not the engine, not -`ignifyr-common`): a server plugin can implement `IgnifyrServerExtension` without pulling in the engine -or Spark. Enterprise-side; it correctly does not opt into the `ban-enterprise-deps` enforcer gate, and it -is a plain jar (no shade). - -**Note the artifactId: `ignifyr-server-common`, with no `_2.13` suffix** (only this module and -`ignifyr-common` are unsuffixed). - -## Shared plumbing -- `config/WebServerConfig` — host / port / base-uri (from the `webserver` HOCON block), plus SSL/keystore - and server-location settings. -- `interceptor/ICORSHandler` — CORS directive trait mixed into endpoints. -- `interceptor/IErrorHandler` — maps exceptions → HTTP responses; also translates onFHIR - `FhirDefinitionsError` into the module's own errors. -- `model/IgnifyrError` — abstract HTTP-error base plus the concrete taxonomy: `BadRequest` (400), - `AlreadyExists` (409), `ResourceNotFound` (404), `InternalError` (500), `UnsupportedMediaType` (415), - `MethodForbidden` (405), `RequestTimeout` (408). Pairs with `IErrorHandler`. -- `model/IgnifyrRestCall` — per-request context. - -## The server extension SPI (`spi/`) -The server-side counterpart to the engine's `io.ignifyr.engine.spi.IgnifyrExtension`. Contributing a -server feature (routes, schema imports, a `/metadata` version) is done by adding an `IgnifyrServerExtension` -+ a `META-INF/services` entry in a plugin module, with **zero `ignifyr-server` code change** — mirroring the -engine's design rule (see `ignifyr-redcap`). -- `spi/IgnifyrServerExtension` — the SPI: `id`, `initialize(rootConfig)`, `rootRoutes`, - `schemaImportRoutes(SchemaImportSink)` (mounted under `/projects/{id}/schemas` by `SchemaDefinitionEndpoint`), - and `externalComponentVersion()`. ⚠️ The last one is **not** generically surfaced: the server's - `MetadataService` looks up `id == "redcap"` and reports only that extension's version, so - implementing the hook in a different module has no effect today. -- `spi/IgnifyrServerExtensions` — the ServiceLoader discovery object (`load(rootConfig)`): finds the - extensions, **sorts by id, then initializes each** (in that order). -- `spi/SchemaImportSink` — a narrow persistence callback (`saveSchemas(projectId, schemas)`) handed to - schema-import routes so an extension never touches the repository layer. - -## Dependency & tests -Declares `com.typesafe:config`, akka-http/-core, akka-stream-typed (provided), scala-logging, and -`onfhir-resource-definitions` — from which `IErrorHandler` takes `FhirDefinitionsError`. Note -`SchemaImportSink`'s `SchemaDefinition` actually comes from **`onfhir-definition-commons`**, reached -transitively. That single onFHIR dependency is not cheap: it drags in the whole onFHIR server stack -(definition-commons, client, config, r4, server-r4, server-r5) for two model types. - -No tests live here; exercised through `ignifyr-server`. diff --git a/ignifyr-server-common/pom.xml b/ignifyr-server-common/pom.xml deleted file mode 100644 index 072fcae5..00000000 --- a/ignifyr-server-common/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-server-common - jar - - - src/main/scala - - - - net.alchim31.maven - scala-maven-plugin - - ${scala.binary.version} - UTF-8 - - -deprecation - -feature - - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - - - - - com.typesafe - config - - - - com.typesafe.akka - akka-http_${scala.binary.version} - - - com.typesafe.akka - akka-http-core_${scala.binary.version} - - - - com.typesafe.akka - akka-stream-typed_${scala.binary.version} - provided - - - - com.typesafe.scala-logging - scala-logging_${scala.binary.version} - - - - io.onfhir - onfhir-resource-definitions_${scala.binary.version} - - - diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/config/WebServerConfig.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/config/WebServerConfig.scala deleted file mode 100644 index 39104a76..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/config/WebServerConfig.scala +++ /dev/null @@ -1,33 +0,0 @@ -package io.ignifyr.server.common.config - -import com.typesafe.config.Config - -import scala.util.Try - -/** - * Configuration for the Http serving (host, port, etc) - * @param webserverConfig - */ -class WebServerConfig(webserverConfig: Config) { - - /** Host name/address to start service on. */ - lazy val serverHost: String = Try(webserverConfig.getString("host")).getOrElse("localhost") - - /** Port to start service on. */ - lazy val serverPort: Int = Try(webserverConfig.getInt("port")).getOrElse(8085) - - /** SSL settings if server will be server over SSL */ - lazy val serverSsl: Boolean = - Try(webserverConfig.getConfig("ssl").isEmpty || webserverConfig.getString("ssl.keystore") != null).getOrElse(false) - lazy val sslKeystorePath: Option[String] = Try(webserverConfig.getString("ssl.keystore")).toOption - lazy val sslKeystorePassword: Option[String] = Try(webserverConfig.getString("ssl.password")).toOption - - /** Protocol of the server. Either http or https and the value is determined from spray properties */ - lazy val serverProtocol: String = if (serverSsl) "https" else "http" - - /** Full address of the server */ - lazy val serverLocation: String = serverProtocol + "://" + serverHost + ":" + serverPort - - /** Base URI for Services to be served from */ - lazy val baseUri: String = Try(webserverConfig.getString("base-uri")).getOrElse("ignifyr") -} diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/ICORSHandler.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/ICORSHandler.scala deleted file mode 100644 index 63e9220d..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/ICORSHandler.scala +++ /dev/null @@ -1,67 +0,0 @@ -package io.ignifyr.server.common.interceptor - -import akka.http.scaladsl.model.headers._ -import akka.http.scaladsl.model.{HttpMethods, HttpResponse, StatusCodes} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.directives.BasicDirectives -import akka.http.scaladsl.server.{Directive0, Directives, Route} - -/** - * Cors Handler for webserver modules - */ -trait ICORSHandler extends BasicDirectives { - - /** - * Fixed CORS Response headers - */ - private val corsResponseHeaders = List( - `Access-Control-Allow-Origin`.*, - `Access-Control-Allow-Credentials`(true), - `Access-Control-Allow-Headers`( - "Origin, X-Requested-With, X-Correlation-Id, Content-Type, Accept, Accept-Encoding, Accept-Language, Authorization, Host, Referer, User-Agent, Link" - ), - `Access-Control-Max-Age`(1728000), - `Access-Control-Expose-Headers`("Location", "Link", ICORSHandler.X_TOTAL_COUNT_HEADER) - ) - - // this directive adds access control headers to normal responses - private def addAccessControlHeaders(): Directive0 = { - Directives.respondWithHeaders(corsResponseHeaders) - } - - // this handles preflight OPTIONS requests. - private def preflightRequestHandler: Route = options { - complete( - HttpResponse(StatusCodes.OK) - .withHeaders( - `Access-Control-Allow-Methods`( - HttpMethods.OPTIONS, - HttpMethods.HEAD, - HttpMethods.POST, - HttpMethods.PUT, - HttpMethods.GET, - HttpMethods.DELETE, - HttpMethods.PATCH - ) - ) - ) - } - - // Wrap the Route with this method to enable adding of CORS headers - def corsHandler(r: Route): Route = addAccessControlHeaders() { - preflightRequestHandler ~ r - } - - // Helper method to add CORS headers to HttpResponse - // preventing duplication of CORS headers across code - def addCORSHeaders(response: HttpResponse): HttpResponse = - response.withHeaders(corsResponseHeaders) - -} - -/** - * Keeps the custom header names - */ -object ICORSHandler { - val X_TOTAL_COUNT_HEADER = "X-Total-Count" // Header to return the total number of resources -} diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/IErrorHandler.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/IErrorHandler.scala deleted file mode 100644 index 4b3384cb..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/interceptor/IErrorHandler.scala +++ /dev/null @@ -1,47 +0,0 @@ -package io.ignifyr.server.common.interceptor - -import com.typesafe.scalalogging.LazyLogging -import akka.http.scaladsl.server.{Directives, ExceptionHandler} -import io.onfhir.definitions.resource.model -import io.onfhir.definitions.resource.model.FhirDefinitionsError -import io.ignifyr.server.common.model.{BadRequest, InternalError, IgnifyrError, IgnifyrRestCall} - -trait IErrorHandler extends LazyLogging { - - /** - * Exception Handler object - * - * @return - */ - def exceptionHandler(ignifyrRestCall: IgnifyrRestCall) = - ExceptionHandler { case e: Exception => - Directives.complete { - val response = exceptionToResponse(e) - response.statusCode -> response.toString - } - } - - /** - * Handling of exceptions by converting them to IgnifyrResponse - * - * @return - */ - private def exceptionToResponse: PartialFunction[Exception, IgnifyrError] = { - case e: IgnifyrError => - logger.error(s"Ignifyr error encountered: ${e.toString}", e) - e - // handle the errors coming from onFHIR Definitions Microservice - case e: FhirDefinitionsError => - e match { - case model.BadRequest(title, detail, cause) => - logger.error(s"Ignifyr error encountered: ${e.toString}", e) - BadRequest(title, detail, cause) - case _ => - logger.error("Unexpected internal error", e) - InternalError(s"Unexpected internal error: ${e.getClass.getName}", e.getMessage, Some(e)) - } - case e: Exception => - logger.error("Unexpected internal error", e) - InternalError(s"Unexpected internal error: ${e.getClass.getName}", e.getMessage, Some(e)) - } -} diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrError.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrError.scala deleted file mode 100644 index 5346213e..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrError.scala +++ /dev/null @@ -1,81 +0,0 @@ -package io.ignifyr.server.common.model - -import java.io.{PrintWriter, StringWriter} - -/** - * Any exception thrown by Ignifyr server - */ -abstract class IgnifyrError extends Exception { - - /** - * HTTP status code to return when this error occurs - */ - val statusCode: Int - - /** - * Type of the error - */ - val `type`: String = s"https://ignifyr.io/errors/${getClass.getSimpleName}" - - /** - * Title of the error - */ - val title: String - - /** - * Details of the error - */ - val detail: String - - /** - * Inner exception - */ - val cause: Option[Throwable] = None - - override def toString: String = { - s"Status Code: $statusCode\n" + - s"Type: ${`type`}\n" + - s"Title: $title\n" + - s"Detail: $detail\n" + - s"Stack Trace: ${if (cause.isDefined) getStackTraceAsString(cause.get)}" - } - - override def getMessage: String = s"$title $detail" - - private def getStackTraceAsString(t: Throwable) = { - val sw = new StringWriter - t.printStackTrace(new PrintWriter(sw)) - sw.toString - } - -} - -case class BadRequest(title: String, detail: String, override val cause: Option[Throwable] = None) - extends IgnifyrError { - val statusCode = 400 -} - -case class AlreadyExists(title: String, detail: String) extends IgnifyrError { - val statusCode = 409 -} - -case class ResourceNotFound(title: String, detail: String) extends IgnifyrError { - val statusCode = 404 -} - -case class InternalError(title: String, detail: String, override val cause: Option[Throwable] = None) - extends IgnifyrError { - val statusCode = 500 -} - -case class UnsupportedMediaType(title: String, detail: String) extends IgnifyrError { - val statusCode = 415 -} - -case class MethodForbidden(title: String, detail: String) extends IgnifyrError { - val statusCode = 405 -} - -case class RequestTimeout(title: String, detail: String) extends IgnifyrError { - val statusCode = 408 -} diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrRestCall.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrRestCall.scala deleted file mode 100644 index 02185bb0..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/model/IgnifyrRestCall.scala +++ /dev/null @@ -1,31 +0,0 @@ -package io.ignifyr.server.common.model - -import akka.http.scaladsl.model.{HttpMethod, RequestEntity, Uri} - -import java.time.Instant - -/** - * Object representing a REST request wrapper for Ignifyr - * - * @param method Http method - * @param uri Uri for the call - * @param requestId Unique request identifier - * @param requestEntity Request entity - */ -class IgnifyrRestCall(val method: HttpMethod, val uri: Uri, val requestId: String, val requestEntity: RequestEntity) { - - /** - * Time of the request - */ - val requestTime: Instant = Instant.now() - - /** - * Project context for the call, if exists - */ - var projectId: Option[String] = None - - /** - * Terminology id for the call, if exists - */ - var terminologyId: Option[String] = None -} diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtension.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtension.scala deleted file mode 100644 index f8654a2e..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtension.scala +++ /dev/null @@ -1,49 +0,0 @@ -package io.ignifyr.server.common.spi - -import akka.http.scaladsl.server.Route -import com.typesafe.config.Config -import io.ignifyr.server.common.model.IgnifyrRestCall - -import scala.concurrent.{ExecutionContext, Future} - -/** - * Aggregate service-provider interface for a module contributing to the Ignifyr web server — the - * server-side counterpart of the engine's `IgnifyrExtension`. One [[IgnifyrServerExtension]] is - * discovered per module through [[java.util.ServiceLoader]] (one - * `META-INF/services/io.ignifyr.server.common.spi.IgnifyrServerExtension` entry per jar), so - * moving a server feature between editions is a plain folder move plus a reactor edit, with no - * server code change. - * - * All contribution accessors default to empty; a module overrides only what it provides. The - * route accessors are consulted once, while the server assembles its route tree — an extension - * decides there (typically from configuration read in [[initialize]]) whether to contribute. - */ -trait IgnifyrServerExtension { - - /** Stable, unique identifier for this extension (e.g. "redcap"). */ - def id: String - - /** - * Called once during discovery with the server's root configuration. Extensions read their own - * top-level blocks from it (e.g. `ignifyr-redcap`) and may disable their contributions when the - * block is absent. - */ - def initialize(rootConfig: Config): Unit = () - - /** Endpoint routes mounted at the API root, beside the server's built-in endpoints. */ - def rootRoutes: Seq[IgnifyrRestCall => Route] = Nil - - /** - * Routes mounted under `/projects/{projectId}/schemas` for module-specific schema imports - * (e.g. a data-dictionary format). Imported schemas are persisted through the server-provided - * [[SchemaImportSink]]. - */ - def schemaImportRoutes(schemaImportSink: SchemaImportSink): Seq[IgnifyrRestCall => Route] = Nil - - /** - * Version of the external component this extension integrates with (e.g. the tofhir-redcap - * service), reported by the server's `/metadata`. None when unknown or not applicable. - */ - def externalComponentVersion()(implicit ec: ExecutionContext): Future[Option[String]] = - Future.successful(None) -} diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtensions.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtensions.scala deleted file mode 100644 index dcc2c837..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/IgnifyrServerExtensions.scala +++ /dev/null @@ -1,32 +0,0 @@ -package io.ignifyr.server.common.spi - -import com.typesafe.config.Config -import com.typesafe.scalalogging.Logger - -import java.util.ServiceLoader -import scala.jdk.CollectionConverters._ - -/** - * Discovers the installed [[IgnifyrServerExtension]] modules through [[java.util.ServiceLoader]] - * over the classpath (the shaded server jar merges `META-INF/services` through the shade - * `ServicesResourceTransformer`). Called once by the server's composition root; the resulting - * sequence is passed to the endpoints that consult extension contributions. - */ -object IgnifyrServerExtensions { - - private val logger: Logger = Logger(this.getClass) - - /** Load all server extensions, ordered by id, each initialized with the server's root config. */ - def load(rootConfig: Config): Seq[IgnifyrServerExtension] = { - val loaded = - ServiceLoader - .load(classOf[IgnifyrServerExtension], classOf[IgnifyrServerExtension].getClassLoader) - .iterator() - .asScala - .toSeq - .sortBy(_.id) - loaded.foreach(_.initialize(rootConfig)) - logger.info(s"Loaded ${loaded.size} Ignifyr server extension(s): ${loaded.map(_.id).mkString(", ")}") - loaded - } -} diff --git a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/SchemaImportSink.scala b/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/SchemaImportSink.scala deleted file mode 100644 index 5c45c7a5..00000000 --- a/ignifyr-server-common/src/main/scala/io/ignifyr/server/common/spi/SchemaImportSink.scala +++ /dev/null @@ -1,20 +0,0 @@ -package io.ignifyr.server.common.spi - -import io.onfhir.definitions.common.model.SchemaDefinition - -import scala.concurrent.Future - -/** - * Persistence callback handed to [[IgnifyrServerExtension.schemaImportRoutes]]: an extension's - * import route parses its format into [[SchemaDefinition]]s and saves them through this sink, - * without depending on the server's repository layer. - */ -trait SchemaImportSink { - - /** - * Persist the imported schema definitions for the given project. - * - * @return the saved schema definitions - */ - def saveSchemas(projectId: String, schemas: Seq[SchemaDefinition]): Future[Seq[SchemaDefinition]] -} diff --git a/ignifyr-server/CLAUDE.md b/ignifyr-server/CLAUDE.md deleted file mode 100644 index 7385ecab..00000000 --- a/ignifyr-server/CLAUDE.md +++ /dev/null @@ -1,100 +0,0 @@ -# ignifyr-server — REST API server - -Akka-HTTP server exposing a REST API to manage Ignifyr projects, mappings, schemas, jobs, -terminology systems, and to trigger executions. Package root `io.ignifyr.server`. - -**It is simultaneously the REST API and the enterprise standalone distribution** — the only module in -the reactor that is both (`ignifyr-cli`, its community counterpart, has no `src/` at all). Its -maven-shade assembly produces `target/ignifyr-server-standalone.jar` (Main-Class -`io.ignifyr.server.Boot`). Its POM declares **17** compile-scope `io.ignifyr` dependencies — engine, -common, server-common, rxnorm, `connector-sql`/`-file`/`-fhir-server`/`-kafka`, -`sink-fhir`/`-file`/`-omop`, `format-json`/`-delta`, `runtime-streaming`/`-scheduling`, redcap, -observability — plus `ignifyr-testkit` at test scope. **That dependency list is the definition of the -Enterprise edition**, so moving a feature between editions is one line here and one in -`ignifyr-cli/pom.xml`. It deliberately does *not* opt into the `ban-enterprise-deps` enforcer gate: it -legitimately pulls Kafka, Delta, cron4j and Logstash/Fluentd transitively. - -Since the engine ships no concrete I/O, the sink modules are as load-bearing as the connectors — drop -`ignifyr-sink-fhir` and the server can no longer write to a FHIR repository. Everything is discovered at -runtime through the `IgnifyrExtension` / `IgnifyrServerExtension` ServiceLoader SPIs; no concrete plugin -type is imported anywhere in `src/main/scala`. Two string-level couplings are the exceptions to that -rule: `MetadataService` hardcodes the extension id `"redcap"`, and `src/main/resources/logback.xml` -hardcodes `io.ignifyr.observability.logback.MapMarkerToLogstashMarkerEncoder`. - -## Entry & wiring -- `Boot` → `IgnifyrServer.start()` → `IgnifyrHttpServer` binds Akka-HTTP (default port **8085**, base path `/ignifyr`). -- `IgnifyrServer.start()` also runs `IgnifyrServerExtensions.load(rootConfig)` and threads the discovered - `Seq[IgnifyrServerExtension]` into `IgnifyrServerEndpoint`, `MetadataEndpoint`, and (via `ProjectEndpoint`) - `SchemaDefinitionEndpoint`. -- `endpoint/IgnifyrServerEndpoint` assembles the full route tree (`ignifyrRoute`) from the per-resource - endpoints, concatenating each extension's `rootRoutes` (`extensionRootRoutes`). - -## Layout (`src/main/scala/io/ignifyr/server/`) -- `endpoint/` — one class per resource: `ProjectEndpoint`, `MappingEndpoint`, `SchemaDefinitionEndpoint`, - `JobEndpoint`, `MappingContextEndpoint`, `TerminologyServiceManagerEndpoint`, `CodeSystemEndpoint`, - `ConceptMapEndpoint`, `ReloadEndpoint`, `MetadataEndpoint`, `FileSystemTreeStructureEndpoint`, - plus the root `IgnifyrServerEndpoint`. Additional routes are contributed by installed modules via - the `IgnifyrServerExtension` SPI in `ignifyr-server-common` (e.g. `ignifyr-redcap`'s `/redcap` - proxy and `/projects/{id}/schemas/redcap` import). Two seams route those contributions: - `SchemaDefinitionEndpoint` mounts each extension's `schemaImportRoutes(SchemaImportSink)` under - `/projects/{id}/schemas` (the sink persists schemas without the module touching the repository - layer), and `MetadataEndpoint` surfaces an extension's `externalComponentVersion()` in `/metadata`. - ⚠️ That second seam is **not** generic today: `MetadataService` looks up `id == "redcap"` and - publishes only that one result, as `Metadata.ignifyrRedcapVersion`. No other extension's version is - ever queried. -- `service/` — business logic per resource (`ProjectService`, `MappingService`, `JobService`, - `ExecutionService`, `SchemaDefinitionService`, terminology services…). -- `repository/` — file-backed persistence; an interface + a `*FolderRepository` impl per resource - (`IProjectRepository`/`ProjectFolderRepository`, `IJobRepository`/`JobFolderRepository`, mapping, - mappingContext, schema, terminology). `FolderDBInitializer` + `FolderRepositoryManager` bootstrap the - on-disk repo (project index `projects.json`). -- `model/` — request/response + domain models (`Project`, `ExecuteJobTask`, `InferTask`, …). -- `util/` — `IgnifyrRejectionHandler`, `FileOperations`, `CsvUtil`, `DataFrameUtil`. - ⚠️ Every `CsvUtil` function rewrites a user-owned file **in place**, so its `Future` must not complete - before the write does. `writeCsvHeaders` used to `map` over the read instead of `flatMap`-ing the write, - so the header endpoint could answer OK over stale content (fixed 2026-08-10; the `Thread.sleep(5000)` - in `MappingContextEndpointTest` is the leftover workaround and can now go). - -The layered pattern is **Endpoint (route/marshalling) → Service (logic) → Repository (persistence)**. -Cross-cutting CORS/error handling and `WebServerConfig` come from `ignifyr-server-common`. - -## Adding / changing an endpoint -1. Add or extend the `*Endpoint` (route + directives), delegating to a `*Service`. -2. Add the `*Service` logic; persist via a `*Repository` (interface `repository//I*.scala`, - folder impl `*FolderRepository`). -3. Register the route in `IgnifyrServerEndpoint` — or, for an optional/edition-specific feature, - contribute it from its own module via `IgnifyrServerExtension` (see `ignifyr-redcap`). -4. **Update [api.yaml](api.yaml)** (OpenAPI spec) to match the change. -5. Add a suite extending `BaseEndpointTest` (`AnyWordSpec` + `ScalatestRouteTest`); use `createProject()` - for tests that need an existing project. - -## Config & run -Reads HOCON sections `ignifyr` (engine), `webserver` (`WebServerConfig` — host/port/base-uri), -`fhir` (definitions), and `ignifyr-redcap`. Run: -`java -Dconfig.file= -jar target/ignifyr-server-standalone.jar`. - -## Tests -`mvn test -pl ignifyr-server` → suites under `io.ignifyr.server` and its sub-packages `endpoint`, -`service`, `repository` and `util`. - -The suites are **tier-split** like every other module (two `scalatest-maven-plugin` executions): -- **short** — `wildcardSuites=io.ignifyr.server` (which covers the sub-packages too): eleven endpoint - suites, `ExecutionServiceTest` + `MetadataServiceTest`, `FolderDBInitializerTest`, `CsvUtilTest` + - `DataFrameUtilTest`, and the `BaseEndpointTest`/`TestUtil` support classes. Docker-free, runs in - `mvn test`. Four of these cover paths no endpoint test can reach: `FolderDBInitializerTest` drives the - two **startup** paths (read `projects.json` back vs. rebuild the index by scanning the folders, plus its - refusal to start on a dangling reference) — a failure there means "the server does not start"; - `IgnifyrErrorHandlerTest` covers a route **throwing** (the counterpart to `IgnifyrRejectionHandlerTest`, - which covers Akka *rejections*) and drives a minimal route because no real endpoint can be made to throw - on demand; `MetadataServiceTest` pins the 1-second bound and swallow-everything behaviour of the - `/metadata` extension lookup, invisible from the endpoint test which only sees a 200; and - `JobExecutionControlEndpointTest` pins the 404-vs-empty-list distinction the web UI polls on. -- **long** — `membersOnlySuites=io.ignifyr.integrationtest`, gated on `${skipITs}`: the three suites that - mix in the testkit's `OnFhirTestContainer` (`FhirDefinitionsEndpointTest`, `SchemaEndpointTest`, - `MappingExecutionEndpointTest`) and so start an `srdc/onfhir:r5` container backed by `mongo:7.0`. They - live in `io.ignifyr.integrationtest`, not under `io.ignifyr.server.endpoint`, precisely because the - package is what selects the tier. Run them with `mvn -B verify -DskipITs=false`. - -`BaseEndpointTest` itself starts no container — it boots the route via `ScalatestRouteTest`. Historically -this module ran everything (containers included) in the `test` phase; `test-flow/check-test-tiers.sh` now -fails the build if a container-backed suite reappears in a short-tier package. diff --git a/ignifyr-server/api.yaml b/ignifyr-server/api.yaml deleted file mode 100644 index 8dffbc2d..00000000 --- a/ignifyr-server/api.yaml +++ /dev/null @@ -1,3593 +0,0 @@ -openapi: 3.0.2 -info: - description: "Lists the REST APIs provided by an Ignifyr server" - version: "1.1" - title: "Ignifyr REST API" - -servers: - - url: http://localhost:8085/ignifyr - description: Local deployment of the Ignifyr server - - -tags: - - name: "Terminology" - description: "Represents a logical grouping for terminolgy entities with concept maps and code systems" - - name: "Project" - description: "Represents a logical grouping for project entities" - - name: "Schema" - description: "Represents a logical grouping for schema entities" - - name: "Mapping" - description: "Represents a logical grouping for mapping entities" - - name: "Job" - description: "Represents a logical grouping for job entities" - - name: "Execution" - description: "Represents a logical grouping for job execution entities" - - name: "Mapping Context" - description: "Represents a logical grouping for mapping context entities" - - name: "Fhir Definition" - description: "Represents a logical grouping for fhir definition entities" - - name: "Fhir Path Functions" - description: "Represents a logical grouping for fhir path function entities" - - name: "File System Path" - description: "Represents a logical grouping for file system path entities." - - name: "Metadata" - description: "Represents a logical grouping for metadata entities." - - name: "REDCap" - description: "Represents a logical grouping for REDCap function entities" - - -paths: - - # Beginning of the Terminology tag - /terminologies: - get: - tags: - - "Terminology" - summary: "Gets all terminologies excluding their code systems and concept maps" - operationId: "getTerminologies" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/TerminologySystem" - example: [{ - "id": "test-id", - "name": "test-name", - "description": "test-description", - "conceptMaps": [], - "codeSystems": [] - }] - - post: - tags: - - "Terminology" - summary: "Creates a new terminology" - operationId: "createTerminology" - requestBody: - description: "Terminology details" - content: - application/json: - schema: - $ref: "#/components/schemas/TerminologySystem" - responses: - '201': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/TerminologySystem" - '400': - $ref: '#/components/responses/400BadRequest' - '409': - $ref: '#/components/responses/409AlreadyExists' - - /terminologies/{terminologyId}: - get: - tags: - - "Terminology" - summary: "Gets the specified terminology" - operationId: "getTerminology" - parameters: - - $ref: "#/components/parameters/terminologyId" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/TerminologySystem" - '404': - $ref: "#/components/responses/404NotFound" - - put: - tags: - - "Terminology" - summary: "Updates the specified terminology" - operationId: "updateTerminology" - parameters: - - $ref: "#/components/parameters/terminologyId" - requestBody: - description: "Terminology details" - content: - application/json: - schema: - $ref: "#/components/schemas/TerminologySystem" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/TerminologySystem" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: '#/components/responses/404NotFound' - - delete: - tags: - - "Terminology" - summary: "Deletes the specified terminology" - operationId: "deleteTerminology" - parameters: - - $ref: "#/components/parameters/terminologyId" - responses: - '204': - description: "Successful operation" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: '#/components/responses/404NotFound' - - /terminologies/{terminologyId}/concept-maps/: - get: - tags: - - "Terminology" - summary: "Gets all concept maps for the specified terminology" - operationId: "getConceptMaps" - parameters: - - $ref: "#/components/parameters/terminologyId" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/TerminologyConceptMap" - - /terminologies/{terminologyId}/concept-maps/{conceptMapId}: - get: - tags: - - "Terminology" - summary: "Gets the specified concept map for the specified terminology" - operationId: "getConceptMap" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/TerminologyConceptMap" - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/concept-maps/{conceptMapId}/content: - get: - tags: - - "Terminology" - summary: "Gets the paginated content of a specified concept map CSV for the specified terminology" - operationId: "getConceptMapContent" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - - in: query - name: page - schema: - type: string - required: false - - in: query - name: size - schema: - type: string - required: false - responses: - '200': - description: "Successful operation" - content: - text/csv: - schema: - type: string - format: binary - example: "source_system,source_code,target_system,target_code,target_display,equivalence\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNE,http://snomed.info/sct,309068002,Specimen from skin,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323008,Pus specimen,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323009,Pus specimen 2,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,BULLA,http://snomed.info/sct,258482009,Pus specimen 2,narrower" - '404': - $ref: "#/components/responses/404NotFound" - - post: - tags: - - "Terminology" - summary: "Updates the paginated content of the specified concept map CSV for the specified terminology" - operationId: "updateConceptMapContent" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - - in: query - name: page - schema: - type: string - required: false - requestBody: - content: - multipart/form-data: - schema: - type: object - properties: - # 'attachment' will be the field name in this multipart request - attachment: - type: string - format: binary - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/concept-maps/{conceptMapId}/header: - post: - tags: - - "Terminology" - summary: "Updates the header of the specified concept map CSV for the specified terminology" - operationId: "updateConceptMapHeader" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - requestBody: - description: " - The array of CSV headers must include the following columns: - `source_system` - `source_code` - `target_system` - `target_code` - Ensure that these column names are included as currentName in the array for a successful operation. - Following columns are optional column names for concept maps: - `target_display` - `equivalence` - Any other column name is forbidden for concept maps." - content: - application/json: - schema: - type: array - items: - $ref: "#/components/examples/CodeSystemCsvHeadersExample" - examples: - HeadersExample: - $ref: "#/components/examples/ConceptMapCsvHeadersExample" - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/concept-maps/{conceptMapId}/file: - get: - tags: - - "Terminology" - summary: "Gets the whole content of a specified concept map for the specified terminology" - operationId: "getConceptMapFile" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - responses: - '200': - description: "Successful operation" - content: - text/csv: - schema: - type: string - format: binary - example: "source_system,source_code,target_system,target_code,target_display,equivalence\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNE,http://snomed.info/sct,309068002,Specimen from skin,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323008,Pus specimen,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323009,Pus specimen 2,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,BULLA,http://snomed.info/sct,258482009,Pus specimen 2,narrower" - '404': - $ref: "#/components/responses/404NotFound" - - post: - tags: - - "Terminology" - summary: "Updates the whole content of the specified concept map for the specified terminology" - operationId: "updateConceptMapFile" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - requestBody: - content: - multipart/form-data: - schema: - type: object - properties: - # 'attachment' will be the field name in this multipart request - attachment: - type: string - format: binary - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/code-systems: - get: - tags: - - "Terminology" - summary: "Gets the code systems for the specified terminology" - operationId: "getCodeSystems" - parameters: - - $ref: "#/components/parameters/terminologyId" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/TerminologyCodeSystem" - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/code-systems/{codeSystemId}: - get: - tags: - - "Terminology" - summary: "Gets the specified code system for the specified terminology" - operationId: "getCodeSystem" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/codeSystemId" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/TerminologyCodeSystem" - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/code-systems/{conceptMapId}/content: - get: - tags: - - "Terminology" - summary: "Gets the paginated content of a specified code system CSV for the specified terminology" - operationId: "getCodeSystemContent" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - - in: query - name: page - schema: - type: string - required: false - - in: query - name: size - schema: - type: string - required: false - responses: - '200': - description: "Successful operation" - content: - text/csv: - schema: - type: string - format: binary - example: "source_system,source_code,target_system,target_code,target_display,equivalence\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNE,http://snomed.info/sct,309068002,Specimen from skin,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323008,Pus specimen,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323009,Pus specimen 2,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,BULLA,http://snomed.info/sct,258482009,Pus specimen 2,narrower" - '404': - $ref: "#/components/responses/404NotFound" - - post: - tags: - - "Terminology" - summary: "Updates the paginated content of the specified code system CSV for the specified terminology" - operationId: "updateCodeSystemContent" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/conceptMapId" - - in: query - name: page - schema: - type: string - required: false - requestBody: - content: - multipart/form-data: - schema: - type: object - properties: - # 'attachment' will be the field name in this multipart request - attachment: - type: string - format: binary - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/code-systems/{codeSystemId}/header: - post: - tags: - - "Terminology" - summary: "Updates the header of the specified code system CSV for the specified terminology" - operationId: "getCodeSystemHeader" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/codeSystemId" - requestBody: - description: " - The array of CSV headers must include the following columns: - `code` - `display` - Ensure that these column names are included as currentName in the array for a successful operation." - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/CsvHeader" - examples: - HeadersExample: - $ref: "#/components/examples/CodeSystemCsvHeadersExample" - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - /terminologies/{terminologyId}/code-systems/{codeSystemId}/file: - get: - tags: - - "Terminology" - summary: "Gets the whole content for the specified code system for the specified terminology" - operationId: "getCodeSystemFile" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/codeSystemId" - responses: - '200': - description: "Successful operation" - content: - text/csv: - schema: - type: string - format: binary - example: "code,display,fr,de\n309068002,Specimen from skin,Spécimen de peau,Probe von der Haut\n119323008,Pus specimen,Spécimen de pus,Eiterprobe" - '404': - $ref: "#/components/responses/404NotFound" - - post: - tags: - - "Terminology" - summary: "Updates the whole content for the specified code system for the specified terminology" - operationId: "updateCodeSystemFile" - parameters: - - $ref: "#/components/parameters/terminologyId" - - $ref: "#/components/parameters/codeSystemId" - requestBody: - content: - multipart/form-data: - schema: - type: object - properties: - # 'attachment' will be the field name in this multipart request - attachment: - type: string - format: binary - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - # Beginning of the Project tag - /projects: - get: - tags: - - "Project" - summary: "Gets all projects" - operationId: "getProjects" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/Project" - - post: - tags: - - "Project" - summary: "Creates a new project" - operationId: "createProject" - requestBody: - description: "Project details" - content: - application/json: - schema: - $ref: "#/components/schemas/Project" - responses: - '201': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/Project" - '400': - $ref: '#/components/responses/400BadRequest' - '409': - $ref: '#/components/responses/409AlreadyExists' - '415': - $ref: '#/components/responses/415UnsupportedMediaType' - - /projects/{projectId}: - get: - tags: - - "Project" - summary: "Gets the specified project" - operationId: "getProject" - parameters: - - $ref: '#/components/parameters/projectId' - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/Project" - '404': - $ref: '#/components/responses/404NotFound' - - patch: - tags: - - "Project" - summary: "Updates the description, schemaUrlPrefix and mappingUrlPrefix project with the given patch" - operationId: "patchProject" - parameters: - - $ref: '#/components/parameters/projectId' - requestBody: - description: "description, schemaUrlPrefix and mappingUrlPrefix" - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/PatchProjectSchema" - responses: - '200': - description: "Successful operation" - content: - application/json: - schema: - $ref: "#/components/schemas/Project" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: '#/components/responses/404NotFound' - '415': - $ref: '#/components/responses/415UnsupportedMediaType' - - delete: - tags: - - "Project" - summary: "Deletes the specified project" - operationId: "deleteProject" - parameters: - - $ref: '#/components/parameters/projectId' - responses: - '204': - description: "Successful operation" - '404': - $ref: '#/components/responses/404NotFound' - - # Beginning the Schema Tag - /projects/{projectId}/schemas: - get: - tags: - - Schema - summary: Gets all schemas in a project - description: "If schema url parameter is given, gets the specified schema otherwise gets all schemas" - operationId: getSchemas - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/schemaUrl' - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/SchemaDefinition" - - post: - tags: - - Schema - summary: Creates the given schema in a project - operationId: createSchema - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/format' - requestBody: - required: true - content: - application/json: - schema: - oneOf: - - $ref: "#/components/schemas/Resource" - - $ref: "#/components/schemas/SchemaDefinition" - examples: - SimpleStructureDefinitionExample: - $ref: "#/components/examples/SimpleStructureDefinitionExample" - StructureDefinitionExample: - $ref: "#/components/examples/StructureDefinitionExample" - description: Schema data to be created - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/SchemaDefinition" - - /projects/{projectId}/schemas/{schemaId}: - get: - tags: - - Schema - summary: Gets the specified schema in the project - operationId: getSchema - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/schemaId' - - $ref: '#/components/parameters/format' - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/SchemaDefinition" - '404': - $ref: '#/components/responses/404NotFound' - - put: - tags: - - Schema - summary: Updates the schema with the given body - operationId: putSchema - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/schemaId' - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/SchemaDefinition" - description: Updated schema data - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/SchemaDefinition" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: '#/components/responses/404NotFound' - - delete: - tags: - - Schema - summary: Deletes the specified schema - operationId: deleteSchema - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/schemaId' - responses: - '204': - description: Successful operation - '404': - $ref: '#/components/responses/404NotFound' - - /projects/{projectId}/schemas/infer: - post: - tags: - - Schema - summary: Infers a schema from the given data - operationId: inferSchema - parameters: - - $ref: "#/components/parameters/projectId" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/InferTask" - description: Data for schema inference - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/SchemaDefinition" - '400': - $ref: '#/components/responses/400BadRequest' - - /projects/{projectId}/schemas/redcap: - post: - tags: - - Schema - summary: Gets the REDCap Data Dictionary file - operationId: importREDCapDataDictionary - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/rootUrl' - requestBody: - content: - multipart/form-data: - schema: - type: object - properties: - # 'attachment' will be the field name in this multipart request - attachment: - type: string - format: binary - description: REDCap data dictionary file and root URL - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/SchemaDefinition" - '400': - $ref: '#/components/responses/400BadRequest' - - /projects/{projectId}/schemas/import: - post: - tags: - - Schema - summary: Imports a schema from the specified FHIR server using the provided settings. - operationId: importFhirProfile - parameters: - - $ref: '#/components/parameters/projectId' - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ImportSchemaSettings" - examples: - ImportSchemaSettings: - $ref: '#/components/examples/ImportSchemaSettingsExample' - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/SchemaDefinition" - '400': - $ref: '#/components/responses/400BadRequest' - - /projects/{projectId}/schemas/import-zip: - post: - tags: - - Schema - summary: Import FHIR profiles (structure definitions) from a ZIP file uploaded to the server. - operationId: importFhirProfileFromZip - parameters: - - $ref: '#/components/parameters/projectId' - requestBody: - required: true - content: - multipart/form-data: - schema: - type: object - properties: - file: - type: string - format: binary - description: "The ZIP file containing FHIR structure definitions." - description: "Upload a ZIP file containing FHIR structure definitions for schema import." - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/SchemaDefinition" - - - # Beginning of the Mappings tag - /projects/{projectId}/mappings: - get: - tags: - - Mapping - summary: Gets all mappings in the project - parameters: - - $ref: '#/components/parameters/projectId' - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/FhirMapping" - '400': - $ref: '#/components/responses/400BadRequest' - - post: - tags: - - Mapping - summary: Creates a mapping in the project - parameters: - - $ref: '#/components/parameters/projectId' - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMapping" - description: Fhir mappping to be created - responses: - '201': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMapping" - '400': - $ref: '#/components/responses/400BadRequest' - '415': - $ref: '#/components/responses/415UnsupportedMediaType' - '409': - $ref: '#/components/responses/409AlreadyExists' - - /projects/{projectId}/mappings/{mappingId}: - get: - tags: - - Mapping - summary: Gets the specified mapping - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/mappingId' - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMapping" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: '#/components/responses/404NotFound' - - put: - tags: - - Mapping - summary: Updates the mapping with the given body - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/mappingId' - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMapping" - description: Fhir mappping to update current mapping - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMapping" - '400': - $ref: '#/components/responses/400BadRequest' - '415': - $ref: '#/components/responses/415UnsupportedMediaType' - '404': - $ref: '#/components/responses/404NotFound' - - delete: - tags: - - Mapping - summary: Deletes the specified mapping - operationId: deleteMapping - parameters: - - $ref: '#/components/parameters/projectId' - - $ref: '#/components/parameters/mappingId' - responses: - '204': - description: Successful operation - '404': - $ref: '#/components/responses/404NotFound' - - # Beginning of the Job tag - /projects/{projectId}/jobs: - get: - tags: - - Job - summary: Gets all mapping jobs in the project - parameters: - - $ref: "#/components/parameters/projectId" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/FhirMappingJob" - '400': - $ref: '#/components/responses/400BadRequest' - - post: - tags: - - Job - summary: Creates a new mapping job - parameters: - - $ref: "#/components/parameters/projectId" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMappingJob" - examples: - FhirMappingJobExample: - $ref: "#/components/examples/FhirMappingJobExample" - description: Fhir mappping job to be created - responses: - '201': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMappingJob" - '400': - $ref: '#/components/responses/400BadRequest' - '415': - $ref: '#/components/responses/415UnsupportedMediaType' - - /projects/{projectId}/jobs/{jobId}: - get: - tags: - - Job - summary: Gets the specified mapping job - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMappingJob" - '404': - $ref: '#/components/responses/404NotFound' - - put: - tags: - - Job - summary: Updates the mapping job with the given body - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/FhirMappingJob" - examples: - FhirMappingJobExample: - $ref: "#/components/examples/FhirMappingJobExample" - responses: - '200': - description: Mapping job updated successfully - '404': - $ref: '#/components/responses/404NotFound' - '400': - $ref: '#/components/responses/400BadRequest' - '415': - $ref: '#/components/responses/415UnsupportedMediaType' - - delete: - tags: - - Job - summary: Deletes the mapping job - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - responses: - '204': - description: Mapping job deleted successfully - '400': - description: The running mapping jobs cannot be deleted - '404': - $ref: '#/components/responses/404NotFound' - - /projects/{projectId}/jobs/{jobId}/run: - post: - tags: - - Job - summary: Runs the mapping job - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: '#/components/parameters/jobId' - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ExecuteJobTask" - responses: - '200': - description: Job execution started successfully - '400': - $ref: "#/components/responses/400BadRequest" - - /projects/{projectId}/jobs/{jobId}/status: - get: - tags: - - Job - summary: Checks whether the mapping job is running or not - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: '#/components/parameters/jobId' - responses: - '200': - description: Mapping job is running - '404': - description: Mapping job not found or not running - - /projects/{projectId}/jobs/{jobId}/test: - post: - tags: - - Job - summary: Tests the mapping job with the given configurations - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/TestResourceCreationRequest" - responses: - '200': - description: Test completed successfully - - # Beginning of the Execution tag - /projects/{projectId}/jobs/{jobId}/executions: - get: - tags: - - Execution - summary: Gets executions of the mapping job - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/Execution" - - delete: - tags: - - Execution - summary: Stops all executions of the mapping job - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - responses: - '200': - description: Executions stopped successfully - '404': - $ref: "#/components/responses/404NotFound" - - /projects/{projectId}/jobs/{jobId}/executions/{executionId}/run: - post: - tags: - - Execution - summary: Continues the job execution - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - - $ref: "#/components/parameters/executionId" - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ExecuteJobTask" - responses: - '200': - description: Job execution runned successfully - - /projects/{projectId}/jobs/{jobId}/executions/{executionId}/stop: - delete: - tags: - - Execution - summary: Stops the job execution - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - - $ref: "#/components/parameters/executionId" - responses: - '200': - description: Job execution stopped successfully - - /projects/{projectId}/jobs/{jobId}/executions/{executionId}/deschedule: - delete: - tags: - - Execution - summary: Stops the scheduled job execution - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - - $ref: "#/components/parameters/executionId" - responses: - '200': - description: Job execution descheduled successfull - - /projects/{projectId}/jobs/{jobId}/executions/{executionId}/mappings/{mappingTaskName}/stop: - delete: - tags: - - Execution - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - - $ref: "#/components/parameters/executionId" - - $ref: "#/components/parameters/mappingTaskName" - summary: Stops the individual mapping task inside the job - responses: - '200': - description: Mapping execution stopped successfully - - # Beginning of the Mapping Context tag - /projects/{projectId}/mapping-contexts: - get: - tags: - - Mapping Context - summary: Gets all mapping contexts in the project - parameters: - - $ref: "#/components/parameters/projectId" - responses: - '200': - description: Successful operation - content: - application/json: - example: - ["anesthesia-observations-concept-map.csv", - "anesthesia-procedures-concept-map.csv", - "anesthesia-types-concept-map.csv"] - - post: - tags: - - Mapping Context - summary: Creates a new mapping context - parameters: - - $ref: "#/components/parameters/projectId" - requestBody: - content: - text/plain: - schema: - type: string - example: "anesthesia-types-concept-map.csv" - responses: - '201': - description: Successful operation - content: - application/json: - schema: - type: string - example: "anesthesia-types-concept-map.csv" - '400': - $ref: "#/components/responses/400BadRequest" - - /projects/{projectId}/mapping-contexts/{mappingContextId}: - delete: - tags: - - Mapping Context - summary: Deletes the mapping context - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/mappingContextId" - responses: - '204': - description: No content - '404': - $ref: "#/components/responses/404NotFound" - - /projects/{projectId}/mapping-contexts/{mappingContextId}/content: - get: - tags: - - "Mapping Context" - summary: "Gets the paginated content of a mapping context CSV" - operationId: "getMappingContextContent" - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/mappingContextId" - - in: query - name: page - schema: - type: string - required: false - - in: query - name: size - schema: - type: string - required: false - responses: - '200': - description: "Successful operation" - content: - text/csv: - schema: - type: string - format: binary - example: "source_system,source_code,target_system,target_code,target_display,equivalence\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNE,http://snomed.info/sct,309068002,Specimen from skin,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323008,Pus specimen,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323009,Pus specimen 2,equivalent\nhttp://terminology.hl7.org/CodeSystem/v2-0487,BULLA,http://snomed.info/sct,258482009,Pus specimen 2,narrower" - '404': - $ref: "#/components/responses/404NotFound" - - post: - tags: - - "Mapping Context" - summary: "Updates the paginated content of a mapping context CSV" - operationId: "updateMappingContextContent" - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/mappingContextId" - - in: query - name: page - schema: - type: string - required: false - requestBody: - content: - multipart/form-data: - schema: - type: object - properties: - # 'attachment' will be the field name in this multipart request - attachment: - type: string - format: binary - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - /projects/{projectId}/mapping-contexts/{mappingContextId}/header: - post: - tags: - - "Mapping Context" - summary: "Updates the header of mapping context CSV. This request must include all headers of the csv content. Otherwise, data may be lost." - operationId: "updateMappingContextHeader" - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/mappingContextId" - requestBody: - content: - application/json: - schema: - type: array - items: - type: string - examples: - HeaderExample: - $ref: "#/components/examples/ConceptMapCsvHeadersExample" - responses: - '200': - description: "Successful operation" - content: - text/plain: - schema: - type: string - example: "OK" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: "#/components/responses/404NotFound" - - - /projects/{projectId}/mapping-contexts/{mappingContextId}/file: - post: - tags: - - Mapping Context - summary: Uploads the given mapping context file - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/mappingContextId" - requestBody: - content: - multipart/form-data: - schema: - type: object - properties: - attachment: - type: string - format: binary - responses: - '200': - description: File uploaded successfully - '400': - description: Bad request - - get: - tags: - - Mapping Context - summary: Downloads the mapping context file - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/mappingContextId" - responses: - '200': - description: File downloaded successfully - content: - text/csv: - schema: - type: string - format: binary - example: "source_code,source_system,source_display\n50697003,http://snomed.info/sct,General Anesthesia\n386761002,http://snomed.info/sct,Local Anesthesia\n27372005,http://snomed.info/sct,Regional Anesthesia\n231249005,http://snomed.info/sct,Spinal Anesthesia\n88815003,http://snomed.info/sct,Peripheral block anesthesia\n26402001,http://snomed.info/sct,Nerve block anesthesia\n72641008,http://snomed.info/sct,Sedation" - '404': - $ref: "#/components/responses/404NotFound" - - # Beginning of the Fhir Definition tag - /fhir-definitions: - get: - tags: - - Fhir Definition - summary: Gets FHIR Definitions - parameters: - - $ref: "#/components/parameters/q" - - $ref: "#/components/parameters/rtype" - - $ref: "#/components/parameters/profile" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/SimpleStructureDefinition' - '400': - $ref: "#/components/responses/400BadRequest" - - /validate: - post: - tags: - - Fhir Definition - summary: Validates the FHIR resource whose validation url is given - parameters: - - $ref: "#/components/parameters/fhirValidationUrl" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/Resource" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: object - examples: - FhirValidationExample: - $ref: "#/components/examples/FhirValidationExample" - '400': - description: Missing query parameter - '500': - description: Proxy request failed - - /base-profiles: - get: - tags: - - Fhir Definition - summary: Gets FHIR Definitions - description: If a specific profile is provided as a query parameter, the response will include only that profile's definition. If no profile is specified, the response will contain all base profiles for the given FHIR version. - parameters: - - $ref: "#/components/parameters/fhirVersion" - - $ref: "#/components/parameters/profile" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/SimpleStructureDefinition' - '400': - $ref: "#/components/responses/400BadRequest" - - # Beginning of the Fhir Path Functions tag - /fhir-path-functions: - get: - tags: - - Fhir Path Functions - summary: Returns the documentations of Fhir path functions - parameters: - - $ref: '#/components/parameters/base_path' - - $ref: '#/components/parameters/include_files' - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/FhirPathFunction" - examples: - FhirPathFunctionsExample: - $ref: "#/components/examples/FhirPathFunctionsExample" - - # Beginning of the File System Path tag - /file-system-path: - get: - tags: - - File System Path - summary: Returns a tree structure of the file system, with the root being the context path where Ignifyr runs. - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/FilePathNode" - examples: - FilePathNodeExample: - $ref: "#/components/examples/FilePathNodeExample" - - # Beginning of the metadata tag - /metadata: - get: - tags: - - Metadata - summary: Return metadata information of the server including version, config etc. - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/Metadata" - examples: - MetadataExample: - $ref: "#/components/examples/MetadataExample" - - # Beginning of the REDCap tag - /redcap: - get: - summary: Retrieve REDCap Projects - description: Retrieves the list of REDCap project configurations from the ignifyr-redcap service. - operationId: getRedCapProjects - tags: - - REDCap - responses: - '200': - description: A list of REDCap projects. - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/RedCapProjectConfig' - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: '#/components/responses/404NotFound' - '500': - $ref: '#/components/responses/500InternalServerError' - post: - summary: Save REDCap Projects - description: | - Saves the provided REDCap project configurations to the ignifyr-redcap service. - operationId: saveRedCapProjects - tags: - - REDCap - requestBody: - description: List of REDCap project configurations to save. - required: true - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/RedCapProjectConfig' - responses: - '200': - description: Projects saved successfully. - content: - application/json: - schema: - type: string - example: "Projects saved successfully." - '400': - $ref: '#/components/responses/400BadRequest' - '415': - $ref: '#/components/responses/415UnsupportedMediaType' - '500': - $ref: '#/components/responses/500InternalServerError' - /redcap/{projectId}: - delete: - summary: Delete REDCap data of the project - description: | - Deletes the REDCap data of the project from the Kafka topics - operationId: deleteRedCapData - tags: - - REDCap - parameters: - - $ref: "#/components/parameters/projectId" - - in: query - name: "reload" - schema: - type: boolean - description: "Whether to reload REDCap records into to Kafka topics or not" - responses: - '200': - description: Project's data deleted successfully. - content: - application/json: - schema: - type: string - example: "Project's data deleted successfully." - /redcap/notification: - get: - summary: Retrieve REDCap Notification URL - description: | - Fetches the notification URL used by the REDCap project. The source of this URL is configured in the ignifyr-redcap service. - - operationId: getNotificationUrl - tags: - - REDCap - responses: - '200': - description: Notification URL returned successfully. - content: - text/plain: - schema: - type: string - example: "https://example.com/redcap/notification" - '400': - $ref: '#/components/responses/400BadRequest' - '404': - $ref: '#/components/responses/404NotFound' - '500': - $ref: '#/components/responses/500InternalServerError' - - -components: - # Type definitions for the objects used - schemas: - TerminologySystem: - type: object - required: - - name - - description - properties: - id: - type: string - description: "Identifier of the terminology system" - name: - type: string - description: "Name of the terminology system" - example: "T-System Example" - description: - type: string - description: "Description of the terminology system" - example: "SNOMED CT is a terminology system for clinical health care" - conceptMaps: - type: array - items: - $ref: "#/components/schemas/TerminologyConceptMap" - description: "URL of the terminology system" - codeSystems: - type: array - items: - $ref: "#/components/schemas/TerminologyCodeSystem" - description: "Version of the terminology system" - - TerminologyConceptMap: - type: object - required: - - name - - conceptMapUrl - - sourceValueSetUrl - - targetValueSetUrl - properties: - id: - type: string - description: "Identifier of the terminology concept map" - pattern: "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" - name: - type: string - description: "Name of the terminology concept map" - example: "CoonceptMap" - conceptMapUrl: - type: string - description: "URL of the terminology concept map" - example: "http://example.com/fhir/ConceptMap/sample1" - sourceValueSetUrl: - type: string - description: "URL of the source value set" - example: "http://terminology.hl7.org/ValueSet/v2-0487" - targetValueSetUrl: - type: string - description: "URL of the target value set" - example: "http://snomed.info/sct?fhir_vs" - - TerminologyCodeSystem: - type: object - required: - - name - - codeSystemUrl - - valueSetUrl - properties: - id: - type: string - description: "Identifier of the terminology code system" - pattern: "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" - name: - type: string - description: "Name of the terminology code system" - example: "CodeSystem" - codeSystem: - type: string - description: "Code system of the terminology code system" - example: "http://snomed.info/sct" - - Project: - type: "object" - required: - - id - - name - properties: - id: - type: string - description: "Unique identifier for the project. First given name to the project is used as id." - example: "projectName" - name: - type: string - description: "Name of the project." - example: "ProjectName" - description: - type: string - description: "A detailed description of the project." - example: "This project is aimed at ..." - schemaUrlPrefix: - type: string - description: "Prefix (beginning) of the URLs to be used while creating schema definitions within this project." - example: "https://aiccelerate.eu/fhir/StructureDefinition/" - mappingUrlPrefix: - type: string - description: "Prefix (beginning) of the URLs to be used while creating mapping definitions within this project." - example: "https://aiccelerate.eu/fhir/mappings/pilot1/" - schemas: - type: array - description: "List of schema definitions associated with the project." - items: - $ref: "#/components/schemas/SchemaDefinition" - example: [] - mappings: - type: array - description: "List of mapping definitions associated with the project." - items: - $ref: "#/components/schemas/FhirMapping" - example: [] - mappingContext: - type: array - description: "List of identifiers for mapping contexts defined in this project." - items: - type: string - example: [] - mappingJobs: - type: array - description: "List of mapping jobs defined in this project." - items: - $ref: "#/components/schemas/FhirMappingJob" - example: [] - - SchemaDefinition: - type: "object" - required: - - "url" - - "type" - - "name" - properties: - id: - type: "string" - description: "Identifier of the schema" - pattern: "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" - url: - type: "string" - description: "Url of the schema" - example: "https://aiccelerate.eu/fhir/StructureDefinition/Schema-id" - version: - description: "Major FHIR version (must be either R4 or R5)" - type: string - enum: - - R4 - - R5 - type: - type: "string" - description: "Type of the schema" - example: "Schema-id" - description: - type: "string" - description: "Description of the schema" - example: "Sample description" - name: - type: "string" - description: "Name of the schema" - example: "Schema-id" - rootDefinition: - example: {} - - fieldDefinitions: - type: array - items: - $ref: '#/components/schemas/SimpleStructureDefinition' - example: - $ref: '#/components/examples/SchemaFieldDefinitionsExample' - - SimpleStructureDefinition: - type: "object" - required: - - "id" - - "path" - - "isPrimitive" - - "isChoiceRoot" - - "isArray" - - "minCardinality" - properties: - id: - type: string - path: - type: string - version: - type: string - enum: - - R4 - - R5 - dataTypes: - type: array - items: - $ref: '#/components/schemas/DataTypeWithProfiles' - isPrimitive: - type: boolean - isChoiceRoot: - type: boolean - isArray: - type: boolean - minCardinality: - type: integer - maxCardinality: - type: integer - boundToValueSet: - type: string - isValueSetBindingRequired: - type: boolean - referencableProfiles: - type: array - items: - type: string - constraintDefinitions: - type: array - items: - $ref: '#/components/schemas/ConstraintDefinition' - sliceDefinition: - $ref: '#/components/schemas/SliceDefinition' - sliceName: - type: string - fixedValue: - type: string - patternValue: - type: string - referringTo: - type: string - short: - type: string - definition: - type: string - comment: - type: string - elements: - type: array - items: - $ref: '#/components/schemas/SimpleStructureDefinition' - - DataTypeWithProfiles: - type: "object" - required: - - "dataType" - properties: - dataType: - type: "string" - profiles: - type: array - items: - type: string - - ConstraintDefinition: - type: object - required: - - key - - desc - - isWarning - properties: - key: - type: string - desc: - type: string - isWarning: - type: boolean - - SliceDefinition: - type: object - required: - - discriminators - - ordered - - rule - properties: - discriminators: - type: array - items: - $ref: '#/components/schemas/SliceDiscriminator' - ordered: - type: boolean - rule: - type: string - - SliceDiscriminator: - type: object - required: - - type - - path - properties: - type: - type: string - path: - type: string - - Resource: - type: object - required: - - url - - type - properties: - id: - type: "string" - description: "Identifier of the schema" - pattern: "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" - url: - type: "string" - description: "Url of the schema" - example: "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-anesthesia-observations" - resourceType: - type: "string" - description: "Resource type of the schema" - example: "Ext-plt1-anesthesia-observations" - name: - type: "string" - description: "Name of the schema" - example: "Ext-plt1-anesthesia-observations" - status: - type: "string" - description: "Publication status of the StructureDefinition" - example: "draft" - fhirVersion: - type: "string" - description: "version of the FHIR standart" - example: "4.0.1" - kind: - type: "string" - description: "Kind of the Structure Definition" - example: "logical" - abstract: - type: "boolean" - description: "Indicates whether this StructureDefinition represents an abstract type" - example: false - baseDefinition: - type: "string" - description: "Specifies the base or foundational definition upon which this StructureDefinition is built" - example: "http://hl7.org/fhir/StructureDefinition/Element" - derivation: - type: "string" - description: "The method by which this StructureDefinition is derived" - example: "specialization" - differential: - properties: - element: - type: "array" - items: - $ref: "#/components/schemas/ElementDefinition" - - ElementDefinition: - type: object - description: "The definition of an element in a resource or an extension" - properties: - id: - type: string - path: - type: string - min: - type: string - max: - type: string - definition: - type: string - short: - type: string - type: - type: array - items: - type: object - properties: - code: - type: string - profile: - type: array - items: - type: string - - InferTask: - type: object - properties: - name: - type: string - description: "Name for source binding" - example: "source" - mappingJobSourceSettings: - type: object - properties: - source: - oneOf: - - $ref: '#/components/schemas/FileSystemSourceSettings' - - $ref: '#/components/schemas/SqlSourceSettings' - sourceBinding: - oneOf: - - $ref: "#/components/schemas/FileSystemSource" - - $ref: "#/components/schemas/SqlSource" - required: - - name - - sourceSettings - - mappingJobSourceSettings - - SourceSettings: - type: object - description: Array including data sources of the job - additionalProperties: - oneOf: - - $ref: "#/components/schemas/FileSystemSourceSettings" - - $ref: "#/components/schemas/SqlSourceSettings" - - $ref: "#/components/schemas/KafkaSourceSettings" - - $ref: "#/components/schemas/FhirServerSourceSettings" - - MappingJobSourceSettings: - type: object - description: Interface for data source settings/configurations of a mapping job - properties: - name: - type: string - description: Human friendly name for the source organization for data source - example: "source" - sourceUri: - type: string - description: Computer friendly canonical url indicating the source of the data (May be used for Resource.meta.source) - example: "https://aiccelerate.eu/data-integration-suite/test-data" - asStream: - type: boolean - description: Whether the data is coming in streaming mode - example: false - jsonClass: - type: string - description: "Indicates the type of the data source (ex: FileSystemSourceSettings)" - enum: - - FileSystemSourceSettings - - SqlSourceSettings - - KafkaSourceSettings - - FhirServerSourceSettings - - FileSystemSourceSettings: - description: "Context/configuration for the source of the mapping-job that will read the source data from a file system" - allOf: - - $ref: '#/components/schemas/MappingJobSourceSettings' - - type: object - properties: - dataFolderPath: - type: string - description: "The data folder path of the file source" - example: "test-data/pilot1" - required: - - dataFolderPath - SqlSourceSettings: - description: "Context/configuration for the source of the mapping-job that will read the source data from a SQL source" - allOf: - - $ref: '#/components/schemas/MappingJobSourceSettings' - - type: object - properties: - databaseUrl: - type: string - description: "Connection URL of the SQL database" - example: "jdbc:postgresql://localhost:5432/db_name" - username: - type: string - description: "Username of the SQL database" - example: "postgres" - password: - type: string - description: "Password of the SQL database" - example: "postgres" - required: - - databaseUrl - - username - - password - KafkaSourceSettings: - description: "Context/configuration for the source of the mapping-job that will read the source data from Kafka as a stream" - allOf: - - $ref: '#/components/schemas/MappingJobSourceSettings' - - type: object - properties: - bootstrapServers: - type: string - description: "Kafka bootstrap server(s) with port, may be comma-separated list (localhost:9092,localhost:9091)" - required: - - bootstrapServers - FhirServerSourceSettings: - description: "Context/configuration for the source of the mapping-job that will read the source data from a FHIR server" - allOf: - - $ref: '#/components/schemas/MappingJobSourceSettings' - - type: object - properties: - serverUrl: - type: string - description: "URL of the FHIR server" - example: "https://fhir.example.com" - securitySettings: - oneOf: - - $ref: "#/components/schemas/BearerTokenAuthorizationSettings" - - $ref: "#/components/schemas/BasicAuthenticationSettings" - - $ref: "#/components/schemas/FixedTokenAuthenticationSettings" - required: - - serverUrl - - MappingSourceBinding: - type: object - properties: - jsonClass: - type: string - enum: - - FileSystemSource - - SqlSource - - KafkaSource - - FhirServerSource - description: "Indicates the type of the mapping (e.g., FileSystemSource, SqlSource, KafkaSource)" - preprocessSql: - type: string - description: "Indicates preprocess SQL query" - example: "null" - sourceRef: - type: string - description: "Reference to the source specified in the sourceSettings of a mapping job." - example: "source1" - required: - - jsonClass - - preprocessSql - - sourceRef - - FileSystemSource: - description: "Defines file system source configurations of a mapping in the mapping-job" - allOf: - - $ref: '#/components/schemas/MappingSourceBinding' - - type: object - properties: - path: - type: string - description: "File path to the source file or folder, e.g., 'patients.csv' or 'patients'." - example: "patients.csv" - contentType: - type: string - enum: - - csv - - tsv - - parquet - - json - description: "Content type of configured file inside of the data folder" - options: - $ref: "#/components/schemas/SparkOptions" - required: - - path - - SqlSource: - description: "Defines SQL source configurations of a mapping in the mapping-job" - allOf: - - $ref: '#/components/schemas/MappingSourceBinding' - - type: object - properties: - tableName: - type: string - description: "Table name of the SQL source" - example: "table1" - query: - type: string - description: "Query to execute in the database" - example: "SELECT * FROM table1" - options: - $ref: "#/components/schemas/SparkOptions" - required: - - tableName - - query - - KafkaSource: - description: "Defines Kafka source configurations of a mapping in the mapping-job" - allOf: - - $ref: '#/components/schemas/MappingSourceBinding' - - type: object - properties: - topicName: - type: string - description: "Topic(s) to subscribe to, may be comma-separated string list (topic1,topic2)" - options: - $ref: "#/components/schemas/SparkOptions" - required: - - topicName - - FhirServerSource: - description: "Defines FHIR server source configurations of a mapping in the mapping-job" - allOf: - - $ref: '#/components/schemas/MappingSourceBinding' - - type: object - properties: - resourceType: - type: string - description: "Type of FHIR resource to query." - example: "Patient" - query: - type: string - description: "Optional query string to filter the FHIR resources." - example: "gender=male" - required: - - resourceType - - FhirMapping: - type: object - properties: - id: - type: string - description: Unique identifier for the mapping - example: "anesthesia-observations-mapping" - url: - type: string - description: Canonical URL for the mapping - example: "https://aiccelerate.eu/fhir/mappings/pilot1/anesthesia-observations-mapping" - name: - type: string - description: Computer-friendly name - example: "anesthesia-observations-mapping" - title: - type: string - description: Human-friendly title - example: "Mapping of schema anesthesia observations symptoms to AIC-AnesthesiaObservation profile resources" - description: - type: string - description: Description of the mapping - example: "" - source: - type: array - items: - $ref: "#/components/schemas/FhirMappingSource" - description: Metadata about source for the mapping - variable: - type: array - items: - $ref: "#/components/schemas/FhirExpression" - description: "Common variables calculated from source data to use in the mappings" - mapping: - type: array - items: - $ref: "#/components/schemas/FhirMappingExpression" - description: Mapping scripts - context: - type: object - additionalProperties: - $ref: "#/components/schemas/FhirMappingContextDefinition" - description: "Further context to use for mapping evaluation e.g., ConceptMap for terminology mapping, definition of unit conversion functions" - required: - - id - - url - - name - - FhirMappingExpression: - type: object - properties: - description: - type: string - example: Sample description - precondition: - allOf: - - $ref: "#/components/schemas/FhirExpression" - description: A precondition FHIR expression for this mapping - expression: - allOf: - - $ref: "#/components/schemas/FhirExpression" - description: FHIR expression that defines the mappinge - example: - $ref: "#/components/examples/FhirExpressionExample/value" - fhirInteraction: - allOf: - - $ref: "#/components/schemas/FhirInteraction" - description: "Provides information about the FHIR Interaction to persist the mapped information. If not given FHIR Update interaction is used and the mapped content is expected to be a FHIR resource." - required: - - expression - - FhirExpression: - type: object - properties: - name: - type: string - description: Name of the expression - example: "encType" - language: - type: string - description: Language of the expression - example: "text/fhirpath" - expression: - type: string - description: Expression string - example: "type.utl:split('-').first()" - reference: - type: string - description: Reference to the expression - value: - type: object - description: Defined value of the expression - required: - - name - - language - - FhirMappingSource: - type: object - properties: - alias: - type: string - description: "Name of the source to be used in expressions" - example: "source" - url: - type: string - description: "URL to the StructureDefinition of the source format" - example: "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-anesthesia-observations" - joinOn: - type: array - items: - type: string - description: "Columns to use from this source data while joining the multiple sources" - description: - type: string - description: "Description of the source" - required: - - alias - - url - - FhirMappingContextDefinition: - type: object - properties: - category: - type: string - description: Category of context - example: "concept-map" - url: - type: string - description: URL if context data will be loaded from a URL - example: "$CONTEXT_REPO/pilot1/anesthesia-observations-concept-map.csv" - value: - type: object - description: Value if context data will be supplied as JSON content - description: - type: string - description: Description of the context data - required: - - category - - FhirInteraction: - type: object - description: "Provides the details about FHIR interaction to persist the new information" - properties: - type: - type: string - description: "FHIR interaction type e.g. create | update | patch" - example: "create" - rid: - type: string - description: "Only required for FHIR Patch and provides either FHIR resource locator" - example: "Observation/315653" - condition: - type: string - description: "FHIR search statement indicating the condition to update or create" - required: - - type - - FhirMappingJob: - type: object - properties: - id: - type: string - description: "Unique identifier for the mapping job" - example: "pilot-example" - name: - type: string - description: "The name given by the user" - example: "Pilot-example" - sourceSettings: - $ref: "#/components/schemas/SourceSettings" - sinkSettings: - $ref: "#/components/schemas/FhirSinkSettings" - terminologyServiceSettings: - oneOf: - - $ref: "#/components/schemas/LocalFhirTerminologyServiceSettings" - - $ref: "#/components/schemas/FhirRepositorySinkSettings" - identityServiceSettings: - $ref: "#/components/schemas/FhirRepositorySinkSettings" - mappings: - type: array - items: - $ref: "#/components/schemas/MappingTask" - schedulingSettings: - $ref: "#/components/schemas/SchedulingSettings" - dataProcessingSettings: - $ref: "#/components/schemas/DataProcessingSettings" - useFhirSinkAsIdentityService: - type: boolean - description: If true it means the FHIR repository to write the mapped resources where the configuration is given in sink settings will be used as identity service (Override identityServiceSettings if given) - example: false - required: - - id - - sinkSettings - - sourceSettings - - mappings - - FhirSinkSettings: - type: object - description: "Common interface for sink settings" - oneOf: - - $ref: "#/components/schemas/FileSystemSinkSettings" - - $ref: "#/components/schemas/FhirRepositorySinkSettings" - - FileSystemSinkSettings: - type: object - description: "Settings to write mapped FHIR resources to file system" - properties: - jsonClass: - type: string - example: "FileSystemSinkSettings" - path: - type: string - description: "Path to the folder or file to write the resources" - example: "/path/to/folder" - contentType: - type: string - description: "Content type of the sink" - example: "ndjson" - numOfPartitions: - type: integer - description: "Number of partitions for the file (for distributed fs)" - options: - $ref: "#/components/schemas/SparkOptions" - partitionByResourceType: - type: boolean - description: "Flag to determine whether to partition the output files by FHIR resource type." - partitioningColumns: - type: object - description: "A map of resource types to a list of columns to partition by" - additionalProperties: - type: array - items: - type: string - required: - - jsonClass - - path - - FhirRepositorySinkSettings: - type: object - description: "Settings for a FHIR repository to store the mapped resources" - properties: - jsonClass: - type: string - example: "FhirRepositorySinkSettings" - fhirRepoUrl: - type: string - description: "Path to the folder or file to write the resources" - example: "http://localhost:8081/fhir" - securitySettings: - type: object - description: "File format if not inferred from the path" - oneOf: - - $ref: "#/components/schemas/BearerTokenAuthorizationSettings" - - $ref: "#/components/schemas/BasicAuthenticationSettings" - - $ref: "#/components/schemas/FixedTokenAuthenticationSettings" - returnMinimal: - type: boolean - description: "Number of partitions for the file (for distributed fs)" - required: - - jsonClass - - fhirRepoUrl - - BearerTokenAuthorizationSettings: - type: object - description: "Security settings for FHIR API access via bearer token" - properties: - jsonClass: - type: string - description: "Type of the security setting choice" - example: "BearerTokenAuthorizationSettings" - clientId: - type: string - description: "OpenID Client identifier assigned to Ignifyr" - clientSecret: - type: string - description: "OpenID Client secret given to Ignifyr" - requiredScopes: - type: array - items: - type: string - description: "List of required scores to write the resources" - authzServerTokenEndpoint: - type: string - description: "Authorization servers token endpoint" - clientAuthenticationMethod: - type: string - description: "Client authentication method" - required: - - clientId - - clientSecret - - requiredScopes - - authzServerTokenEndpoint - - BasicAuthenticationSettings: - type: object - description: "Security settings for FHIR API access via basic authentication" - properties: - jsonClass: - type: string - description: "Type of the security setting choice" - username: - type: string - description: "Username for basic authentication" - password: - type: string - description: "Password for basic authentication" - required: - - username - - password - - FixedTokenAuthenticationSettings: - type: object - description: "Security settings for FHIR API access via fixed token" - properties: - jsonClass: - type: string - description: "Type of the security setting choice" - token: - type: string - description: "The fixed token" - required: - - token - - ImportSchemaSettings: - type: object - properties: - baseUrl: - type: string - description: "The base URL of the FHIR server from which to import the schema." - example: "https://fhir.server.com" - resourceId: - type: string - description: "The ID of the resource representing the schema i.e. StructureDefinition to be imported." - securitySettings: - $ref: '#/components/schemas/FhirRepositorySinkSettings' - required: - - baseUrl - - resourceId - - LocalFhirTerminologyServiceSettings: - type: object - description: "A local terminology service where each concept map is provided via a CSV file" - properties: - jsonClass: - type: string - description: "Type of the terminology service, (remote or local)" - example: "LocalFhirTerminologyServiceSettings" - folderPath: - type: string - description: "Path to the folder that all concept map files exists" - example: "D:/ignifyr/data" - conceptMapFiles: - type: array - items: - $ref: '#/components/schemas/ConceptMapFile' - codeSystemFiles: - type: array - items: - $ref: '#/components/schemas/CodeSystemFile' - required: - - folderPath - - ConceptMapFile: - type: object - description: "Metadata for a Concept map file for Ignifyr local terminology service" - properties: - id: - type: string - description: "unique id" - name: - type: string - description: "Name of the file" - conceptMapUrl: - type: string - description: "Corresponding concept map url e.g. http://cds-hooks.hl7.org/ConceptMap/indicator-to-request-priority" - sourceValueSetUrl: - type: string - description: "Url for Source value set" - targetValueSetUrl: - type: string - description: "Url for Target value set" - required: - - name - - conceptMapUrl - - sourceValueSetUrl - - targetValueSetUrl - - CodeSystemFile: - type: object - description: "Metadata for a code system file for Ignifyr local terminology service" - properties: - id: - type: string - description: "unique id" - name: - type: string - description: "Name of the file" - codeSystem: - type: string - description: "Corresponding code system url" - required: - - id - - name - - codeSystem - - MappingTask: - description: "FHIR Mapping task instance" - type: object - properties: - name: - type: string - description: "Name of the given FhirMappingTask" - example: "example" - mappingRef: - type: string - description: "Canonical URL of the FhirMapping definition to execute" - example: "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping" - sourceBinding: - type: object - description: "Provide details how to load each source data for the mapping" - additionalProperties: - oneOf: - - $ref: "#/components/schemas/FileSystemSource" - - $ref: "#/components/schemas/SqlSource" - - $ref: "#/components/schemas/KafkaSource" - - $ref: "#/components/schemas/FhirServerSource" - batchingStrategy: - $ref: "#/components/schemas/BatchingStrategy" - - BatchingStrategy: - description: "Batching strategy for processing source data in multiple batches based on custom parameters" - type: object - properties: - batchParameterSets: - type: array - description: "Sequence of parameter sets, where each set represents one batch. Each parameter set is a map of (parameterName -> parameterValue). All parameters in a set will be substituted in preprocessSql as $parameterName." - items: - type: object - additionalProperties: - type: string - example: - - year: "2020" - month: "1" - - year: "2020" - month: "2" - - year: "2021" - month: "1" - required: - - batchParameterSets - - SchedulingSettings: - type: object - description: "Cron expression showing the times of scheduled task needed. More info: https://www.sauronsoftware.it/projects/cron4j/" - properties: - cronExpression: - type: string - description: "A UNIX crontab-like pattern is a string split in five space separated parts" - jsonClass: - type: string - description: "Specifies the class of this object." - example: "SchedulingSettings" - required: - - cronExpression - - DataProcessingSettings: - type: object - description: "Data source processing settings" - properties: - saveErroneousRecords: - type: boolean - description: "If true, erroneous records will be saved to archive folder with the same path as the input file" - archiveMode: - $ref: "#/components/schemas/ArchiveModes" - required: - - saveErroneousRecords - - archiveMode - - ArchiveModes: - type: string - enum: - - off - - delete - - archive - - ExecuteJobTask: - type: object - description: "The data transfer object to keep mappingTask names to be executed and corresponding mapping error handling." - properties: - clearCheckpoints: - type: boolean - mappingTaskNames: - description: "mapping names of the mappingTask's that to be executed" - type: array - items: - type: string - - Execution: - type: object - description: Executions of a mapping job - properties: - id: - type: string - description: "Identifier of the execution" - example: "c531d5b0-40fc-42a9-8f2d-c6105e3a5d39" - oneOf: - - properties: - runningStatus: - type: boolean - description: "Indicates whether the job is currently running" - example: true - - properties: - scheduled: - type: boolean - description: "Indicates whether the job is scheduled" - example: true - - TestResourceCreationRequest: - type: object - description: "FhirMappingTask and resource filter to test a mapping" - properties: - fhirMappingTask: - allOf: - - $ref: "#/components/schemas/MappingTask" - - type: object - properties: - mapping: - $ref: "#/components/schemas/FhirMapping" - description: "task to be used for testing" - resourceFilter: - $ref: "#/components/schemas/ResourceFilter" - - ResourceFilter: - description: "Resource filter config for testing a mapping" - properties: - numberOfRows: - type: integer - description: "number of rows to select" - example: 10 - order: - type: string - description: "order of rows to select" - example: "start" - - FhirPathFunction: - type: object - description: "the documentations of FhirPath functions" - properties: - documentation: - type: string - insertText: - type: string - detail: - type: string - label: - type: string - kind: - type: string - returnType: - type: array - items: - type: string - inputType: - type: array - items: - type: string - - FilePathNode: - type: object - description: "Represent each file/folder in the file system, with the root being the context path where Ignifyr runs." - properties: - label: - type: string - isFolder: - type: boolean - children: - type: array - items: - $ref: '#/components/schemas/FilePathNode' - - Metadata: - type: object - description: "Model that represents the metadata of the server." - properties: - name: - type: string - description: "The name of the server." - description: - type: string - description: "A description of the server." - version: - type: string - description: "The current version of the server." - majorFhirVersion: - type: string - description: "The major FHIR version of definitions (profiles, valuesets, codesystems) supported by the server." - ignifyrRedcapVersion: - type: string - description: "The optional Ignifyr-Redcap server version." - definitionsRootUrls: - type: array - items: - type: string - description: "Optional list of root URLs for definitions." - schemasFhirVersion: - type: string - description: "The FHIR version used for schemas." - repositoryNames: - type: object - additionalProperties: - $ref: '#/components/schemas/RepositoryNames' - description: "The configured repository names." - archiving: - type: object - additionalProperties: - $ref: '#/components/schemas/Archiving' - description: "The archiving configuration." - - - RepositoryNames: - type: object - description: "Represents the configured repository names included in the metadata." - properties: - mappings: - type: string - description: "The path to the folder where the mappings are kept." - schemas: - type: string - description: "The path to the folder where the schema definitions are kept." - contexts: - type: string - description: "The path to the folder where the mapping context files are kept." - jobs: - type: string - description: "The path to the folder where the job definitions are kept." - terminologySystems: - type: string - description: "The path to the folder where the terminology system definitions are kept." - - Archiving: - type: object - description: "Represents the archiving configurations included in the metadata." - properties: - erroneousRecordsFolder: - type: string - description: "The parent folder where the data sources of errors received while running mapping are stored." - archiveFolder: - type: string - description: "The folder path where the archive of the processed source data is stored." - streamArchivingFrequency: - type: number - description: "The period (in milliseconds) to run the archiving task for file streaming jobs." - - CsvHeader: - type: object - description: Header of a csv column - properties: - currentName: - type: string - description: "The current name of the header." - example: "patient-id" - previousName: - type: string - description: "The previously saved name of the header, used for column name change without data loss" - example: "id" - - # Fields that is used while editing a project - PatchProjectSchema: - type: object - properties: - description: - type: string - description: "A detailed description of the project." - example: "This project is aimed at ..." - schemaUrlPrefix: - type: string - description: "Prefix (beginning) of the URLs to be used while creating schema definitions within this project." - example: "https://aiccelerate.eu/fhir/StructureDefinition/" - mappingUrlPrefix: - type: string - description: "Prefix (beginning) of the URLs to be used while creating mapping definitions within this project." - example: "https://aiccelerate.eu/fhir/mappings/pilot1/" - - SparkOptions: - type: object - description: Options used by Spark while reading data from File system, SQL or Kafka - additionalProperties: - type: string - - RedCapProjectConfig: - type: object - properties: - id: - type: string - description: The unique identifier of the REDCap project. - token: - type: string - description: The API token used for authentication with REDCap. - required: - - id - - token - - # HTTP responses for requests - responses: - 400BadRequest: - description: "Bad or Invalid request" - 409AlreadyExists: - description: "Given object already exists" - 404NotFound: - description: "Given resource does not exist" - 415UnsupportedMediaType: - description: "Given resource is not a valid application/json instance" - 500InternalServerError: - description: "An unexpected error occurred on the server." - - # Path and query parameters for requests - parameters: - projectId: - in: path - name: "projectId" - required: true - schema: - type: string - description: "Identifier of the project" - - schemaId: - in: path - name: "schemaId" - required: true - schema: - type: string - description: "Identifier of the schema" - - format: - in: query - name: "format" - schema: - type: string - enum: [ "StructureDefinition", "SimpleStructureDefinition" ] - description: "Requested format of the schema" - - schemaUrl: - in: query - name: url - schema: - type: string - format: url - description: "Url of the schema" - example: "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-anesthesia-observations" - - rootUrl: - in: query - name: "rootUrl" - schema: - type: string - format: url - required: true - description: "The root url of schemas to be created" - - mappingId: - in: path - name: "mappingId" - schema: - type: string - required: true - description: "Identifier of the mapping" - - jobId: - in: path - name: "jobId" - schema: - type: string - required: true - description: "Identifier of the job" - - executionId: - name: executionId - in: path - required: true - schema: - type: string - - mappingTaskName: - in: path - name: "mappingTaskName" - schema: - type: string - format: url - required: true - description: "Name of a mappingTask" - - mappingContextId: - in: path - name: "mappingContextId" - schema: - type: string - format: file name - required: true - description: "Identifier of the mapping context" - - terminologyId: - in: path - name: "terminologyId" - required: true - schema: - type: string - description: "Identifier of the terminology" - - conceptMapId: - in: path - name: "conceptMapId" - required: true - schema: - type: string - description: "Identifier of the concept map" - - codeSystemId: - in: path - name: "codeSystemId" - required: true - schema: - type: string - description: "Identifier of the code system" - - q: - name: q - in: query - required: true - description: Query parameter - schema: - type: string - enum: - - rtypes - - profiles - - elements - - rtype: - name: rtype - in: query - description: Resource type (required for profiles query) - schema: - type: string - - profile: - name: profile - in: query - description: Profile URL (required for elements query) - schema: - type: string - - fhirValidationUrl: - name: fhirValidationUrl - in: query - required: true - description: FHIR validation URL - schema: - type: string - - base_path: - name: base_path - in: query - required: false - description: The base path of the file system from which to retrieve the folder tree structure. Defaults to the root ("/"). - schema: - type: string - default: "/" - - fhirVersion: - name: fhirVersion - in: query - required: true - description: Major FHIR version (must be either R4 or R5) - schema: - type: string - enum: - - R4 - - R5 - - include_files: - name: include_files - in: query - required: false - description: Boolean flag indicating whether to include files in the folder tree structure. Defaults to `false`. - schema: - type: boolean - default: false - - # Examples to use in the swagger api - examples: - # Example for FhirMappingExpression structure - FhirExpressionExample: - value: - name: "result" - language: "en" - value: - '{{#ind}}': '{{utl:evaluateExpression(utl:indices(1, 65).select(''iif(v'' & $this.toString() & ''.exists(), '' & $this.toString() & '', {})'').mkString('' | '')).select(''v'' & $this.toString())}}' - '{{*}}': - resourceType: Observation - id: '{{mpp:getHashedId(''Observation'', pid & %ind & time.toString())}}' - meta: - profile: - - https://aiccelerate.eu/fhir/StructureDefinition/AIC-AnesthesiaObservation - source: '{{%sourceSystem.sourceUri}}' - status: final - code: - coding: - - system: '{{mpp:getConcept(%anstObsConceptMap, %ind, ''target_system'')}}' - code: '{{mpp:getConcept(%anstObsConceptMap, %ind, ''target_code'')}}' - display: '{{mpp:getConcept(%anstObsConceptMap, %ind, ''target_display'')}}' - subject: '{{mpp:createFhirReferenceWithHashedId(''Patient'', pid)}}' - encounter: '{{? mpp:createFhirReferenceWithHashedId(''Encounter'', encounterId)}}' - effectiveDateTime: '{{time}}' - valueQuantity: - value: '{{utl:evaluateExpression(%ind)}}' - unit: '{{mpp:getConcept(%anstObsConceptMap, %ind, ''target_unit'')}}' - system: http://unitsofmeasure.org - code: '{{mpp:getConcept(%anstObsConceptMap, %ind, ''target_unit'')}}' - - # Example response for the path: POST, /projects/{projectId}/jobs/{jobId}/test - FhirValidationExample: - value: - resourceType: OperationOutcome - issue: - - severity: information - code: informational - diagnostics: All OK :) - expression: [] - - severity: warning - code: invalid - diagnostics: '[Validating against ''https://aiccelerate.eu/fhir/StructureDefinition/AIC-AnesthesiaObservation''] => Constraint ''dom-6'' is not satisfied for the given value! Constraint Description: ''A resource should have narrative for robust management''. FHIR Path expression: ''text.`div`.exists()''' - expression: - - $this - - # Example response for the path: GET, /fhir-path-functions - FhirPathFunctionsExample: - value: - - documentation: Generates an UUID - insertText: utl:uuid() - detail: utl - label: utl:uuid - kind: 1 - returnType: - - string - inputType: [] - - documentation: 'Returns the input string with all instances of pattern replaced with substitution. Ex: ''abcdefg''.replace(''cde'', ''123'')' - insertText: replace(,) - detail: '' - label: replace - kind: 0 - returnType: - - string - inputType: - - string - - # Example response for the path: GET, /file-system-path - FilePathNodeExample: - value: - label: test-data - isFolder: true - children: - - label: icrc - isFolder: true - children: - - label: encounters.csv - isFolder: false - children: [] - - label: patients.csv - isFolder: false - children: [] - - label: ucl - isFolder: true - children: - - label: OBSERVATION.csv - isFolder: false - children: [] - - label: PERSON.csv - isFolder: false - children: [] - - # Example response for the path: GET, /metadata - MetadataExample: - value: - name: Ignifyr - description: Ignifyr is a tool for mapping data from various sources to FHIR resources. - version: 1.1-SNAPSHOT - ignifyrRedcapVersion: 1.1-SNAPSHOT - majorFhirVersion: R5 - definitionsRootUrls: - - https://datatools4heart.eu - schemasFhirVersion: 5.0.0 - repositoryNames: - mappings: mappings - schemas: schemas - contexts: mapping-contexts - jobs: mapping-jobs - terminologySystems: terminology-systems - archiving: - erroneousRecordsFolder: erroneous-records-folder - archiveFolder: archive-folder - streamArchivingFrequency: 5000 - - # A simple structure definition example for schema post - SimpleStructureDefinitionExample: - value: - id: Ext-plt1-condition-test - url: https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-condition-test - type: Ext-plt1-condition-test - name: Ext-plt1-condition-test - version: R4 - rootDefinition: - id: Ext-plt1-condition-test - path: Ext-plt1-condition-test - dataTypes: - - dataType: Element - isPrimitive: false - isChoiceRoot: false - isArray: false - minCardinality: 0 - fieldDefinitions: - - id: pid - path: Ext-plt1-condition-test.pid - dataTypes: - - dataType: id - profiles: - - http://hl7.org/fhir/StructureDefinition/id - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 1 - maxCardinality: 1 - short: Patient identifier - definition: Unique patient identifier (a pseudoynm) - - id: code - path: Ext-plt1-condition-test.code - dataTypes: - - dataType: code - profiles: - - http://hl7.org/fhir/StructureDefinition/code - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 1 - maxCardinality: 1 - short: Condition code - definition: ICD-10 code for diagnosis - - id: codeDisplay - path: Ext-plt1-condition-test.codeDisplay - dataTypes: - - dataType: string - profiles: - - http://hl7.org/fhir/StructureDefinition/string - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 0 - maxCardinality: 1 - short: Condition name - definition: Name of the diagnosis - - id: onsetDateTime - path: Ext-plt1-condition-test.onsetDateTime - dataTypes: - - dataType: dateTime - profiles: - - http://hl7.org/fhir/StructureDefinition/dateTime - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 1 - maxCardinality: 1 - short: Condition start date - definition: 'When this condition is started or diagnosis is set. ' - - id: abatementDateTime - path: Ext-plt1-condition-test.abatementDateTime - dataTypes: - - dataType: dateTime - profiles: - - http://hl7.org/fhir/StructureDefinition/dateTime - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 0 - maxCardinality: 1 - short: Condition abatement date - definition: If the disease is resolved/no longer active, the end date. - - id: encounterId - path: Ext-plt1-condition-test.encounterId - dataTypes: - - dataType: id - profiles: - - http://hl7.org/fhir/StructureDefinition/id - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 0 - maxCardinality: 1 - short: Encounter - definition: Unique identifier of the visit/encounter that this diagnosis is set (if exists and related) - - # A structure definition example for schema post - StructureDefinitionExample: - value: - resourceType: StructureDefinition - url: https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-operation-episode - name: Ext-plt1-operation-episode - status: draft - fhirVersion: 4.0.1 - kind: logical - abstract: false - type: Ext-plt1-operation-episode - baseDefinition: http://hl7.org/fhir/StructureDefinition/Element - derivation: specialization - differential: - element: - - id: Ext-plt1-operation-episode - path: Ext-plt1-operation-episode - min: 0 - max: '*' - type: - - code: Element - - id: Ext-plt1-operation-episode.pid - path: Ext-plt1-operation-episode.pid - short: Patient identifier - definition: Unique patient identifier (a pseudoynm) - min: 1 - max: '1' - type: - - code: id - profile: - - http://hl7.org/fhir/StructureDefinition/id - - id: Ext-plt1-operation-episode.episodeId - path: Ext-plt1-operation-episode.episodeId - short: Episode identifier - definition: Unique identifier of the episode of care that surgery is performed - min: 1 - max: '1' - type: - - code: id - profile: - - http://hl7.org/fhir/StructureDefinition/id - - # Example response for the path: POST, /terminologies/{terminologyId}/code-systems/{codeSystemId}/header - CodeSystemCsvHeadersExample: - summary: "Example of code system headers which is used to update csv headers" - value: - - currentName: "code" - previousName: "code" - - currentName: "display" - previousName: "display" - - currentName: "new_column" - previousName: "new_column" - - currentName: "new_name" - previousName: "old_name" - - # Example response for the path: POST, /terminologies/{terminologyId}/concept-maps/{conceptMapId}/header - ConceptMapCsvHeadersExample: - summary: "Example of concept map headers which is used to update csv headers" - value: - - currentName: "source_system" - previousName: "source_system" - - currentName: "source_code" - previousName: "source_code" - - currentName: "target_system" - previousName: "target_system" - - currentName: "target_code" - previousName: "target_code" - - currentName: "new_name" - previousName: "target_display" - - # Example settings to import schema - ImportSchemaSettingsExample: - value: - baseUrl: "fhir/example" - resourceId: "structureDefinitionId" - securitySettings: - jsonClass: "BasicAuthenticationSettings" - username: "username" - password: "password" - - # An example of schema field definitions - SchemaFieldDefinitionsExample: - value: - - id: pid - path: Ext-plt1-anesthesia-observations.pid - dataTypes: - - dataType: id - profiles: - - http://hl7.org/fhir/StructureDefinition/id - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 1 - maxCardinality: 1 - short: Patient identifier - definition: Unique patient identifier (a pseudonym) - elements: [ ] - - id: encounterId - path: Ext-plt1-anesthesia-observations.encounterId - dataTypes: - - dataType: id - profiles: - - http://hl7.org/fhir/StructureDefinition/id - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 1 - maxCardinality: 1 - short: Encounter id - definition: Reference to the encounter that this assessment is done - elements: [ ] - - id: time - path: Ext-plt1-anesthesia-observations.time - dataTypes: - - dataType: dateTime - profiles: - - http://hl7.org/fhir/StructureDefinition/dateTime - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 1 - maxCardinality: 1 - short: Time of evaluation - definition: Date or dateTime that this evaluation is done - elements: [ ] - - id: v1 - path: Ext-plt1-anesthesia-observations.v1 - dataTypes: - - dataType: decimal - profiles: - - http://hl7.org/fhir/StructureDefinition/decimal - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 0 - maxCardinality: 1 - short: FiSevofl (%) - definition: FiSevofl (%) - elements: [ ] - - id: v2 - path: Ext-plt1-anesthesia-observations.v2 - dataTypes: - - dataType: decimal - profiles: - - http://hl7.org/fhir/StructureDefinition/decimal - isPrimitive: true - isChoiceRoot: false - isArray: false - minCardinality: 0 - maxCardinality: 1 - short: FiDesfl (%) - definition: FiDesfl (%) - elements: [ ] - - FhirMappingJobExample: - value: - id: "test-name" - name: "test-name" - useFhirSinkAsIdentityService: false - sourceSettings: - source1: - name: "source1" - sourceUri: "suri" - asStream: false - jsonClass: "FileSystemSourceSettings" - dataFolderPath: "test-data" - source2: - name: "source2" - sourceUri: "suri" - jsonClass: "SqlSourceSettings" - databaseUrl: "dat" - username: "us" - password: "pas" - source3: - name: "source3" - sourceUri: "suri" - asStream: true - jsonClass: "KafkaSourceSettings" - bootstrapServers: "bootstrap" - source4: - name: "source4" - sourceUri: "suri" - jsonClass: "FhirServerSourceSettings" - serverUrl: "fhirla" - securitySettings: - jsonClass: "BasicAuthenticationSettings" - username: "us" - password: "pas" - sinkSettings: - jsonClass: "FileSystemSinkSettings" - path: "/path/to/folder" - contentType: "ndjson" - options: - ignoreCorruptFiles: "val" - ignoreMissingFiles: "val" - terminologyServiceSettings: - jsonClass: "FhirRepositorySinkSettings" - fhirRepoUrl: "terminology-service" - securitySettings: - jsonClass: "FixedTokenAuthenticationSettings" - token: "token" - identityServiceSettings: - jsonClass: "FhirRepositorySinkSettings" - fhirRepoUrl: "idi" - securitySettings: - jsonClass: "BasicAuthenticationSettings" - username: "us" - password: "pas" - mappings: - - name: "example-mapping-1" - mappingRef: "https://aiccelerate.eu/fhir/mappings/pilot1/surgery-details-mapping" - sourceBinding: - source: - sourceRef: "source1" - jsonClass: "FileSystemSource" - options: - ignoreCorruptFiles: "val" - path: "pilot1" - contentType: "csv" - - name: "example-mapping-2" - mappingRef: "https://aiccelerate.eu/fhir/mappings/pilot1/medication-administration-mapping" - sourceBinding: - source: - sourceRef: "source2" - jsonClass: "SqlSource" - options: - prepareQuery: "q" - tableName: "table" - s1: - sourceRef: "source2" - jsonClass: "SqlSource" - options: - prepareQuery: "q" - query: "query" - - name: "example-mapping-3" - mappingRef: "https://aiccelerate.eu/fhir/mappings/pilot1/medication-used-mapping" - sourceBinding: - source: - sourceRef: "source3" - jsonClass: "KafkaSource" - topicName: "topic" - options: - startingOffsets: "latest" - maxFilesPerTrigger: "1" - - name: "example-mapping-4" - mappingRef: "https://aiccelerate.eu/fhir/mappings/pilot1/preoperative-risks-mapping" - sourceBinding: - source: - sourceRef: "source4" - jsonClass: "FhirServerSource" - resourceType: "resource" - query: "query" - dataProcessingSettings: - saveErroneousRecords: true - archiveMode: "delete" - schedulingSettings: null \ No newline at end of file diff --git a/ignifyr-server/pom.xml b/ignifyr-server/pom.xml deleted file mode 100644 index 3363c948..00000000 --- a/ignifyr-server/pom.xml +++ /dev/null @@ -1,374 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-server_2.13 - jar - - - - - src/main/resources - true - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.scalatest - scalatest-maven-plugin - - - - test - - test - - - io.ignifyr.server - - - - - integration-test - integration-test - - test - - - - ${skipITs} - io.ignifyr.integrationtest - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - make-assembly - package - - shade - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - false - ignifyr-server-standalone - - - - io.ignifyr.server.Boot - - - - - reference.conf - - - - - - - - - Ignifyr - SRDC Corp. - https://www.srdc.com.tr - 2022 - - - - ${project.basedir}/../LICENSE - META-INF/LICENSE - - - - - - - - - - - - - org.scala-lang - scala-library - - - - - com.typesafe.scala-logging - scala-logging_${scala.binary.version} - - - - - ch.qos.logback - logback-classic - - - ch.qos.logback - logback-core - - - - org.slf4j - log4j-over-slf4j - - - - - io.onfhir - onfhir-fhirpath-functions_${scala.binary.version} - - - io.onfhir - onfhir-resource-definitions_${scala.binary.version} - - - - - com.typesafe - config - - - - - org.json4s - json4s-jackson_${scala.binary.version} - - - - - com.typesafe.akka - akka-http_${scala.binary.version} - - - com.typesafe.akka - akka-http-core_${scala.binary.version} - - - com.typesafe.akka - akka-http-caching_${scala.binary.version} - - - com.typesafe.akka - akka-actor-typed_${scala.binary.version} - - - com.typesafe.akka - akka-stream-typed_${scala.binary.version} - - - - - io.onfhir - onfhir-config_${scala.binary.version} - - - io.onfhir - onfhir-r4_${scala.binary.version} - - - io.onfhir - onfhir-r5_${scala.binary.version} - - - - io.onfhir - onfhir-definitions-r4 - runtime - - - io.onfhir - onfhir-definitions-r5 - runtime - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - io.ignifyr - ignifyr-testkit_2.13 - test - - - io.ignifyr - ignifyr-common - - - io.ignifyr - ignifyr-server-common - - - io.ignifyr - ignifyr-rxnorm_2.13 - - - - io.ignifyr - ignifyr-connector-sql_2.13 - - - io.ignifyr - ignifyr-connector-file_2.13 - - - - io.ignifyr - ignifyr-sink-fhir_2.13 - - - - io.ignifyr - ignifyr-sink-file_2.13 - - - - io.ignifyr - ignifyr-sink-omop_2.13 - - - - io.ignifyr - ignifyr-format-json_2.13 - - - - io.ignifyr - ignifyr-format-delta_2.13 - - - io.ignifyr - ignifyr-connector-fhir-server_2.13 - - - io.ignifyr - ignifyr-connector-kafka_2.13 - - - - io.ignifyr - ignifyr-runtime-streaming_2.13 - - - - io.ignifyr - ignifyr-runtime-scheduling_2.13 - - - - io.ignifyr - ignifyr-redcap_2.13 - - - - io.ignifyr - ignifyr-observability_2.13 - - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.mockito - mockito-scala_2.13 - test - - - - - com.typesafe.akka - akka-stream-testkit_${scala.binary.version} - ${akka.version} - - - - com.typesafe.akka - akka-http-testkit_${scala.binary.version} - ${akka-http.version} - - - - com.h2database - h2 - test - - - - com.opencsv - opencsv - - - - org.testcontainers - testcontainers - test - - - - org.testcontainers - mongodb - test - - - org.testcontainers - junit-jupiter - test - - - - diff --git a/ignifyr-server/src/main/resources/application.conf b/ignifyr-server/src/main/resources/application.conf deleted file mode 100644 index 45e777f9..00000000 --- a/ignifyr-server/src/main/resources/application.conf +++ /dev/null @@ -1,223 +0,0 @@ -# Ignifyr configurations -ignifyr { - - # A path to a file/directory from where any File system readings should use within the mappingjob definition. - # e.g., FileSystemSourceSettings.dataFolderPath or LocalFhirTerminologyServiceSettings.folderPath - context-path = "./" - - mappings = { - - # The repository where the mapping definition are kept. - repository = { - folder-path = "mappings" - } - - # Configuration of the schemas used in the mapping definitions. - schemas = { - repository = { # The repository where the schema definitions are kept. - folder-path = "schemas" - } - # Specific FHIR version used for schemas in the schema repository. - # Represents fhirVersion field in the standard StructureDefinition e.g. 4.0.1, 5.0.0 - fhir-version = "4.0.1" - } - - contexts = { - # The repository where the context definitions are kept. - repository = { - folder-path = "mapping-contexts" - } - } - - # Timeout for each mapping execution on an individual input record - timeout = 5 seconds - } - - mapping-jobs = { - repository = { # The repository where the schema definitions are kept. - folder-path = "mapping-jobs" - } - # Absolute path to the JSON file for the MappingJob definition to load at the beginning - # initial-job-file-path = "mapping-jobs/pilot1-mappingjob.json" - - # Number of partitions to repartition the source data before executing the mappings for the mapping jobs - # numOfPartitions = 10 - - # Maximum number of records for batch mapping execution, if source data exceeds this it is divided into chunks - # maxChunkSize = 10000 - } - - terminology-systems = { - # The path to the folder where Terminology System files (config files, CodeSystems, ConceptMaps etc.) are kept. - folder-path = "terminology-systems" - } - - archiving = { - # Folder to keep erroneous records - erroneous-records-folder = "erroneous-records-folder" - - # Folder to keep archived files - archive-folder = "archive-folder" - - # Frequency in milliseconds to run the archiving task for file streaming jobs - stream-archiving-frequency = 5000 - } - - # Settings for FHIR repository writer - fhir-server-writer { - # The # of FHIR resources in the group while executing (create/update) a FHIR batch operation. - batch-group-size = 50 - } - - # Database folder of Ignifyr (e.g., to maintain synchronization times for scheduled jobs) - db-path = "ignifyr-db" - - # External function libraries containing function to be used within FHIRPath expressions - functionLibraries { - rxn { - className = "io.ignifyr.rxnorm.RxNormApiFunctionLibraryFactory" - args = ["https://rxnav.nlm.nih.gov", 2] - } - cst { - className = "io.ignifyr.common.util.CustomMappingFunctionsFactory" - } - } -} - -fhir = { - # major FHIR version, currently R4 and R5 is supported - fhir-version = "R5" - - # List of root URLs while retrieving the definitions (profiles, valuesets, codesystems). - # The definitions below the given root URLs will be retrieved from the configured paths or FHIR endpoints. - # All definitions will be retrieved if no root URLs are provided. - # e.g. ["https://aiccelerate.eu", "https://fair4health.eu"] - #definitions-root-urls = ["http://hl7.org/fhir/"] - definitions-root-urls = null - - # FHIR URL to retrieve resource definitions (profiles, valuesets and codesystems). - # If this URL is defined, file paths (profiles-path, valuesets-path, codesystems-path) will be ignored even if they are also provided. - # For now, Ignifyr can read definitions from a single FHIR endpoint. - #definitions-fhir-endpoint = "http://localhost:8080/fhir" - definitions-fhir-endpoint = null - - fhir-endpoint-auth = { - # basic | token | fixed-token - # If one of the auth methods is selected, its configurations must be provided as shown below. - method = null - -# # basic configurations are used if the auth method is basic -# basic = { -# username = "user" -# password = "pass" -# } -# -# # token configurations are used if the auth method is token -# token = { -# client-id = "id" -# client-secret = "secret" -# scopes = [] -# token-endpoint = "https://onauth.srdc.com.tr" -# } - -# # fixed token configurations are used if the auth method is fixed-token -# fixed-token = "XXX" - } - - # A path to a context file/directory from where profiles, value sets and code systems reading should start. - context-path = "./" - # Path to the zip file or folder that includes the FHIR resource and data type profile definitions (FHIR StructureDefinition) to be served by Ignifyr webserver so that mappings can be performed accordingly. - profiles-path = null - - # Path to the zip file or folder that includes the FHIR Value Set definitions (FHIR ValueSet) that are referenced by your FHIR profiles. - valuesets-path = null - - # Path to the zip file or folder that includes the FHIR Code system definitions (FHIR CodeSystem) that are referenced by your FHIR value sets. - codesystems-path = null -} - -webserver = { - # Hostname that Ignifyr server will work. Using 0.0.0.0 will bind the server to both localhost and the IP of the server that you deploy it. - host = 0.0.0.0 - - # Port to listen - port = 8085 - - # Base Uri for server e.g. With this default configuration, the root path of Ignifyr server will be http://localhost:8085/ignifyr - base-uri = ignifyr - - ssl { - # Path to the java keystore for enabling ssl for Ignifyr server, use null to disable ssl - keystore = null - # Password of the keystore for enabling ssl for Ignifyr server - password = null - } -} - -# Spark configurations -spark = { - app.name = "Ignifyr Server" - master = "local[4]" - ui.enabled = "false" - # Directory to store Spark checkpoints - checkpoint-dir = "checkpoint" - -# # Cluster configurations for spark -# spark.submit.deployMode = "cluster" -# spark.driver.host = "192.168.1.103" -# spark.driver.core = "8" -# spark.driver.memory = "12g" -# spark.executor.cores = "8" -# spark.executor.memory = "14g" -# spark.network.timeout = "600s" -# spark.jars = "ignifyr-server-standalone.jar" -} - -akka = { - daemonic = "on" - - # Configurations for Akka HTTP - http = { - parsing = { - max-header-value-length = 82k - } - server = { - # Header for server - server-header = Ignifyr Server - parsing{ - uri-parsing-mode = relaxed - # Default maximum content length which should not be exceeded by incoming request entities. - # Set to `infinite` to completely disable entity length checks. - max-content-length = 8m - } - # Request timeout for all REST services - request-timeout = 60 s - # Maximum inactivity time of a given HTTP connection - idle-timeout = 60 s - # Should be on in order to get IP address of the clients for audits - remote-address-header = on - } - } -} - -# Configuration for the ignifyr-redcap service -ignifyr-redcap { - # Base URL of the ignifyr-redcap service - endpoint = "http://localhost:8095/tofhir-redcap" - - # Paths for the endpoints provided by ignifyr-redcap - paths { - # Specifies the path to the notification endpoint relative to the endpoint URL i.e. http://localhost:8095/tofhir-redcap/notification - notification = "notification" - - # Specifies the path to the projects endpoint relative to the endpoint URL i.e. http://localhost:8095/tofhir-redcap/projects - projects = "projects" - - # Specifies the path to the projects' data endpoint relative to the endpoint URL - projectData = "data" - } - # Parameters for the endpoints provided by ignifyr-redcap - parameters { - reload = "reload" - } -} \ No newline at end of file diff --git a/ignifyr-server/src/main/resources/logback.xml b/ignifyr-server/src/main/resources/logback.xml deleted file mode 100644 index 85222480..00000000 --- a/ignifyr-server/src/main/resources/logback.xml +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - - - - - - - - - - %d{yyyy-MM-dd_HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n - - - - - - ${LOG_FOLDER}/ignifyr-server.log - - %d{yyyy-MM-dd_HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - ${LOG_FOLDER}/ignifyr-server.%i.log.zip - 1 - 10 - - - - 10MB - - - - - - - - - ${LOG_FOLDER}/ignifyr-mappings.log - - INFO - - - false - - [ignore] - [ignore] - [ignore] - [ignore] - - - - - - - ${LOG_FOLDER}/ignifyr-mappings.%i.log.zip - 1 - 10 - - - - 10MB - - - - - - ${FLUENT_HOST} - 24224 - - - INFO - - - true - - - - - - - - - - 0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ignifyr-server/src/main/resources/version.properties b/ignifyr-server/src/main/resources/version.properties deleted file mode 100644 index 71dfb336..00000000 --- a/ignifyr-server/src/main/resources/version.properties +++ /dev/null @@ -1 +0,0 @@ -application.version=${project.version} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/Boot.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/Boot.scala deleted file mode 100644 index 20168e7d..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/Boot.scala +++ /dev/null @@ -1,11 +0,0 @@ -package io.ignifyr.server - -import com.typesafe.scalalogging.Logger -import io.ignifyr.common.app.AppVersion - -object Boot extends App { - val logger: Logger = Logger(this.getClass) - logger.info(s"Starting Ignifyr version: ${AppVersion.getVersion}") - - IgnifyrServer.start() -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrHttpServer.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrHttpServer.scala deleted file mode 100644 index d971dc5f..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrHttpServer.scala +++ /dev/null @@ -1,69 +0,0 @@ -package io.ignifyr.server - -import akka.Done -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.server.common.config.WebServerConfig - -import java.util.concurrent.TimeUnit -import scala.concurrent.duration.{Duration, FiniteDuration} -import scala.concurrent._ -import scala.io.StdIn -import scala.util.{Failure, Success} - -object IgnifyrHttpServer extends LazyLogging { - - def start(route: Route, webServerConfig: WebServerConfig)(implicit actorSystem: ActorSystem): Unit = { - implicit val executionContext: ExecutionContext = actorSystem.dispatcher - - val serverBindingFuture = Http() - .newServerAt(webServerConfig.serverHost, webServerConfig.serverPort) - .bind(route) - .map(serverBinding => { - serverBinding.addToCoordinatedShutdown(hardTerminationDeadline = FiniteDuration(10, TimeUnit.SECONDS)) - serverBinding.whenTerminated onComplete { - case Success(_) => - logger.info("Closing Ignifyr HTTP server...") - actorSystem.terminate() - logger.info("Ignifyr HTTP server gracefully terminated.") - case Failure(exception) => - logger.error("Problem while gracefully terminating Ignifyr HTTP server!", exception) - } - serverBinding - }) - - var serverBinding: Option[Http.ServerBinding] = None - try { - serverBinding = Some(Await.result(serverBindingFuture, FiniteDuration(10L, TimeUnit.SECONDS))) - logger.info(s"Ignifyr server ready at ${webServerConfig.serverHost}:${webServerConfig.serverPort}") - } catch { - case e: Exception => - logger.error("Problem while binding to the given HTTP address and port!", e) - actorSystem.terminate() - } - - // Wait for a shutdown signal - Await.ready(waitForShutdownSignal(), Duration.Inf) - serverBinding.get.terminate(FiniteDuration.apply(10L, TimeUnit.SECONDS)) - } - - protected def waitForShutdownSignal()(implicit executionContext: ExecutionContext): Future[Done] = { - val promise = Promise[Done]() - sys.addShutdownHook { - promise.trySuccess(Done) - } - Future { - blocking { - do { - val line = StdIn.readLine("Write 'q' or 'quit' to stop the server...\n") - if (line.equalsIgnoreCase("quit")) - promise.trySuccess(Done) - } while (!promise.isCompleted) - } - } - promise.future - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrServer.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrServer.scala deleted file mode 100644 index 8021bbf6..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/IgnifyrServer.scala +++ /dev/null @@ -1,23 +0,0 @@ -package io.ignifyr.server - -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.server.common.config.WebServerConfig -import io.ignifyr.server.common.spi.IgnifyrServerExtensions -import io.ignifyr.server.endpoint.IgnifyrServerEndpoint -import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig - -object IgnifyrServer { - def start(): Unit = { - import io.ignifyr.engine.Execution.actorSystem - - val webServerConfig = new WebServerConfig(actorSystem.settings.config.getConfig("webserver")) - val fhirDefinitionsConfig = new FhirDefinitionsConfig(actorSystem.settings.config.getConfig("fhir")) - // Discover installed server extension modules (e.g. REDCap); each decides from the root config - // whether to contribute its routes. - val serverExtensions = IgnifyrServerExtensions.load(actorSystem.settings.config) - val endpoint = - new IgnifyrServerEndpoint(IgnifyrConfig.engineConfig, webServerConfig, fhirDefinitionsConfig, serverExtensions) - - IgnifyrHttpServer.start(endpoint.ignifyrRoute, webServerConfig) - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/CodeSystemEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/CodeSystemEndpoint.scala deleted file mode 100644 index 0013a254..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/CodeSystemEndpoint.scala +++ /dev/null @@ -1,206 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.headers.RawHeader -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.{ResourceNotFound, IgnifyrRestCall} -import io.ignifyr.server.endpoint.CodeSystemEndpoint.SEGMENT_CODE_SYSTEMS -import io.ignifyr.server.endpoint.TerminologyServiceManagerEndpoint._ -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.service.terminology.CodeSystemService -import io.ignifyr.server.repository.terminology.codesystem.ICodeSystemRepository - -class CodeSystemEndpoint(codeSystemRepository: ICodeSystemRepository) extends LazyLogging { - - val service: CodeSystemService = new CodeSystemService(codeSystemRepository) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_CODE_SYSTEMS) { - val terminologyId: String = request.terminologyId.get - pathEndOrSingleSlash { - getAllCodeSystemsRoute(terminologyId) - } ~ pathPrefix(Segment) { codeSystemId => - pathEndOrSingleSlash { - getCodeSystemRoute(terminologyId, codeSystemId) - } ~ pathPrefix(SEGMENT_CONTENT) { // code-systems//content (paginated operations) - pathEndOrSingleSlash { - getOrSaveCodeSystemContentRoute(terminologyId, codeSystemId) - } - } ~ pathPrefix(SEGMENT_HEADER) { // code-systems//header (CSV headers update) - pathEndOrSingleSlash { - updateCodeSystemHeaderRoute(terminologyId, codeSystemId) - } - } ~ pathPrefix(SEGMENT_FILE) { // code-systems//file (import/export as whole CSV file) - pathEndOrSingleSlash { - uploadDownloadCodeSystemFileRoute(terminologyId, codeSystemId) - } - } - } - } - } - - /** - * Route to get all code systems within a terminology - * - * @return - */ - private def getAllCodeSystemsRoute(terminologyId: String): Route = { - get { - complete { - service.getCodeSystems(terminologyId) map { metadata => - StatusCodes.OK -> metadata - } - } - } - } - - /** - * Route to get a code system terminology - * - * @param terminologyId id of code system terminology - * @param codeSystemId id of code system - * @return - */ - private def getCodeSystemRoute(terminologyId: String, codeSystemId: String): Route = { - get { - complete { - service.getCodeSystem(terminologyId, codeSystemId) - } - } - } - - /** - * Route to update code system CSV headers. - * - * Headers are passed as a list of `CsvHeader` objects, where each `CsvHeader` contains both the `currentName` (new header name) and `previousName` (the previously saved header name). - * The existing headers in the first line of the CSV file are overwritten with the new headers provided. The rows are adjusted to match the new headers if necessary. - * - * e.g. 1: - * Existing headers in a CSV: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "header2", previousName = "header2") - * New headers: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "headerChanged", previousName = "header2") - * - CsvHeader(currentName = "header3", previousName = "header3") - * Rows under "header1" are preserved, rows belonging to "header2" are moved to "headerChanged", and "header3" is added with a default value for each row (``). - * - * e.g. 2: - * Existing headers in a CSV: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "header2", previousName = "header2") - * New headers: - * - CsvHeader(currentName = "header2", previousName = "header2") - * - CsvHeader(currentName = "header3", previousName = "header3") - * Rows under "header1" are removed, rows under "header2" are preserved and shifted to the first column, and "header3" is added as the second column with a default value for each row (``). - * - * @param terminologyId terminology id - * @param codeSystemId code system id - * @return - */ - private def updateCodeSystemHeaderRoute(terminologyId: String, codeSystemId: String): Route = { - post { - entity(as[Seq[CsvHeader]]) { headers => - complete { - service.updateCodeSystemHeader(terminologyId, codeSystemId, headers) map { _ => - StatusCodes.OK - } - } - } - } - } - - /** - * Route to update or get a paginated code system content - * File to be updated is found by terminology id and code system id - * It updates/returns only the part of the CSV that corresponds to that page information. e.g: - * If the CSV has 1000 records and the page size is 10, then the first page will have records from 1 to 10 excluding the header. - * If the page number is 2, then the records from 11 to 20 will be returned/updated. - * - * @param terminologyId id of code system terminology - * @param codeSystemId id of code system - * @return - */ - private def getOrSaveCodeSystemContentRoute(terminologyId: String, codeSystemId: String): Route = { - - /** - * POST request to update part of the code system content - * byteSource contains the CSV content to be updated - * Returns empty body with a total records header in case of success - */ - post { - fileUpload(ATTACHMENT) { case (fileInfo, byteSource) => - parameterMap { queryParams => - complete { - val pageNumber = queryParams.getOrElse("page", "1").toInt - val pageSize = queryParams.getOrElse("size", "10").toInt - service.saveCodeSystemContent(terminologyId, codeSystemId, byteSource, pageNumber, pageSize) map { - totalRecords => - HttpResponse( - StatusCodes.OK, - headers = List(RawHeader("X-Total-Count", totalRecords.toString)) - ) - } - } - } - } - - /** - * GET request to get a paginated code system content - * Returns the paginated CSV content with a total records header - */ - } ~ get { - parameterMap { queryParams => - complete { - val pageNumber = queryParams.getOrElse("page", "1").toInt - val pageSize = queryParams.getOrElse("size", "10").toInt - service.getCodeSystemContent(terminologyId, codeSystemId, pageNumber, pageSize) map { - case (byteSource, totalRecords) => - HttpResponse( - StatusCodes.OK, - headers = List(RawHeader("X-Total-Count", totalRecords.toString)), - entity = HttpEntity(ContentTypes.`text/csv(UTF-8)`, byteSource) - ) - } - } - } - } - } - - /** - * Route to upload/download code system content - * File to be uploaded/downloaded is found by terminology id and code system id - * It uploads/downloads the whole CSV file as a single operation - * Operations with large files may take a long time to complete - * - * @param terminologyId id of code system terminology - * @param codeSystemId id of code system - * @return - */ - private def uploadDownloadCodeSystemFileRoute(terminologyId: String, codeSystemId: String): Route = { - post { - fileUpload(ATTACHMENT) { case (fileInfo, byteSource) => - complete { - service.uploadCodeSystemFile(terminologyId, codeSystemId, byteSource) map { _ => - StatusCodes.OK - } - } - } - } ~ get { - complete { - service.downloadCodeSystemFile(terminologyId, codeSystemId) map { byteSource => - HttpResponse(StatusCodes.OK, entity = HttpEntity(ContentTypes.`text/csv(UTF-8)`, byteSource)) - } - } - } - } - -} - -object CodeSystemEndpoint { - val SEGMENT_CODE_SYSTEMS = "code-systems" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ConceptMapEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ConceptMapEndpoint.scala deleted file mode 100644 index 1265fd29..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ConceptMapEndpoint.scala +++ /dev/null @@ -1,206 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.headers.RawHeader -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.{ResourceNotFound, IgnifyrRestCall} -import io.ignifyr.server.endpoint.ConceptMapEndpoint.SEGMENT_CONCEPT_MAPS -import io.ignifyr.server.endpoint.TerminologyServiceManagerEndpoint._ -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.service.terminology.ConceptMapService -import io.ignifyr.server.repository.terminology.conceptmap.IConceptMapRepository - -class ConceptMapEndpoint(conceptMapRepository: IConceptMapRepository) extends LazyLogging { - - val service: ConceptMapService = new ConceptMapService(conceptMapRepository) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_CONCEPT_MAPS) { - val terminologyId: String = request.terminologyId.get - pathEndOrSingleSlash { - getAllConceptMapsRoute(terminologyId) - } ~ pathPrefix(Segment) { conceptMapId => - pathEndOrSingleSlash { - getConceptMapRoute(terminologyId, conceptMapId) - } ~ pathPrefix(SEGMENT_CONTENT) { // concept-maps//content (paginated operations) - pathEndOrSingleSlash { - getOrSaveConceptMapContentRoute(terminologyId, conceptMapId) - } - } ~ pathPrefix(SEGMENT_HEADER) { // concept-maps//header (CSV headers update) - pathEndOrSingleSlash { - updateConceptMapHeaderRoute(terminologyId, conceptMapId) - } - } ~ pathPrefix(SEGMENT_FILE) { // code-systems//file (import/export as whole CSV file) - pathEndOrSingleSlash { - uploadDownloadConceptMapFileRoute(terminologyId, conceptMapId) - } - } - } - } - } - - /** - * Route to get all concept maps within a terminology - * - * @return - */ - private def getAllConceptMapsRoute(terminologyId: String): Route = { - get { - complete { - service.getConceptMaps(terminologyId) map { metadata => - StatusCodes.OK -> metadata - } - } - } - } - - /** - * Route to get a concept map terminology - * - * @param terminologyId id of concept map terminology - * @param conceptMapId id of concept map - * @return - */ - private def getConceptMapRoute(terminologyId: String, conceptMapId: String): Route = { - get { - complete { - service.getConceptMap(terminologyId, conceptMapId) - } - } - } - - /** - * Route to update concept map csv header - * - * Headers are passed as a list of `CsvHeader` objects and overwrite the existing headers in the first line of the CSV file. - * The `CsvHeader` class represents each header with both its current name and previously saved name. - * The rows are adjusted to match the new headers if necessary. - * - * e.g. 1: - * Existing headers in a CSV: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "header2", previousName = "header2") - * New headers: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "headerChanged", previousName = "header2") - * - CsvHeader(currentName = "header3", previousName = "header3") - * Rows under "header1" are preserved, rows belonging to "header2" are moved to "headerChanged", and "header3" is added with a default value for each row (``). - * - * e.g. 2: - * Existing headers in a CSV: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "header2", previousName = "header2") - * New headers: - * - CsvHeader(currentName = "header2", previousName = "header2") - * - CsvHeader(currentName = "header3", previousName = "header3") - * Rows under "header1" are removed, rows under "header2" are preserved and shifted to the first column, and "header3" is added as the second column with a default value for each row (``). - * - * @param terminologyId terminology id - * @param conceptMapId concept map id - * @return - */ - private def updateConceptMapHeaderRoute(terminologyId: String, conceptMapId: String): Route = { - post { - entity(as[Seq[CsvHeader]]) { headers => - complete { - service.updateConceptMapHeader(terminologyId, conceptMapId, headers) map { _ => - StatusCodes.OK - } - } - } - } - } - - /** - * Route to update or get a paginated concept map content - * File to be updated is found by terminology id and concept map id - * It updates/returns only the part of the CSV that corresponds to that page information. e.g: - * If the CSV has 1000 records and the page size is 10, then the first page will have records from 1 to 10 excluding the header. - * If the page number is 2, then the records from 11 to 20 will be returned/updated. - * - * @param terminologyId id of concept map terminology - * @param conceptMapId id of concept map - * @return - */ - private def getOrSaveConceptMapContentRoute(terminologyId: String, conceptMapId: String): Route = { - - /** - * POST request to update part of the concept map content - * byteSource contains the CSV content to be updated - * Returns empty body with a total records header in case of success - */ - post { - fileUpload(ATTACHMENT) { case (fileInfo, byteSource) => - parameterMap { queryParams => - complete { - val pageNumber = queryParams.getOrElse("page", "1").toInt - val pageSize = queryParams.getOrElse("size", "10").toInt - service.saveConceptMapContent(terminologyId, conceptMapId, byteSource, pageNumber, pageSize) map { - totalRecords => - HttpResponse( - StatusCodes.OK, - headers = List(RawHeader("X-Total-Count", totalRecords.toString)) - ) - } - } - } - } - - /** - * GET request to get a paginated concept map content - * Returns the paginated CSV content with a total records header - */ - } ~ get { - parameterMap { queryParams => - complete { - val pageNumber = queryParams.getOrElse("page", "1").toInt - val pageSize = queryParams.getOrElse("size", "10").toInt - service.getConceptMapContent(terminologyId, conceptMapId, pageNumber, pageSize) map { - case (byteSource, totalRecords) => - HttpResponse( - StatusCodes.OK, - headers = List(RawHeader("X-Total-Count", totalRecords.toString)), - entity = HttpEntity(ContentTypes.`text/csv(UTF-8)`, byteSource) - ) - } - } - } - } - } - - /** - * Route to upload/download a concept map content - * File to be uploaded/downloaded is found by terminology id and code system id - * It uploads/downloads the whole CSV file as a single operation - * Operations with large files may take a long time to complete - * @param terminologyId id of concept map terminology - * @param conceptMapId id of concept map - * @return - */ - private def uploadDownloadConceptMapFileRoute(terminologyId: String, conceptMapId: String): Route = { - post { - fileUpload(ATTACHMENT) { case (fileInfo, byteSource) => - complete { - service.uploadConceptMapFile(terminologyId, conceptMapId, byteSource) map { _ => - StatusCodes.OK - } - } - } - } ~ get { - complete { - service.downloadConceptMapFile(terminologyId, conceptMapId) map { byteSource => - HttpResponse(StatusCodes.OK, entity = HttpEntity(ContentTypes.`text/csv(UTF-8)`, byteSource)) - } - } - } - } - -} - -object ConceptMapEndpoint { - val SEGMENT_CONCEPT_MAPS = "concept-maps" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpoint.scala deleted file mode 100644 index 17809c9a..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpoint.scala +++ /dev/null @@ -1,44 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.server.Directives.{pathEndOrSingleSlash, pathPrefix} -import akka.http.scaladsl.server.Route -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.ignifyr.server.endpoint.FileSystemTreeStructureEndpoint._ -import akka.http.scaladsl.server.Directives._ -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.service.fhir.FileSystemTreeStructureService - -class FileSystemTreeStructureEndpoint { - - val service: FileSystemTreeStructureService = new FileSystemTreeStructureService() - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_FILE_SYSTEM_PATH) { - pathEndOrSingleSlash { - getFolderTreeStructure - } - } - } - - /** - * Returns the folder tree structure of the file system. - * @return - */ - private def getFolderTreeStructure: Route = { - get { - parameterMap { queryParams => - val basePath = queryParams.getOrElse(QUERY_PARAM_BASE_PATH, ".") - val includeFiles = queryParams.getOrElse(QUERY_PARAM_INCLUDE_FILES, "false").toBoolean - complete { - service.getFolderTreeStructure(basePath, includeFiles) - } - } - } - } -} - -object FileSystemTreeStructureEndpoint { - val SEGMENT_FILE_SYSTEM_PATH = "file-system-path" - val QUERY_PARAM_BASE_PATH = "base_path" - val QUERY_PARAM_INCLUDE_FILES = "include_files" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/IgnifyrServerEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/IgnifyrServerEndpoint.scala deleted file mode 100644 index 8201764b..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/IgnifyrServerEndpoint.scala +++ /dev/null @@ -1,116 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{HttpMethod, Uri} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.{RejectionHandler, Route} -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.server.common.config.WebServerConfig -import io.ignifyr.server.common.spi.IgnifyrServerExtension -import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig -import io.onfhir.definitions.resource.endpoint.FhirDefinitionsEndpoint -import io.ignifyr.server.common.interceptor.{ICORSHandler, IErrorHandler} -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.ignifyr.server.repository.job.JobFolderRepository -import io.ignifyr.server.repository.mapping.ProjectMappingFolderRepository -import io.ignifyr.server.repository.mappingContext.MappingContextFolderRepository -import io.ignifyr.server.repository.project.{IProjectRepository, ProjectFolderRepository} -import io.ignifyr.server.repository.schema.SchemaFolderRepository -import io.ignifyr.server.repository.terminology.{ITerminologySystemRepository, TerminologySystemFolderRepository} -import io.ignifyr.server.repository.terminology.codesystem.{CodeSystemFolderRepository, ICodeSystemRepository} -import io.ignifyr.server.repository.terminology.conceptmap.{ConceptMapFolderRepository, IConceptMapRepository} -import io.ignifyr.server.util.IgnifyrRejectionHandler -import io.onfhir.definitions.fhirpath.endpoint.FhirPathFunctionsEndpoint -import io.ignifyr.server.repository.{FolderDBInitializer, FolderRepositoryManager, IRepositoryManager} - -import java.util.UUID - -/** - * Encapsulates all services and directives - * Main Endpoint for Ignifyr server - */ -class IgnifyrServerEndpoint( - ignifyrEngineConfig: IgnifyrEngineConfig, - webServerConfig: WebServerConfig, - fhirDefinitionsConfig: FhirDefinitionsConfig, - serverExtensions: Seq[IgnifyrServerExtension] -) extends ICORSHandler - with IErrorHandler { - - private val repositoryManager: IRepositoryManager = new FolderRepositoryManager(ignifyrEngineConfig) - // Initialize repositories by reading the resources available in the file system - repositoryManager.init() - - private val projectEndpoint = new ProjectEndpoint( - repositoryManager.schemaRepository, - repositoryManager.mappingRepository, - repositoryManager.mappingJobRepository, - repositoryManager.mappingContextRepository, - repositoryManager.projectRepository, - serverExtensions - ) - - val terminologyServiceManagerEndpoint = new TerminologyServiceManagerEndpoint( - repositoryManager.terminologySystemRepository, - repositoryManager.conceptMapRepository, - repositoryManager.codeSystemRepository, - repositoryManager.mappingJobRepository - ) - - val fhirDefinitionsEndpoint = new FhirDefinitionsEndpoint(fhirDefinitionsConfig) - - val functionLibraryPackages: Seq[String] = - Seq("io.onfhir.path", "io.ignifyr.engine.mapping") ++ - ignifyrEngineConfig.functionLibrariesConfig // add external function libraries - .map(_.libraryPackageNames) - .getOrElse(Seq.empty) - val fhirPathFunctionsEndpoint = new FhirPathFunctionsEndpoint(functionLibraryPackages) - - // Routes contributed by installed server extension modules (e.g. the REDCap proxy endpoints), - // collected once at assembly; an extension contributes nothing when unconfigured. - private val extensionRootRoutes: Seq[IgnifyrRestCall => Route] = serverExtensions.flatMap(_.rootRoutes) - val fileSystemTreeStructureEndpoint = new FileSystemTreeStructureEndpoint() - val metadataEndpoint = - new MetadataEndpoint(ignifyrEngineConfig, webServerConfig, fhirDefinitionsConfig, serverExtensions) - - val reloadEndpoint = new ReloadEndpoint(repositoryManager) - - // Custom rejection handler to send proper messages to user - val ignifyrRejectionHandler: RejectionHandler = IgnifyrRejectionHandler.getRejectionHandler() - - lazy val ignifyrRoute: Route = - pathPrefix(webServerConfig.baseUri) { - corsHandler { - withRequestTimeoutResponse(_ => IgnifyrRejectionHandler.timeoutResponseHandler()) { - extractMethod { httpMethod: HttpMethod => - extractUri { requestUri: Uri => - extractRequestEntity { requestEntity => - optionalHeaderValueByName("X-Correlation-Id") { correlationId => - val restCall = new IgnifyrRestCall( - method = httpMethod, - uri = requestUri, - requestId = correlationId.getOrElse(UUID.randomUUID().toString), - requestEntity = requestEntity - ) - handleRejections(ignifyrRejectionHandler) { - handleExceptions(exceptionHandler(restCall)) { // Handle exceptions - val routes = Seq( - terminologyServiceManagerEndpoint.route(restCall), - projectEndpoint.route(restCall), - fhirDefinitionsEndpoint.route(), - fhirPathFunctionsEndpoint.route(), - fileSystemTreeStructureEndpoint.route(restCall), - metadataEndpoint.route(restCall), - reloadEndpoint.route(restCall) - ) ++ extensionRootRoutes.map(_(restCall)) - - concat(routes: _*) - } - } - } - } - } - } - } - } - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/JobEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/JobEndpoint.scala deleted file mode 100644 index 8d4ef26d..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/JobEndpoint.scala +++ /dev/null @@ -1,306 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{HttpEntity, StatusCodes} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.model.FhirMappingJob -import io.ignifyr.server.endpoint.JobEndpoint.{ - SEGMENT_DESCHEDULE, - SEGMENT_EXECUTIONS, - SEGMENT_JOB, - SEGMENT_MAPPINGS, - SEGMENT_RUN, - SEGMENT_STATUS, - SEGMENT_STOP, - SEGMENT_TEST -} -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.model.{ExecuteJobTask, RowSelectionOrder, TestResourceCreationRequest} -import io.ignifyr.server.service.{ExecutionService, JobService} -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.server.common.model.{ResourceNotFound, IgnifyrRestCall} -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.repository.schema.ISchemaRepository -import io.ignifyr.server.common.interceptor.ICORSHandler -import scala.concurrent.Future - -class JobEndpoint( - jobRepository: IJobRepository, - mappingRepository: IMappingRepository, - schemaRepository: ISchemaRepository -) extends LazyLogging - with ICORSHandler { - - val service: JobService = new JobService(jobRepository) - val executionService: ExecutionService = new ExecutionService(jobRepository, mappingRepository, schemaRepository) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_JOB) { - val projectId: String = request.projectId.get - pathEndOrSingleSlash { // Operations on all mapping jobs - getAllJobs(projectId) ~ createJob(projectId) - } ~ pathPrefix(Segment) { jobId: String => - pathEndOrSingleSlash { // Operations on a single job, jobs/ - getJob(projectId, jobId) ~ updateJob(projectId, jobId) ~ deleteJob(projectId, jobId) - } ~ pathPrefix(SEGMENT_RUN) { // run a mapping job, jobs//run - pathEndOrSingleSlash { - runJob(projectId, jobId) - } - } ~ pathPrefix(SEGMENT_STATUS) { // check whether a mapping job is running, jobs//status - pathEndOrSingleSlash { - isJobRunning(jobId) - } - } ~ pathPrefix(SEGMENT_TEST) { // test a mapping with mapping job configurations, jobs//test - pathEndOrSingleSlash { - testMappingWithJob(projectId, jobId) - } - } ~ pathPrefix(SEGMENT_EXECUTIONS) { // Operations on all executions, jobs//executions - pathEndOrSingleSlash { - getExecutions(projectId, jobId) ~ stopExecutions(jobId) - } ~ pathPrefix(Segment) { - executionId: String => // operations on a single execution, jobs//executions/ - pathPrefix(SEGMENT_RUN) { // jobs//executions//run - pathEndOrSingleSlash { - continueJobExecution(projectId, jobId, executionId) - } - } ~ pathPrefix(SEGMENT_STOP) { // jobs//executions//stop - pathEndOrSingleSlash { - stopJobExecution(jobId, executionId) - } - } ~ pathPrefix(SEGMENT_DESCHEDULE) { // jobs//executions//deschedule - pathEndOrSingleSlash { - descheduleJobExecution(jobId, executionId) - } - } ~ pathPrefix(SEGMENT_MAPPINGS) { // jobs//executions//mappings - pathPrefix(Segment) { - mappingTaskName: String => // jobs//executions//mappings/ - pathPrefix(SEGMENT_STOP) { // jobs//executions//mappings//stop - pathEndOrSingleSlash { - stopMappingExecution(jobId, executionId, mappingTaskName) - } - } - } - } - } - } - } - } - } - - private def getAllJobs(projectId: String): Route = { - get { - complete { - service.getAllMetadata(projectId) - } - } - } - - private def createJob(projectId: String): Route = { - post { - entity(as[FhirMappingJob]) { job => - complete { - service.createJob(projectId, job) map { created => - StatusCodes.Created -> created - } - } - } - } - } - - private def getJob(projectId: String, jobId: String): Route = { - get { - complete { - service.getJob(projectId, jobId) map { - case Some(mappingJob) => StatusCodes.OK -> mappingJob - case None => - StatusCodes.NotFound -> { - throw ResourceNotFound("Job not found", s"Mapping job with name $jobId not found") - } - } - } - } - } - - private def updateJob(projectId: String, jobId: String): Route = { - put { - entity(as[FhirMappingJob]) { job => - complete { - service.updateJob(projectId, jobId, job) map { _ => - StatusCodes.OK -> job - } - } - } - } - } - - /** - * Route to delete a mapping job if it is not currently running. - * - * If the job is running, a BadRequest response is returned, indicating that running mapping jobs cannot - * be deleted. If the job is not running, the job is deleted. - * - * @param projectId The identifier of the project to which the mapping job belongs. - * @param jobId The identifier of the mapping job to be deleted. - */ - private def deleteJob(projectId: String, jobId: String): Route = { - delete { - complete { - executionService.isJobRunning(jobId) flatMap { result => - if (result) - Future { - StatusCodes.BadRequest -> s"The running mapping jobs cannot be deleted." - } - else - service.deleteJob(projectId, jobId) map { _ => - StatusCodes.NoContent -> HttpEntity.Empty - } - } - } - } - } - - private def runJob(projectId: String, jobId: String): Route = { - post { - entity(as[Option[ExecuteJobTask]]) { executeJobTask => - complete { - executionService.runJob(projectId, jobId, None, executeJobTask) map { _ => - StatusCodes.OK - } - } - } - } - } - - /** - * Route to check if a mapping job with the specified ID is currently running. - * - * @param jobId The identifier of the mapping job to be checked for running status. - */ - private def isJobRunning(jobId: String): Route = { - get { - complete { - executionService.isJobRunning(jobId).map(result => result.toString) - } - } - } - - /** - * Route to test a mapping with mapping job configurations i.e. source data configurations - * */ - private def testMappingWithJob(projectId: String, jobId: String): Route = { - post { - entity(as[TestResourceCreationRequest]) { requestBody => - validate( - RowSelectionOrder.isValid(requestBody.resourceFilter.order), - s"Invalid row selection order. Available options are: ${RowSelectionOrder.START}, ${RowSelectionOrder.RANDOM}" - ) { - complete { - executionService.testMappingWithJob(projectId, jobId, requestBody) - } - } - } - } - } - - /** - * Route to get executions of a mapping job - * */ - private def getExecutions(projectId: String, jobId: String): Route = { - get { - complete { - executionService.getExecutions(projectId, jobId) - } - } - } - - /** - * Route to handle the stopping of executions for a mapping job. - * - * @param jobId The identifier of the mapping job for which executions should be stopped. - */ - private def stopExecutions(jobId: String): Route = { - delete { - complete { - executionService.stopJobExecutions(jobId) - } - } - } - - /** - * Route to continue a job execution with parameters (e.g. clearCheckpoint) - * - * @param projectId - * @param jobId - * @param executionId - * @return - */ - private def continueJobExecution(projectId: String, jobId: String, executionId: String): Route = { - post { - entity(as[Option[ExecuteJobTask]]) { executeJobTask => - complete { - executionService.runJob(projectId, jobId, Some(executionId), executeJobTask) map { _ => - StatusCodes.OK - } - } - } - } - } - - /** - * Route to stop a job (i.e. all the mappings included inside a job) - * - * @param jobId Identifier of the job - * @return - */ - private def stopJobExecution(jobId: String, executionId: String): Route = { - delete { - complete { - executionService.stopJobExecution(jobId, executionId).map(_ => StatusCodes.OK) - } - } - } - - /** - * Route to deschedule a mapping job execution. - * - * @param jobId Identifier of the job - * @param executionId Identifier of job execution - * @return - */ - private def descheduleJobExecution(jobId: String, executionId: String): Route = { - delete { - complete { - executionService.descheduleJobExecution(jobId, executionId).map(_ => StatusCodes.OK) - } - } - } - - /** - * Route to stop an individual mapping task inside a job. - * - * @param jobId Identifier of the job containing the mapping. - * @param mappingTaskName Name of the mappingTask to be stopped - * @return - */ - private def stopMappingExecution(jobId: String, executionId: String, mappingTaskName: String): Route = { - delete { - complete { - executionService.stopMappingExecution(jobId, executionId, mappingTaskName).map(_ => StatusCodes.OK) - } - } - } -} - -object JobEndpoint { - val SEGMENT_JOB = "jobs" - val SEGMENT_RUN = "run" - val SEGMENT_STATUS = "status" - val SEGMENT_EXECUTIONS = "executions" - val SEGMENT_TEST = "test" - val SEGMENT_STOP = "stop" - val SEGMENT_DESCHEDULE = "deschedule" - val SEGMENT_MAPPINGS = "mappings" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingContextEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingContextEndpoint.scala deleted file mode 100644 index a4c662a1..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingContextEndpoint.scala +++ /dev/null @@ -1,235 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.headers.RawHeader -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.ignifyr.server.endpoint.MappingContextEndpoint.{ - ATTACHMENT, - SEGMENT_CONTENT, - SEGMENT_CONTEXTS, - SEGMENT_FILE, - SEGMENT_HEADER -} -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.mappingContext.IMappingContextRepository -import io.ignifyr.server.service.MappingContextService - -class MappingContextEndpoint(mappingContextRepository: IMappingContextRepository) extends LazyLogging { - - val service: MappingContextService = new MappingContextService(mappingContextRepository) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_CONTEXTS) { - val projectId: String = request.projectId.get - pathEndOrSingleSlash { // Operations on all mapping contexts - getAllMappingContexts(projectId) ~ createMappingContext(request, projectId) - } ~ // Operations on a single mapping contexts identified by its id - pathPrefix(Segment) { id: String => - pathEndOrSingleSlash { - deleteMappingContext(projectId, id) // Delete a mapping context: mapping-contexts/ - } ~ pathPrefix(SEGMENT_CONTENT) { - pathEndOrSingleSlash { - getOrSaveMappingContextContentRoute( - projectId, - id - ) // mapping-contexts//content (paginated operations) - } - } ~ pathPrefix(SEGMENT_HEADER) { // mapping-contexts//header (CSV headers update) - pathEndOrSingleSlash { - updateMappingContextHeaderRoute(projectId, id) - } - } ~ pathPrefix(SEGMENT_FILE) { // mapping-contexts//file (import/export as whole CSV file) - pathEndOrSingleSlash { - uploadDownloadMappingContextRoute(projectId, id) - } - } - } - } - } - - /** - * Route to get all mapping contexts - * @param projectId - * @return - */ - private def getAllMappingContexts(projectId: String): Route = { - get { - complete { - service.getAllMappingContext(projectId) - } - } - } - - /** - * Route to create a new mapping context - * @param request - * @param projectId - * @return - */ - private def createMappingContext(request: IgnifyrRestCall, projectId: String): Route = { - post { // Create a new mapping context definition - val mappingContextId = request.requestEntity.asInstanceOf[HttpEntity.Strict].data.utf8String - complete { - service.createMappingContext(projectId, mappingContextId) map { created => - StatusCodes.Created -> created - } - } - } - } - - /** - * Route to delete a mapping context - * @param projectId - * @param id - * @return - */ - private def deleteMappingContext(projectId: String, id: String): Route = { - delete { - complete { - service.deleteMappingContext(projectId, id) map { _ => - StatusCodes.NoContent - } - } - } - } - - /** - * Route to update the mapping context CSV headers - * - * Headers are passed as a list of `CsvHeader` objects and overwrite the existing headers in the first line of the CSV file. - * The `CsvHeader` class represents each header with both its current name and previously saved name. - * The rows are adjusted to match the new headers if necessary. - * - * e.g. 1: - * Existing headers in a CSV: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "header2", previousName = "header2") - * New headers: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "headerChanged", previousName = "header2") - * - CsvHeader(currentName = "header3", previousName = "header3") - * Rows under "header1" are preserved, rows belonging to "header2" are moved to "headerChanged", and "header3" is added with a default value for each row (``). - * - * e.g. 2: - * Existing headers in a CSV: - * - CsvHeader(currentName = "header1", previousName = "header1") - * - CsvHeader(currentName = "header2", previousName = "header2") - * New headers: - * - CsvHeader(currentName = "header2", previousName = "header2") - * - CsvHeader(currentName = "header3", previousName = "header3") - * Rows under "header1" are removed, rows under "header2" are preserved and shifted to the first column, and "header3" is added as the second column with a default value for each row (``). - * - * @param projectId project id - * @param id mapping context id - * @return - */ - private def updateMappingContextHeaderRoute(projectId: String, id: String): Route = { - post { - entity(as[Seq[CsvHeader]]) { headers => - complete { - service.updateMappingContextHeader(projectId, id, headers) map { _ => - StatusCodes.OK - } - } - } - } - } - - /** - * Route to get or post a paginated mapping context content - * - * File to be updated is found by project id and context id - * It updates/returns only the part of the CSV that corresponds to that page information. e.g: - * If the CSV has 1000 records and the page size is 10, then the first page will have records from 1 to 10 excluding the header. - * If the page number is 2, then the records from 11 to 20 will be returned/updated. - * - * @param projectId project id - * @param id mapping context id - * @return - */ - private def getOrSaveMappingContextContentRoute(projectId: String, id: String): Route = { - - /** - * POST request to update part of the mapping context content - * byteSource contains the CSV content to be updated - * Returns empty body with a total records header in case of success - */ - post { - fileUpload(ATTACHMENT) { case (fileInfo, byteSource) => - parameterMap { queryParams => - complete { - val pageNumber = queryParams.getOrElse("page", "1").toInt - val pageSize = queryParams.getOrElse("size", "10").toInt - service.saveMappingContextContent(projectId, id, byteSource, pageNumber, pageSize) map { totalRecords => - HttpResponse( - StatusCodes.OK, - headers = List(RawHeader("X-Total-Count", totalRecords.toString)) - ) - } - } - } - } - - /** - * GET request to get a paginated mapping context content - * Returns the paginated CSV content with a total records header - */ - } ~ get { - parameterMap { queryParams => - complete { - val pageNumber = queryParams.getOrElse("page", "1").toInt - val pageSize = queryParams.getOrElse("size", "10").toInt - service.getMappingContextContent(projectId, id, pageNumber, pageSize) map { case (byteSource, totalRecords) => - HttpResponse( - StatusCodes.OK, - headers = List(RawHeader("X-Total-Count", totalRecords.toString)), - entity = HttpEntity(ContentTypes.`text/csv(UTF-8)`, byteSource) - ) - } - } - } - - } - } - - /** - * Route to upload/download a mapping context file - * File to be uploaded/downloaded is found by project id and mapping context id - * It uploads/downloads the whole CSV file as a single operation - * Operations with large files may take a long time to complete - * @param projectId project id - * @param id mapping context id - * @return - */ - private def uploadDownloadMappingContextRoute(projectId: String, id: String): Route = { - post { - fileUpload(ATTACHMENT) { case (fileInfo, byteSource) => - complete { - service.uploadMappingContextFile(projectId, id, byteSource) map { _ => - StatusCodes.OK - } - } - } - } ~ get { - complete { - service.downloadMappingContextFile(projectId, id) map { byteSource => - HttpResponse(StatusCodes.OK, entity = HttpEntity(ContentTypes.`text/csv(UTF-8)`, byteSource)) - } - } - } - } - -} - -object MappingContextEndpoint { - val SEGMENT_CONTEXTS = "mapping-contexts" - val SEGMENT_CONTENT = "content" - val SEGMENT_HEADER = "header" - val SEGMENT_FILE = "file" - val ATTACHMENT = "attachment" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingEndpoint.scala deleted file mode 100644 index 66c23793..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MappingEndpoint.scala +++ /dev/null @@ -1,92 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.model.FhirMapping -import io.ignifyr.server.common.model.{ResourceNotFound, IgnifyrRestCall} -import io.ignifyr.server.endpoint.MappingEndpoint.SEGMENT_MAPPINGS -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.service.MappingService - -class MappingEndpoint(mappingRepository: IMappingRepository, jobRepository: IJobRepository) extends LazyLogging { - - val service: MappingService = new MappingService(mappingRepository, jobRepository) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_MAPPINGS) { - val projectId: String = request.projectId.get - pathEndOrSingleSlash { // Operations on all mappings - getAllMappings(projectId) ~ createMapping(projectId) - } ~ // Operations on a single mapping identified by its id - pathPrefix(Segment) { id: String => - getMapping(projectId, id) ~ putMapping(projectId, id) ~ deleteMapping(projectId, id) - } - } - } - - private def getAllMappings(projectId: String): Route = { - get { - complete { - service.getAllMappings(projectId) - } - } - } - - private def createMapping(projectId: String): Route = { - post { // Create a new mapping definition - entity(as[FhirMapping]) { fhirMapping => - complete { - service.createMapping(projectId, fhirMapping) map { created => - StatusCodes.Created -> created - } - } - } - } - } - - private def getMapping(projectId: String, mappingId: String): Route = { - get { - complete { - service.getMapping(projectId, mappingId) map { - case Some(fhirMapping) => StatusCodes.OK -> fhirMapping - case None => - StatusCodes.NotFound -> { - throw ResourceNotFound("Mapping not found", s"Mapping with name $mappingId not found") - } - } - } - } - } - - private def putMapping(projectId: String, mappingId: String): Route = { - put { - entity(as[FhirMapping]) { fhirMapping => - complete { - service.updateMapping(projectId, mappingId, fhirMapping) map { res: FhirMapping => - StatusCodes.OK -> res - } - } - } - } - } - - private def deleteMapping(projectId: String, mappingId: String): Route = { - delete { - complete { - service.deleteMapping(projectId, mappingId) map { _ => - StatusCodes.NoContent - } - } - } - } - -} - -object MappingEndpoint { - val SEGMENT_MAPPINGS = "mappings" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MetadataEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MetadataEndpoint.scala deleted file mode 100644 index 6878450c..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/MetadataEndpoint.scala +++ /dev/null @@ -1,54 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.server.common.config.WebServerConfig -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.ignifyr.server.common.spi.IgnifyrServerExtension -import io.ignifyr.server.endpoint.MetadataEndpoint.SEGMENT_METADATA -import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig -import io.ignifyr.server.service.MetadataService - -/** - * Endpoint to return metadata of the server. - * */ -class MetadataEndpoint( - ignifyrEngineConfig: IgnifyrEngineConfig, - webServerConfig: WebServerConfig, - fhirDefinitionsConfig: FhirDefinitionsConfig, - serverExtensions: Seq[IgnifyrServerExtension] -) extends LazyLogging { - - val service: MetadataService = new MetadataService( - ignifyrEngineConfig, - webServerConfig, - fhirDefinitionsConfig, - serverExtensions - ) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_METADATA) { - pathEndOrSingleSlash { - getMetadata - } - } - } - - /** - * Returns the documentations of FhirPath functions. - * */ - private def getMetadata: Route = { - get { - complete { - service.getMetadata - } - } - } -} - -object MetadataEndpoint { - val SEGMENT_METADATA = "metadata" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ProjectEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ProjectEndpoint.scala deleted file mode 100644 index 2c09e17e..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ProjectEndpoint.scala +++ /dev/null @@ -1,162 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.{ResourceNotFound, IgnifyrRestCall} -import io.ignifyr.server.common.spi.IgnifyrServerExtension -import io.ignifyr.server.endpoint.ProjectEndpoint.SEGMENT_PROJECTS -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.model.Project -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.repository.mappingContext.IMappingContextRepository -import io.ignifyr.server.repository.project.IProjectRepository -import io.ignifyr.server.repository.schema.ISchemaRepository -import io.ignifyr.server.service.ProjectService -import org.json4s.JObject - -import scala.concurrent.Future - -/** - * Endpoints to manage projects. - * */ -class ProjectEndpoint( - schemaRepository: ISchemaRepository, - mappingRepository: IMappingRepository, - jobRepository: IJobRepository, - mappingContextRepository: IMappingContextRepository, - projectRepository: IProjectRepository, - serverExtensions: Seq[IgnifyrServerExtension] -) extends LazyLogging { - - val service: ProjectService = - new ProjectService(projectRepository, jobRepository, mappingRepository, mappingContextRepository, schemaRepository) - val schemaDefinitionEndpoint: SchemaDefinitionEndpoint = - new SchemaDefinitionEndpoint(schemaRepository, mappingRepository, serverExtensions) - val mappingEndpoint: MappingEndpoint = new MappingEndpoint(mappingRepository, jobRepository) - val jobEndpoint: JobEndpoint = new JobEndpoint(jobRepository, mappingRepository, schemaRepository) - val mappingContextEndpoint: MappingContextEndpoint = new MappingContextEndpoint(mappingContextRepository) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_PROJECTS) { - // operations on all projects - pathEndOrSingleSlash { - createProjectRoute() ~ getProjectsRoute - } ~ // operations on individual projects - pathPrefix(Segment) { projectId: String => - getProjectRoute(projectId) ~ patchProjectRoute(projectId) ~ deleteProjectRoute(projectId) ~ { - val projectExists: Future[Option[Project]] = service.getProject(projectId) - onSuccess(projectExists) { - case None => - complete { - StatusCodes.NotFound -> { - throw ResourceNotFound("Project not found", s"Project with id $projectId not found") - } - } - case Some(_) => { - request.projectId = Some(projectId) - schemaDefinitionEndpoint.route(request) ~ mappingEndpoint.route(request) ~ jobEndpoint.route( - request - ) ~ mappingContextEndpoint.route(request) - } - } - } - } - } - } - - /** - * Route to create a project - * - * @return - */ - private def createProjectRoute(): Route = { - post { - entity(as[Project]) { project => - complete { - service.createProject(project) map { createdDefinition => - StatusCodes.Created -> createdDefinition - } - } - } - } - } - - /** - * Route to get all projects - * - * @return - */ - private def getProjectsRoute: Route = { - get { - complete { - service.getAllProjects - } - } - } - - /** - * Route to get a project - * - * @param projectId id of project - * @return - */ - private def getProjectRoute(projectId: String): Route = { - pathEndOrSingleSlash { - get { - complete { - service.getProject(projectId) map { - case Some(project) => StatusCodes.OK -> project - case None => - StatusCodes.NotFound -> { - throw ResourceNotFound("Project not found", s"Project with id $projectId not found") - } - } - } - } - } - } - - /** - * Route to patch a project - * - * @param projectId id of project - * @return - */ - private def patchProjectRoute(projectId: String): Route = { - patch { - entity(as[JObject]) { project => - complete { - service.updateProject(projectId, project) map { updatedProject => - StatusCodes.OK -> updatedProject - } - } - } - } - } - - /** - * Route to delete a project - * - * @param projectId id of project to be deleted - * @return - */ - private def deleteProjectRoute(projectId: String): Route = { - pathEndOrSingleSlash { - delete { - complete { - service.removeProject(projectId) map { _ => - StatusCodes.NoContent - } - } - } - } - } -} - -object ProjectEndpoint { - val SEGMENT_PROJECTS = "projects" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ReloadEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ReloadEndpoint.scala deleted file mode 100644 index d2dcd949..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/ReloadEndpoint.scala +++ /dev/null @@ -1,50 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Directives.{complete, get, pathEndOrSingleSlash, pathPrefix} -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.IgnifyrRestCall -import io.ignifyr.server.endpoint.ReloadEndpoint.SEGMENT_RELOAD -import io.ignifyr.server.repository.{FolderDBInitializer, IRepositoryManager} -import io.ignifyr.server.repository.job.JobFolderRepository -import io.ignifyr.server.repository.mapping.ProjectMappingFolderRepository -import io.ignifyr.server.repository.mappingContext.MappingContextFolderRepository -import io.ignifyr.server.repository.schema.SchemaFolderRepository -import io.ignifyr.server.repository.terminology.TerminologySystemFolderRepository -import io.ignifyr.server.service.ReloadService - -/** - * Endpoint to reload resources from the file system. - * */ -class ReloadEndpoint(repositoryManager: IRepositoryManager) extends LazyLogging { - - val reloadService: ReloadService = new ReloadService(repositoryManager) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_RELOAD) { - pathEndOrSingleSlash { - reloadResources - } - } - } - - /** - * Route to reload all resources - * @return - */ - private def reloadResources: Route = { - get { - complete { - reloadService.reloadResources() map { _ => - StatusCodes.NoContent - } - } - } - } -} - -object ReloadEndpoint { - val SEGMENT_RELOAD = "reload" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/SchemaDefinitionEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/SchemaDefinitionEndpoint.scala deleted file mode 100644 index 5a74a446..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/SchemaDefinitionEndpoint.scala +++ /dev/null @@ -1,281 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.endpoint.SchemaDefinitionEndpoint.{ - SEGMENT_IMPORT, - SEGMENT_IMPORT_ZIP, - SEGMENT_INFER, - SEGMENT_SCHEMAS -} -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.server.model.{ImportSchemaSettings, InferTask} -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.server.common.model.{BadRequest, InternalError, ResourceNotFound, IgnifyrRestCall} -import io.ignifyr.server.common.spi.{IgnifyrServerExtension, SchemaImportSink} -import io.onfhir.api.Resource -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.repository.schema.ISchemaRepository -import io.ignifyr.server.service.SchemaDefinitionService -import akka.http.scaladsl.server.directives.FileInfo -import io.ignifyr.server.util.FileOperations - -import java.io.File -import java.nio.file.Files -import scala.util.{Failure, Success} - -class SchemaDefinitionEndpoint( - schemaRepository: ISchemaRepository, - mappingRepository: IMappingRepository, - serverExtensions: Seq[IgnifyrServerExtension] -) extends LazyLogging { - - val service: SchemaDefinitionService = new SchemaDefinitionService(schemaRepository, mappingRepository) - - // Module-specific schema import routes (e.g. the REDCap data-dictionary import), contributed by - // installed server extensions; imported schemas are persisted through the service. - private val schemaImportSink: SchemaImportSink = (projectId, schemas) => - service.saveImportedSchemas(projectId, schemas) - private val extensionImportRoutes: Seq[IgnifyrRestCall => Route] = - serverExtensions.flatMap(_.schemaImportRoutes(schemaImportSink)) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_SCHEMAS) { - val projectId: String = request.projectId.get - pathEndOrSingleSlash { - parameterMap { queryParams => - queryParams.get(SchemaDefinitionEndpoint.QUERY_PARAM_URL) match { - case Some(url) => getSchemaByUrl(url, queryParams.get(SchemaDefinitionEndpoint.QUERY_PARAM_TYPE)) - case None => - getAllSchemas(request) ~ createSchema( - projectId, - queryParams.getOrElse("format", SchemaFormats.SIMPLE_STRUCTURE_DEFINITION) - ) // Operations on all schemas - } - } - } ~ pathPrefix(SEGMENT_INFER) { // infer a schema - inferSchema() - } ~ concat(extensionImportRoutes.map(_(request)): _*) ~ // module-specific schema imports - pathPrefix(SEGMENT_IMPORT) { // import a schema from a Fhir Server - importFromFhirServer(projectId) - } ~ pathPrefix(SEGMENT_IMPORT_ZIP) { - importFromZipOfFHIRProfiles(projectId) - } ~ pathPrefix(Segment) { schemaId: String => // Operations on a single schema identified by its id - getSchema(projectId, schemaId) ~ updateSchema(projectId, schemaId) ~ deleteSchema(projectId, schemaId) - } - } - } - - private def getAllSchemas(request: IgnifyrRestCall): Route = { - get { - complete(service.getAllSchemas(request.projectId.get)) - } - } - - /** - * Create a new schema with the given body - * @param projectId Id of the project in which the schemas will be created - * @param format format of the schema in the request, there are two options StructureDefinition and SimpleStructureDefinition - * @return the SchemaDefinition of the created schema - */ - private def createSchema(projectId: String, format: String): Route = { - post { // Create a new schema definition - // If the schema is in the form of StructureDefinition, convert into SimpleStructureDefinition and save - if (format == SchemaFormats.STRUCTURE_DEFINITION) { - entity(as[Resource]) { schemaStructureDefinition => - complete { - service.createSchemaFromStructureDefinition(projectId, schemaStructureDefinition) - } - } - } else { - entity(as[SchemaDefinition]) { schemaDefinition => - complete { - service.createSchema(projectId, schemaDefinition) map { createdDefinition => - StatusCodes.Created -> createdDefinition - } - } - } - } - } - } - - private def getSchema(projectId: String, schemaId: String): Route = { - get { - parameterMap { queryParams => - complete { - // Requested format of the schema: "StructureDefinition" or "SimpleStructureDefinition" - val format: String = queryParams.getOrElse("format", SchemaFormats.SIMPLE_STRUCTURE_DEFINITION) - // Send structure definition for the user to export - if (format == SchemaFormats.STRUCTURE_DEFINITION) { - service.getSchemaAsStructureDefinition(projectId, schemaId) map { - case Some(schemaStructureDefinition) => StatusCodes.OK -> schemaStructureDefinition - case None => { - throw ResourceNotFound("Schema not found", s"Schema definition with name $schemaId not found") - } - } - } - // Send simple structure definition for general use in frontend - else { - service.getSchema(projectId, schemaId) map { - case Some(schemaSimpleStructureDefinition) => StatusCodes.OK -> schemaSimpleStructureDefinition - case None => - StatusCodes.NotFound -> { - throw ResourceNotFound("Schema not found", s"Schema definition with name $schemaId not found") - } - } - } - } - } - } - } - - private def getSchemaByUrl(url: String, returnType: Option[String]): Route = { - get { - complete { - service.getSchemaByUrl(url) map { - case Some(schemaDefinition) => - returnType match { - case Some(SchemaFormats.SIMPLE_STRUCTURE_DEFINITION) => - // Return only the element definitions within the schema, excluding metadata such as name and description. - // This is useful when only the element-level details are needed, without additional schema metadata. - StatusCodes.OK -> schemaDefinition.fieldDefinitions.get - case _ => - // Return the full schema definition, including all metadata and element definitions. - StatusCodes.OK -> schemaDefinition - } - case None => - StatusCodes.NotFound -> { - throw ResourceNotFound("Schema not found", s"Schema definition with url $url not found") - } - } - } - } - } - - private def updateSchema(projectId: String, schemaId: String): Route = { - put { - entity(as[SchemaDefinition]) { schemaDefinition => - complete { - service.putSchema(projectId, schemaId, schemaDefinition) map { res: SchemaDefinition => - StatusCodes.OK -> res - } - } - } - } - } - - private def deleteSchema(projectId: String, schemaId: String): Route = { - delete { - complete { - service.deleteSchema(projectId, schemaId) map { _ => - StatusCodes.NoContent - } - } - } - } - - /** - * Route to infer a schema - * */ - private def inferSchema(): Route = { - post { - entity(as[InferTask]) { inferTask => - complete { - service.inferSchema(inferTask) map { - case Some(schemaDefinition) => StatusCodes.OK -> schemaDefinition - case None => - StatusCodes.BadRequest -> { - throw BadRequest("Schema inferring problem", s"Schema cannot be inferred") - } - } - } - } - } - } - - /** - * Route to import a schema i.e. FHIR Structure Definition from the given FHIR Server. - * */ - private def importFromFhirServer(projectId: String): Route = { - post { - entity(as[ImportSchemaSettings]) { importSchemaSettings => - complete { - service.importSchema(projectId, importSchemaSettings) - } - } - } - } - - /** - * Route to import FHIR profiles (structure definitions) from a ZIP file uploaded to the server. - * - * This route handles the uploading and processing of a ZIP file containing FHIR structure definitions. - * It parses the ZIP file, validates and processes the JSON resources, and then creates schemas in the - * specified project. - * - * @param projectId The ID of the project where the FHIR profiles will be imported. - * @return A Route that handles POST requests to import the schema from the ZIP file. - */ - private def importFromZipOfFHIRProfiles(projectId: String): Route = { - post { - storeUploadedFile("file", createTempFile) { case (_, file) => - val zipProcessingResult = FileOperations.processZipFile(file.toPath) - onComplete(zipProcessingResult) { - case Success(result) => - complete( - service.createSchemas(projectId, result) - ) - case Failure(ex) => - throw InternalError("Processing of ZIP file failed!", s"Cannot process the ZIP file: ${ex.getMessage}") - } - } - } - } - - /** - * Creates a temporary file with the given file name and a ".tmp" extension. - * - * This method creates a temporary file in the system's default temporary-file directory. - * The file is marked for deletion when the JVM exits, ensuring that it will be cleaned up automatically. - * - * @param fileInfo Information about the uploaded file, including the file name. - * @return A temporary file object. - */ - private def createTempFile(fileInfo: FileInfo): File = { - val tempFile = Files.createTempFile(fileInfo.fileName, ".tmp").toFile - tempFile.deleteOnExit() - tempFile - } -} - -object SchemaDefinitionEndpoint { - val SEGMENT_SCHEMAS = "schemas" - val SEGMENT_INFER = "infer" - val SEGMENT_IMPORT = "import" - val SEGMENT_IMPORT_ZIP = "import-zip" - val QUERY_PARAM_TYPE = "type" - val QUERY_PARAM_URL = "url" -} - -/** - * The schema formats available for POST and GET schema methods - */ -object SchemaFormats { - - /** - * Specifies that the format of the schema to be created or retrieved follows the FHIR StructureDefinition resource format. - * For more information on this format, see the FHIR StructureDefinition documentation at: - * https://www.hl7.org/fhir/structuredefinition.html. - */ - val STRUCTURE_DEFINITION = "StructureDefinition" - - /** - * Specifies a format that retrieves only the element definitions of a schema, where each element is represented - * by a {@link io.ignifyr.common.model.SimpleStructureDefinition}. This format provides a simplified structure for the schema's elements. - */ - val SIMPLE_STRUCTURE_DEFINITION = "SimpleStructureDefinition" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpoint.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpoint.scala deleted file mode 100644 index c59d702b..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpoint.scala +++ /dev/null @@ -1,163 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.onfhir.definitions.common.model.Json4sSupport._ -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.{ResourceNotFound, IgnifyrRestCall} -import io.ignifyr.server.endpoint.TerminologyServiceManagerEndpoint._ -import io.ignifyr.server.model.TerminologySystem -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.terminology.ITerminologySystemRepository -import io.ignifyr.server.repository.terminology.codesystem.ICodeSystemRepository -import io.ignifyr.server.repository.terminology.conceptmap.IConceptMapRepository -import io.ignifyr.server.service.terminology.TerminologySystemService - -import scala.concurrent.Future - -class TerminologyServiceManagerEndpoint( - terminologySystemRepository: ITerminologySystemRepository, - conceptMapRepository: IConceptMapRepository, - codeSystemRepository: ICodeSystemRepository, - mappingJobRepository: IJobRepository -) extends LazyLogging { - - private val terminologySystemService: TerminologySystemService = - new TerminologySystemService(terminologySystemRepository, mappingJobRepository) - - private val conceptMapEndpoint: ConceptMapEndpoint = new ConceptMapEndpoint(conceptMapRepository) - private val codeSystemEndpoint: CodeSystemEndpoint = new CodeSystemEndpoint(codeSystemRepository) - - def route(request: IgnifyrRestCall): Route = { - pathPrefix(SEGMENT_TERMINOLOGY) { - // operations on all terminology systems - pathEndOrSingleSlash { - createTerminologySystemRoute() ~ getAllTerminologySystemsRoute - } ~ // operations on individual terminology systems - pathPrefix(Segment) { terminologySystemId: String => - pathEndOrSingleSlash { - getTerminologySystemRoute(terminologySystemId) ~ putTerminologySystemRoute( - terminologySystemId - ) ~ deleteTerminologySystemRoute(terminologySystemId) - } ~ { - val terminologyExists: Future[Option[TerminologySystem]] = - terminologySystemService.getTerminologySystem(terminologySystemId) - onSuccess(terminologyExists) { - case None => - complete { - StatusCodes.NotFound -> { - throw ResourceNotFound( - "Terminology system not found", - s"TerminologySystem with id $terminologySystemId not found" - ) - } - } - case Some(_) => { - request.terminologyId = Some(terminologySystemId) - conceptMapEndpoint.route(request) ~ codeSystemEndpoint.route(request) - } - } - } - } - } - } - - /** - * Route to get all local TerminologySystems - * - * @return - */ - private def getAllTerminologySystemsRoute: Route = { - get { - complete { - terminologySystemService.getAllMetadata - } - } - } - - /** - * Route to create a local TerminologySystem - * - * @return - */ - private def createTerminologySystemRoute(): Route = { - post { - entity(as[TerminologySystem]) { terminologySystem => - complete { - terminologySystemService.createTerminologySystem(terminologySystem) map { createdTerminologySystem => - StatusCodes.Created -> createdTerminologySystem - } - } - } - } - } - - /** - * Route to get a local TerminologySystem - * - * @param terminologySystemId id of local TerminologySystem - * @return - */ - private def getTerminologySystemRoute(terminologySystemId: String): Route = { - get { - complete { - terminologySystemService.getTerminologySystem(terminologySystemId) map { - case Some(terminologySystem) => StatusCodes.OK -> terminologySystem - case None => - StatusCodes.NotFound -> { - throw ResourceNotFound( - "Terminology system not found", - s"TerminologySystem with id $terminologySystemId not found." - ) - } - } - } - } - } - - /** - * Route to update a local TerminologySystem - * - * @param terminologySystemId id of local TerminologySystem - * @return - */ - private def putTerminologySystemRoute(terminologySystemId: String): Route = { - put { - entity(as[TerminologySystem]) { newTerminologySystem => - complete { - terminologySystemService.updateTerminologySystem(terminologySystemId, newTerminologySystem) map { - updatedTerminologySystem => - StatusCodes.OK -> updatedTerminologySystem - } - } - } - } - } - - /** - * Route to delete a local TerminologySystem - * - * @param terminologySystemId id of local TerminologySystem - * @return - */ - private def deleteTerminologySystemRoute(terminologySystemId: String): Route = { - delete { - complete { - terminologySystemService.deleteTerminologySystem(terminologySystemId) map { _ => - StatusCodes.NoContent - } - } - } - } - -} - -object TerminologyServiceManagerEndpoint { - val SEGMENT_TERMINOLOGY = "terminologies" - val SEGMENT_CONTENT = "content" - val SEGMENT_HEADER = "header" - val SEGMENT_FILE = "file" - val ATTACHMENT = "attachment" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/ExecuteJobTask.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/ExecuteJobTask.scala deleted file mode 100644 index e554217d..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/ExecuteJobTask.scala +++ /dev/null @@ -1,7 +0,0 @@ -package io.ignifyr.server.model - -/** - * The data transfer object to keep mapping names to be executed and corresponding mapping error handling. - * @param mappingTaskNames mappingTask names of the mappingTask's that to be executed - */ -case class ExecuteJobTask(clearCheckpoints: Boolean, mappingTaskNames: Option[Seq[String]] = None) diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/FilePathNode.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/FilePathNode.scala deleted file mode 100644 index d494de4c..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/FilePathNode.scala +++ /dev/null @@ -1,10 +0,0 @@ -package io.ignifyr.server.model - -/** - * Represents a node for each file in the file system - * @param label The name of the file - * @param isFolder True if the file is a folder - * @param path file path of the node - * @param children The children of the folder - */ -case class FilePathNode(label: String, isFolder: Boolean, path: String, children: List[FilePathNode]) diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/ImportSchemaSettings.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/ImportSchemaSettings.scala deleted file mode 100644 index f55fcf26..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/ImportSchemaSettings.scala +++ /dev/null @@ -1,16 +0,0 @@ -package io.ignifyr.server.model - -import io.onfhir.client.model.IFhirRepositorySecuritySettings - -/** - * Represents the settings required for importing a schema from a FHIR server. - * - * @param baseUrl The base URL of the FHIR server from which to import the schema. - * @param resourceId The ID of the resource representing the schema i.e. StructureDefinition to be imported. - * @param securitySettings Optional security settings for accessing the FHIR server, if applicable. - */ -case class ImportSchemaSettings( - baseUrl: String, - resourceId: String, - securitySettings: Option[IFhirRepositorySecuritySettings] -) diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/InferTask.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/InferTask.scala deleted file mode 100644 index 6d3b6ced..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/InferTask.scala +++ /dev/null @@ -1,16 +0,0 @@ -package io.ignifyr.server.model - -import io.ignifyr.engine.model.{MappingJobSourceSettings, MappingSourceBinding} - -/** - * Infer task instance for getting source settings and source binding settings. - * InferTask object is implemented only for SQL sources and file system sources. - * @param name alias for source binding - * @param mappingJobSourceSettings connection details for data source (e.g. JDBC connection details, file path) - * @param sourceBinding mapping task source configuration (e.g. file name, table name, query) - */ -case class InferTask( - name: String, - mappingJobSourceSettings: Map[String, MappingJobSourceSettings], - sourceBinding: MappingSourceBinding -) diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/Metadata.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/Metadata.scala deleted file mode 100644 index 2c580e45..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/Metadata.scala +++ /dev/null @@ -1,64 +0,0 @@ -package io.ignifyr.server.model - -/** - * Model that represents the metadata of the Ignifyr server. - * - * @param name The name of the server. - * @param description A description of the server. - * @param version The current version of the server. - * @param fhirDefinitionsVersion The major FHIR version of definitions (profiles, valuesets, codesystems) supported by the server. - * @param ignifyrRedcapVersion The optional Ignifyr-Redcap server version. - * @param definitionsRootUrls Optional list of root URLs for definitions. - * @param schemasFhirVersion The FHIR version used for schemas. - * @param repositoryNames The configured repository names. - * @param archiving The archiving configuration. - * @param executionConfigurations The list of configurations used during the execution of mappings. - */ -case class Metadata( - name: String, - description: String, - version: String, - fhirDefinitionsVersion: String, - ignifyrRedcapVersion: Option[String], - definitionsRootUrls: Option[Seq[String]], - schemasFhirVersion: String, - repositoryNames: RepositoryNames, - archiving: Archiving, - environmentVariables: Map[String, String], - executionConfigurations: Seq[MappingExecutionConfiguration] -) - -/** - * Represents the configured repository names included in the metadata. - * - * @param mappings The path to the folder where the mappings are kept. - * @param schemas The path to the folder where the schema definitions are kept. - * @param contexts The path to the folder where the mapping context files are kept. - * @param jobs The path to the folder where the job definitions are kept. - * @param terminologySystems The path to the folder where the terminology system definitions are kept. - */ -case class RepositoryNames( - mappings: String, - schemas: String, - contexts: String, - jobs: String, - terminologySystems: String -) - -/** - * Archiving configs included inside the metadata. - * - * @param erroneousRecordsFolder The parent folder where the data sources of errors received while running mapping are stored. - * @param archiveFolder The folder path where the archive of the processed source data is stored. - * @param streamArchivingFrequency The period (in milliseconds) to run the archiving task for file streaming jobs. - */ -case class Archiving(erroneousRecordsFolder: String, archiveFolder: String, streamArchivingFrequency: Int) - -/** - * Represents a configuration used during the execution of mappings. - * - * @param name The name of the configuration. - * @param description A brief description of the configuration, explaining its purpose or usage. - * @param value The value assigned to the configuration, specifying its current setting. - */ -case class MappingExecutionConfiguration(name: String, description: String, value: String) diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/Project.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/Project.scala deleted file mode 100644 index ea232e13..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/Project.scala +++ /dev/null @@ -1,79 +0,0 @@ -package io.ignifyr.server.model - -import io.ignifyr.engine.model.{FhirMapping, FhirMappingJob} -import org.json4s.{JArray, JObject, JString} - -import java.util.UUID - -import io.onfhir.definitions.common.model.SchemaDefinition - -/** - * Definition of a project which holds relevant schemas, mapping, mapping-jobs, concept maps etc. - * - * @param id Unique identifier for the project - * @param name Project name - * @param description Description of the project - * @param schemaUrlPrefix Prefix (beginning) of the URLs to be used while creating schema definitions within this project. - * @param mappingUrlPrefix Prefix (beginning) of the URLs to be used while creating mapping definitions within this project. - * @param schemas Schemas defined in this project - * @param mappingContexts Identifiers of the mapping contexts defined in this project - * @param mappingJobs Mapping jobs defined in this project - */ -case class Project( - id: String = UUID.randomUUID().toString, - name: String, - description: Option[String] = None, - schemaUrlPrefix: Option[String] = None, - mappingUrlPrefix: Option[String] = None, - schemas: Seq[SchemaDefinition] = Seq.empty, - mappings: Seq[FhirMapping] = Seq.empty, - mappingContexts: Seq[String] = Seq.empty, - mappingJobs: Seq[FhirMappingJob] = Seq.empty -) { - - /** - * Extracts the project metadata to be written to the metadata file. - * - * @return - */ - def getMetadata(): JObject = { - JObject( - List( - "id" -> JString(this.id), - "name" -> JString(this.name), - "description" -> JString(this.description.getOrElse("")), - "schemaUrlPrefix" -> JString(this.schemaUrlPrefix.getOrElse("")), - "mappingUrlPrefix" -> JString(this.mappingUrlPrefix.getOrElse("")), - "schemas" -> JArray( - List( - this.schemas.map(_.getMetadata()): _* - ) - ), - "mappings" -> JArray( - List( - this.mappings.map(_.getMetadata): _* - ) - ), - "mappingContexts" -> JArray( - List( - this.mappingContexts.map(cid => JString(cid)): _* - ) - ), - "mappingJobs" -> JArray( - List( - this.mappingJobs.map(_.getMetadata()): _* - ) - ) - ) - ) - } -} - -/** - * Editable fields of a project via Patch API - * */ -object ProjectEditableFields { - val DESCRIPTION = "description" - val SCHEMA_URL_PREFIX = "schemaUrlPrefix" - val MAPPING_URL_PREFIX = "mappingUrlPrefix" -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/TerminologySystem.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/TerminologySystem.scala deleted file mode 100644 index e4f4bf66..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/TerminologySystem.scala +++ /dev/null @@ -1,48 +0,0 @@ -package io.ignifyr.server.model - -import io.ignifyr.engine.model.{CodeSystemFile, ConceptMapFile} -import io.ignifyr.server.model.TerminologySystem.{TerminologyCodeSystem, TerminologyConceptMap} - -import java.util.UUID - -/** - * Local Terminology System - * - * @param id unique id - * @param name name of the terminology - * @param description description of the terminology - */ -case class TerminologySystem( - id: String = UUID.randomUUID().toString, - name: String, - description: String, - conceptMaps: Seq[TerminologyConceptMap] = Seq.empty, - codeSystems: Seq[TerminologyCodeSystem] = Seq.empty -) - -object TerminologySystem { - type TerminologyConceptMap = ConceptMapFile - type TerminologyCodeSystem = CodeSystemFile -} - -// -///** -// * Local Terminology ConceptMap -// * -// * @param id unique id -// * @param name name of the terminology concept map -// * @param conceptMapUrl url of the terminology concept map -// * @param sourceValueSetUrl source value set url of the terminology concept map -// * @param targetValueSetUrl target value set url of the terminology concept map -// */ -//case class TerminologyConceptMap(id: String = UUID.randomUUID().toString, override val name: String, override val conceptMapUrl: String, override val sourceValueSetUrl: String, override val targetValueSetUrl: String) -// extends ConceptMapFile(name: String, conceptMapUrl: String, sourceValueSetUrl: String, targetValueSetUrl: String) -// -///** -// * Local Terminology CodeSystem -// * -// * @param id unique id -// * @param name name of the terminology code system -// * @param codeSystem code system of the terminology code system -// */ -//case class TerminologyCodeSystem(id: String = UUID.randomUUID().toString, name: String, codeSystem: String) diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/TestResourceCreationRequest.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/TestResourceCreationRequest.scala deleted file mode 100644 index 6297862e..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/TestResourceCreationRequest.scala +++ /dev/null @@ -1,31 +0,0 @@ -package io.ignifyr.server.model - -import io.ignifyr.engine.model.FhirMappingTask - -/** - * FhirMappingTask and resource filter to test a mapping - * - * @param fhirMappingTask task to be used for testing - * @param resourceFilter resource filter for testing - */ -case class TestResourceCreationRequest(fhirMappingTask: FhirMappingTask, resourceFilter: ResourceFilter) - -/** - * Resource filter config for testing a mapping - * - * @param numberOfRows number of rows to select - * @param order order of rows to select - */ -case class ResourceFilter(numberOfRows: Int, order: String) - -/** - * Available row selection orders - */ -object RowSelectionOrder { - final val START = "start" - final val RANDOM = "random" - - def isValid(order: String): Boolean = allValues().contains(order) - - private def allValues(): Set[String] = Set(START, RANDOM) -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/model/csv/CsvHeader.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/model/csv/CsvHeader.scala deleted file mode 100644 index 97a532c3..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/model/csv/CsvHeader.scala +++ /dev/null @@ -1,9 +0,0 @@ -package io.ignifyr.server.model.csv - -/** - * Represents a CSV header with a new name and a previously saved name. - * @param currentName The current name of the header. - * @param previousName The previously saved name of the header, used for column name change without data loss - * If a new header is desired to be created, saved name may be any placeholder - */ -case class CsvHeader(currentName: String, previousName: String) \ No newline at end of file diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderDBInitializer.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderDBInitializer.scala deleted file mode 100644 index 6c2770dd..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderDBInitializer.scala +++ /dev/null @@ -1,199 +0,0 @@ -package io.ignifyr.server.repository - -import com.typesafe.scalalogging.Logger -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.model.{FhirMapping, FhirMappingJob} -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.model.Project -import io.ignifyr.server.repository.job.JobFolderRepository -import io.ignifyr.server.repository.mapping.ProjectMappingFolderRepository -import io.ignifyr.server.repository.mappingContext.MappingContextFolderRepository -import io.ignifyr.server.repository.project.ProjectFolderRepository -import io.ignifyr.server.repository.schema.SchemaFolderRepository -import io.ignifyr.server.util.FileOperations -import org.json4s.{JArray, JObject} - -import scala.collection.mutable -import scala.concurrent.duration.DurationInt -import scala.concurrent.{Await, Future} -import scala.language.postfixOps - -/** - * Folder/Directory based database initializer implementation. - * */ -class FolderDBInitializer( - projectFolderRepository: ProjectFolderRepository, - schemaFolderRepository: SchemaFolderRepository, - mappingFolderRepository: ProjectMappingFolderRepository, - mappingJobFolderRepository: JobFolderRepository, - mappingContextRepository: MappingContextFolderRepository -) { - - private val logger: Logger = Logger(this.getClass) - - /** - * Initializes the projects managed by this repository and populates the cache. - */ - def init(): Unit = { - - val file = FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile - - val parsedProjects = if (file.exists()) { - val projects: JArray = FileOperations.readFileIntoJson(file).asInstanceOf[JArray] - projects.arr - .map(p => { - val project: Project = initProjectFromMetadata(p.asInstanceOf[JObject]) - project.id -> project - }) - .toMap - } else { - logger.debug( - s"There does not exist a metadata file (${ProjectFolderRepository.PROJECTS_JSON}) for projects. Creating it..." - ) - file.createNewFile() - // Parse the folder structure of the respective resource and to initialize projects with the resources found. - val projects = initProjectsWithResources() - projects - } - // Inject the parsed projects to the repository - projectFolderRepository.setProjects(parsedProjects) - } - - /** - * Constructs a project based on project metadata stored in the project json file. Resources are resolved via respective repositories. - * - * @param projectMetadata - * @return - */ - private def initProjectFromMetadata(projectMetadata: JObject): Project = { - val id: String = (projectMetadata \ "id").extract[String] - val name: String = (projectMetadata \ "name").extract[String] - val description: Option[String] = (projectMetadata \ "description").extractOpt[String] - val schemaUrlPrefix: Option[String] = (projectMetadata \ "schemaUrlPrefix").extractOpt[String] - val mappingUrlPrefix: Option[String] = (projectMetadata \ "mappingUrlPrefix").extractOpt[String] - val mappingContexts: Seq[String] = (projectMetadata \ "mappingContexts").extract[Seq[String]] - // resolve schemas via the schema repository - val schemaFutures: Future[Seq[Option[SchemaDefinition]]] = Future.sequence( - (projectMetadata \ "schemas") - .asInstanceOf[JArray] - .arr - .map(schemaMetadata => { - val schemaId: String = (schemaMetadata \ "id").extract[String] - schemaFolderRepository - .getSchema(id, schemaId) - .map { - case None => throw new IllegalStateException(s"Failed to retrieve schema with id: $schemaId") - case schema => schema - } - }) - ) - val schemas: Seq[SchemaDefinition] = - Await.result[Seq[Option[SchemaDefinition]]](schemaFutures, 2 seconds).map(_.get) - - // resolve mappings via the mapping repository - val mappingFutures: Future[Seq[Option[FhirMapping]]] = Future.sequence( - (projectMetadata \ "mappings") - .asInstanceOf[JArray] - .arr - .map(mappingMetadata => { - val mappingId: String = (mappingMetadata \ "id").extract[String] - mappingFolderRepository - .getMapping(id, mappingId) - .map { - case None => throw new IllegalStateException(s"Failed to retrieve mapping with id: $mappingId") - case mapping => mapping - } - }) - ) - val mappings: Seq[FhirMapping] = Await.result[Seq[Option[FhirMapping]]](mappingFutures, 2 seconds).map(_.get) - - // resolve mapping jobs via the mapping repository - val mappingJobFutures: Future[Seq[Option[FhirMappingJob]]] = Future.sequence( - (projectMetadata \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .map(jobMetadata => { - val jobId: String = (jobMetadata \ "id").extract[String] - mappingJobFolderRepository - .getJob(id, jobId) - .map { - case None => throw new IllegalStateException(s"Failed to retrieve job with id: $jobId") - case job => job - } - }) - ) - val jobs: Seq[FhirMappingJob] = Await.result[Seq[Option[FhirMappingJob]]](mappingJobFutures, 2 seconds).map(_.get) - - Project(id, name, description, schemaUrlPrefix, mappingUrlPrefix, schemas, mappings, mappingContexts, jobs) - } - - private def dropLastPart(url: String): String = { - // Remove trailing slash if it exists, to handle cases where URL ends with "/" - val cleanedUrl = if (url.endsWith("/")) url.dropRight(1) else url - // Find the last slash and drop everything after it - val lastSlashIndex = cleanedUrl.lastIndexOf('/') - if (lastSlashIndex != -1) cleanedUrl.substring(0, lastSlashIndex + 1) - else url // Return original URL if no slashes found - } - - /** - * Parses the folder structure of project resources and initializes projects with the resources discovered. - * - * @return - */ - private def initProjectsWithResources(): Map[String, Project] = { - // Map keeping the projects. It uses the project name as a key. - val projects: mutable.Map[String, Project] = mutable.Map.empty - - // Parse schemas - schemaFolderRepository.getProjectPairs.foreach { case (projectId, schemaDefinitionList) => - val schemaUrl = schemaDefinitionList.head.url - // If there is no project create a new one. Use id as name as well - val project: Project = projects.get(projectId) match { - case Some(existingProject) => existingProject.copy(schemaUrlPrefix = Some(dropLastPart(schemaUrl))) - case None => Project(id = projectId, name = projectId, schemaUrlPrefix = Some(dropLastPart(schemaUrl))) - } - projects.put(projectId, project.copy(schemas = schemaDefinitionList)) - } - - // Parse mappings - mappingFolderRepository.getProjectPairs.foreach { case (projectId, mappingList) => - val mappingUrl: String = mappingList.head.url - // If there is no project create a new one. Use id as name as well - val project: Project = projects.get(projectId) match { - case Some(existingProject) => existingProject.copy(mappingUrlPrefix = Some(dropLastPart(mappingUrl))) - case None => Project(id = projectId, name = projectId, mappingUrlPrefix = Some(dropLastPart(mappingUrl))) - } - projects.put(projectId, project.copy(mappings = mappingList)) - } - - // Parse mapping jobs - mappingJobFolderRepository.getProjectPairs.foreach { case (projectId, jobList) => - // If there is no project create a new one. Use id as name as well - val project: Project = projects.getOrElse(projectId, Project(id = projectId, name = projectId)) - projects.put(projectId, project.copy(mappingJobs = jobList)) - } - - // Parse mapping contexts - mappingContextRepository.getProjectPairs.foreach { case (projectId, mappingContextIdList) => - // If there is no project create a new one. Use id as name as well - val project: Project = projects.getOrElse(projectId, Project(id = projectId, name = projectId)) - projects.put(projectId, project.copy(mappingContexts = mappingContextIdList)) - } - - projects.toMap - } - - /** - * Removes the projects.json file. - */ - def removeProjectsJsonFile(): Unit = { - val file = FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile - if (file.exists()) { - file.delete() - } - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderRepositoryManager.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderRepositoryManager.scala deleted file mode 100644 index 5298d702..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/FolderRepositoryManager.scala +++ /dev/null @@ -1,63 +0,0 @@ -package io.ignifyr.server.repository - -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.server.repository.job.JobFolderRepository -import io.ignifyr.server.repository.mapping.ProjectMappingFolderRepository -import io.ignifyr.server.repository.mappingContext.MappingContextFolderRepository -import io.ignifyr.server.repository.project.ProjectFolderRepository -import io.ignifyr.server.repository.schema.SchemaFolderRepository -import io.ignifyr.server.repository.terminology.TerminologySystemFolderRepository -import io.ignifyr.server.repository.terminology.codesystem.CodeSystemFolderRepository -import io.ignifyr.server.repository.terminology.conceptmap.ConceptMapFolderRepository - -/** - * Folder/file based implementation of the RepositoryManager where all managed repositories are folder-based. - * - * @param ignifyrEngineConfig - */ -class FolderRepositoryManager(ignifyrEngineConfig: IgnifyrEngineConfig) extends IRepositoryManager { - - override val projectRepository: ProjectFolderRepository = new ProjectFolderRepository(ignifyrEngineConfig) - override val mappingRepository: ProjectMappingFolderRepository = - new ProjectMappingFolderRepository(ignifyrEngineConfig.mappingRepositoryFolderPath, projectRepository) - override val schemaRepository: SchemaFolderRepository = - new SchemaFolderRepository(ignifyrEngineConfig.schemaRepositoryFolderPath, projectRepository) - override val mappingJobRepository: JobFolderRepository = - new JobFolderRepository(ignifyrEngineConfig.jobRepositoryFolderPath, projectRepository) - override val mappingContextRepository: MappingContextFolderRepository = - new MappingContextFolderRepository(ignifyrEngineConfig.mappingContextRepositoryFolderPath, projectRepository) - - override val terminologySystemRepository: TerminologySystemFolderRepository = new TerminologySystemFolderRepository( - ignifyrEngineConfig.terminologySystemFolderPath - ) - override val conceptMapRepository: ConceptMapFolderRepository = new ConceptMapFolderRepository( - ignifyrEngineConfig.terminologySystemFolderPath - ) - override val codeSystemRepository: CodeSystemFolderRepository = new CodeSystemFolderRepository( - ignifyrEngineConfig.terminologySystemFolderPath - ) - - private val folderDBInitializer = new FolderDBInitializer( - projectRepository, - schemaRepository, - mappingRepository, - mappingJobRepository, - mappingContextRepository - ) - - /** - * Initializes the Repository Manager's internal database (the projects.json file) after initialization of - * each individual repository. - */ - override def init(): Unit = { - folderDBInitializer.init() - } - - /** - * Deletes the internal repository database (the projects.json file) for a fresh start (usually after cache invalidate operations) - */ - override def clear(): Unit = { - folderDBInitializer.removeProjectsJsonFile() - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/IRepositoryManager.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/IRepositoryManager.scala deleted file mode 100644 index 5cbf4a4a..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/IRepositoryManager.scala +++ /dev/null @@ -1,35 +0,0 @@ -package io.ignifyr.server.repository - -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.repository.mappingContext.IMappingContextRepository -import io.ignifyr.server.repository.project.IProjectRepository -import io.ignifyr.server.repository.schema.ISchemaRepository -import io.ignifyr.server.repository.terminology.ITerminologySystemRepository -import io.ignifyr.server.repository.terminology.codesystem.ICodeSystemRepository -import io.ignifyr.server.repository.terminology.conceptmap.IConceptMapRepository - -/** - * Manage the repositories throughout Ignifyr - */ -trait IRepositoryManager { - val projectRepository: IProjectRepository - val mappingRepository: IMappingRepository - val schemaRepository: ISchemaRepository - val mappingJobRepository: IJobRepository - val mappingContextRepository: IMappingContextRepository - - val terminologySystemRepository: ITerminologySystemRepository - val conceptMapRepository: IConceptMapRepository - val codeSystemRepository: ICodeSystemRepository - - /** - * Initialize the repository - */ - def init(): Unit - - /** - * Clean-up the repository database - */ - def clear(): Unit -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/IJobRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/IJobRepository.scala deleted file mode 100644 index c79176ee..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/IJobRepository.scala +++ /dev/null @@ -1,72 +0,0 @@ -package io.ignifyr.server.repository.job - -import io.ignifyr.engine.model.FhirMappingJob -import io.ignifyr.engine.repository.ICachedRepository -import io.ignifyr.server.repository.project.IProjectList - -import scala.concurrent.Future - -trait IJobRepository extends ICachedRepository with IProjectList[FhirMappingJob] { - - /** - * Retrieve all jobs of a project - * - * @param projectId project id the jobs belong to - * @return - */ - def getAllJobs(projectId: String): Future[Seq[FhirMappingJob]] - - /** - * Retrieves the jobs referencing the given mapping in their definitions. - * - * @param projectId identifier of project whose jobs will be checked - * @param mappingUrl the url of mapping - * @return the jobs referencing the given mapping in their definitions - */ - def getJobsReferencingMapping(projectId: String, mappingUrl: String): Future[Seq[FhirMappingJob]] - - /** - * Save the job to the repository. - * - * @param projectId project id the job belongs to - * @param job job to save - * @return - */ - def saveJob(projectId: String, job: FhirMappingJob): Future[FhirMappingJob] - - /** - * Get the job by its id - * - * @param projectId project id the job belongs to - * @param jobId job id - * @return - */ - def getJob(projectId: String, jobId: String): Future[Option[FhirMappingJob]] - - /** - * Update the job in the repository - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param job job to save - * @return - */ - def updateJob(projectId: String, jobId: String, job: FhirMappingJob): Future[FhirMappingJob] - - /** - * Delete the job from the repository - * - * @param projectId project id the job belongs to - * @param jobId job id - * @return - */ - def deleteJob(projectId: String, jobId: String): Future[Unit] - - /** - * Deletes all jobs associated with a specific project. - * - * @param projectId The unique identifier of the project for which jobs should be deleted. - */ - def deleteAllJobs(projectId: String): Future[Unit] - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/JobFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/JobFolderRepository.scala deleted file mode 100644 index d5f222f1..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/job/JobFolderRepository.scala +++ /dev/null @@ -1,265 +0,0 @@ -package io.ignifyr.server.repository.job - -import com.typesafe.scalalogging.Logger -import io.onfhir.api.util.IOUtil -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.model.FhirMappingJob -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.engine.util.{FhirMappingJobFormatter, FileUtils} -import io.ignifyr.server.common.model.{AlreadyExists, ResourceNotFound} -import io.ignifyr.server.repository.project.IProjectRepository -import io.ignifyr.server.util.FileOperations -import org.json4s.MappingException -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -import java.io.{File, FileWriter} -import java.nio.charset.StandardCharsets -import scala.collection.mutable -import scala.concurrent.Future -import scala.io.Source - -class JobFolderRepository(jobRepositoryFolderPath: String, projectRepository: IProjectRepository) - extends IJobRepository { - - private val logger: Logger = Logger(this.getClass) - - // In-memory cache to maintain the job definitions. - // project id -> mapping job id -> mapping job - private val jobDefinitions: mutable.Map[String, mutable.Map[String, FhirMappingJob]] = - mutable.Map.empty[String, mutable.Map[String, FhirMappingJob]] - - // Initialize the map for the first time - initMap(jobRepositoryFolderPath) - - /** - * Retrieve all jobs of a given project - * - * @param projectId project id the jobs belong to - * @return - */ - override def getAllJobs(projectId: String): Future[Seq[FhirMappingJob]] = { - Future { - jobDefinitions - .get(projectId) - .map(_.values.toSeq) // If such a project exists, return the jobs as a sequence - .getOrElse(Seq.empty[FhirMappingJob]) // Else, return an empty list - } - } - - /** - * Save the job to the repository. - * - * @param projectId project id the job belongs to - * @param job job to save - * @return - */ - override def saveJob(projectId: String, job: FhirMappingJob): Future[FhirMappingJob] = { - jobDefinitions.get(projectId).flatMap(_.get(job.id)).foreach { _ => - throw AlreadyExists( - "Fhir mapping job already exists.", - s"A job definition with id ${job.id} already exists in the job repository at ${FileUtils.getPath(jobRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // Write to the repository as a new file - getFileForJob(projectId, job.id).flatMap(file => { - val fw = new FileWriter(file) - fw.write(writePretty(job)) - fw.close() - // Update the internal cache of job repository - jobDefinitions.getOrElseUpdate(projectId, mutable.Map.empty[String, FhirMappingJob]).put(job.id, job) - // add the job to the project repository - projectRepository.addJob(projectId, job) map { _ => - job - } - }) - } - - /** - * Get the job by its id - * - * @param projectId project id the job belongs to - * @param jobId job id - * @return - */ - override def getJob(projectId: String, jobId: String): Future[Option[FhirMappingJob]] = { - Future { - jobDefinitions.get(projectId).flatMap(_.get(jobId)) - } - } - - /** - * Update the job in the repository - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param job job to save - * @return - */ - override def updateJob(projectId: String, jobId: String, job: FhirMappingJob): Future[FhirMappingJob] = { - if (!jobDefinitions.get(projectId).exists(_.contains(jobId))) { - throw ResourceNotFound( - "Mapping job does not exists.", - s"A mapping job with id $jobId does not exists in the mapping job repository at ${FileUtils.getPath(jobRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // update the job in the repository - getFileForJob(projectId, job.id).flatMap(file => { - val fw = new FileWriter(file) - fw.write(writePretty(job)) - fw.close() - // update the mapping job in the map - jobDefinitions(projectId).put(jobId, job) - // update the job in the project - projectRepository.updateJob(projectId, job) map { _ => - job - } - }) - } - - /** - * Delete the job from the repository - * - * @param projectId project id the job belongs to - * @param jobId job id - * @return - */ - override def deleteJob(projectId: String, jobId: String): Future[Unit] = { - if (!jobDefinitions.get(projectId).exists(_.contains(jobId))) { - throw ResourceNotFound( - "Mapping job does not exists.", - s"A mapping job with id $jobId does not exists in the mapping job repository at ${FileUtils.getPath(jobRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - - // delete the mapping job from the repository - getFileForJob(projectId, jobDefinitions(projectId)(jobId).id).flatMap(file => { - file.delete() - jobDefinitions(projectId).remove(jobId) - // delete the job from the project - projectRepository.deleteJob(projectId, Some(jobId)) - }) - } - - /** - * Deletes all jobs associated with a specific project. - * - * @param projectId The unique identifier of the project for which jobs should be deleted. - */ - override def deleteAllJobs(projectId: String): Future[Unit] = { - Future { - // delete job definitions for the project - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath(jobRepositoryFolderPath, projectId).toFile) - // remove project from the cache - jobDefinitions.remove(projectId) - } flatMap { _ => - // delete project jobs - projectRepository.deleteJob(projectId) - } - } - - /** - * Retrieves the jobs referencing the given mapping in their definitions. - * - * @param projectId identifier of project whose jobs will be checked - * @param mappingUrl the url of mapping - * @return the jobs referencing the given mapping in their definitions - */ - override def getJobsReferencingMapping(projectId: String, mappingUrl: String): Future[Seq[FhirMappingJob]] = { - Future { - val jobs = jobDefinitions.get(projectId).toSeq.flatMap(_.values) - jobs.filter(_.mappings.exists(_.mappingRef == mappingUrl)) - } - } - - /** - * Get the mapping job file for the given project id and job id - * - * @param projectId - * @param fhirMapping - * @return - */ - private def getFileForJob(projectId: String, jobId: String): Future[File] = { - projectRepository.getProject(projectId) map { project => - if (project.isEmpty) - throw new IllegalStateException( - s"This should not be possible. ProjectId: $projectId does not exist in the project folder repository." - ) - FileOperations.getFileForEntityWithinProject(jobRepositoryFolderPath, project.get.id, jobId) - } - } - - /** - * Initialize the job definitions from the given folder - * - * @param jobRepositoryFolderPath folder path to the job repository - * @return - */ - private def initMap(jobRepositoryFolderPath: String): Unit = { - val jobRepositoryFolder = FileUtils.getPath(jobRepositoryFolderPath).toFile - logger.info(s"Initializing the Mapping Job Repository from path ${jobRepositoryFolder.getAbsolutePath}.") - if (!jobRepositoryFolder.exists()) { - jobRepositoryFolder.mkdirs() - } - var projectDirectories = Seq.empty[File] - projectDirectories = jobRepositoryFolder.listFiles.filter(_.isDirectory).toSeq - projectDirectories.foreach { projectDirectory => - // job-id -> FhirMappingJob - val fhirJobMap: mutable.Map[String, FhirMappingJob] = mutable.Map.empty - val files = IOUtil.getFilesFromFolder( - projectDirectory, - recursively = true, - ignoreHidden = true, - withExtension = Some(FileExtensions.JSON.toString) - ) - files.map { file => - val source = Source.fromFile(file, StandardCharsets.UTF_8.name()) // read the JSON file - val fileContent = - try source.mkString - finally source.close() - // Try to parse the file content as FhirMappingJob - try { - val job = JsonMethods.parse(fileContent).extract[FhirMappingJob] - // check there are no duplicate name on mappingTasks of the job - val duplicateMappingTasks = FhirMappingJobFormatter.findDuplicateMappingTaskNames(job.mappings) - if (duplicateMappingTasks.nonEmpty) { - throw new MappingException( - s"Duplicate 'name' fields detected in the MappingTasks of job '${job.id}': ${duplicateMappingTasks.mkString(", ")}. Please ensure that each MappingTask has a unique name." - ) - } - // discard if the job id and file name not match - if (FileOperations.checkFileNameMatchesEntityId(job.id, file, "job")) { - fhirJobMap.put(job.id, job) - } - } catch { - case e: Throwable => - logger.error(s"Failed to parse mapping job definition at ${file.getPath}", e) - System.exit(1) - } - } - this.jobDefinitions.put(projectDirectory.getName, fhirJobMap) - } - } - - /** - * Reload the job definitions from the given folder - * - * @return - */ - override def invalidate(): Unit = { - this.jobDefinitions.clear() - initMap(jobRepositoryFolderPath) - } - - /** - * Retrieve the projects and FhirMappingJobs within. - * - * @return Map of projectId -> Seq[FhirMappingJob] - */ - override def getProjectPairs: Map[String, Seq[FhirMappingJob]] = { - jobDefinitions.map { case (projectId, jobPairs) => - projectId -> jobPairs.values.toSeq - }.toMap - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/IMappingRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/IMappingRepository.scala deleted file mode 100644 index b4afacbb..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/IMappingRepository.scala +++ /dev/null @@ -1,74 +0,0 @@ -package io.ignifyr.server.repository.mapping - -import io.ignifyr.engine.model.FhirMapping -import io.ignifyr.engine.repository.mapping.IFhirMappingRepository -import io.ignifyr.server.repository.project.IProjectList - -import scala.concurrent.Future - -/** - * Interface to save and load mappings so that the client applications can manage the mappings through CRUD operations - */ -trait IMappingRepository extends IFhirMappingRepository with IProjectList[FhirMapping] { - - /** - * Retrieve all mappings for the given project - * - * @param projectId Identifier of the project for which the mappings to be retrieved - * @return - */ - def getAllMappings(projectId: String): Future[Seq[FhirMapping]] - - /** - * Save the mapping to the repository. - * - * @param projectId subfolder to save the mapping in - * @param mapping mapping to save - * @return - */ - def saveMapping(projectId: String, mapping: FhirMapping): Future[FhirMapping] - - /** - * Get the mapping by its id - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @return - */ - def getMapping(projectId: String, mappingId: String): Future[Option[FhirMapping]] - - /** - * Update the mapping in the repository - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @param mapping mapping to save - * @return - */ - def updateMapping(projectId: String, mappingId: String, mapping: FhirMapping): Future[FhirMapping] - - /** - * Delete the mapping from the repository - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @return - */ - def deleteMapping(projectId: String, mappingId: String): Future[Unit] - - /** - * Deletes all mappings associated with a specific project. - * - * @param projectId The unique identifier of the project for which mappings should be deleted. - */ - def deleteAllMappings(projectId: String): Future[Unit] - - /** - * Retrieves the identifiers of mappings referencing the given schema in their definitions. - * - * @param projectId identifier of project whose mappings will be checked - * @param schemaUrl the url of schema - * @return the identifiers of mappings referencing the given schema in their definitions - */ - def getMappingsReferencingSchema(projectId: String, schemaUrl: String): Future[Seq[String]] -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/ProjectMappingFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/ProjectMappingFolderRepository.scala deleted file mode 100644 index c7de4ca1..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mapping/ProjectMappingFolderRepository.scala +++ /dev/null @@ -1,309 +0,0 @@ -package io.ignifyr.server.repository.mapping - -import com.typesafe.scalalogging.Logger -import io.onfhir.api.util.IOUtil -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.model.FhirMapping -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.server.common.model.{AlreadyExists, ResourceNotFound} -import io.ignifyr.server.repository.project.IProjectRepository -import io.ignifyr.server.util.FileOperations -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -import java.io.{File, FileWriter} -import java.nio.charset.StandardCharsets -import scala.collection.mutable -import scala.concurrent.Future -import scala.io.Source - -/** - * Folder/Directory based mapping repository implementation. This implementation manages [[FhirMapping]]s per project. - * It also extends the engine's folder-based mapping repository implementation to be able to use the same business logic to load mappings from folders. - * - * @param mappingRepositoryFolderPath root folder path to the mapping repository - * @param projectRepository project repository to update corresponding projects based on updates on the mappings - */ -class ProjectMappingFolderRepository(mappingRepositoryFolderPath: String, projectRepository: IProjectRepository) - extends IMappingRepository { - - private val logger: Logger = Logger(this.getClass) - - // project id -> mapping id -> mapping - private val mappingDefinitions: mutable.Map[String, mutable.Map[String, FhirMapping]] = - mutable.Map.empty[String, mutable.Map[String, FhirMapping]] - // Initialize the map for the first time - initMap(mappingRepositoryFolderPath) - - /** - * Retrieve all mappings of a given project - * - * @param projectId project id the jobs belong to - * @return - */ - override def getAllMappings(projectId: String): Future[Seq[FhirMapping]] = { - Future { - mappingDefinitions - .get(projectId) - .map(_.values.toSeq) // If such a project exists, return the mappings as a sequence - .getOrElse(Seq.empty[FhirMapping]) // Else, return an empty list - } - } - - /** - * Save the mapping to the repository. - * - * @param projectId subfolder to save the mapping in - * @param mapping mapping to save - * @return the saved FhirMapping - */ - override def saveMapping(projectId: String, mapping: FhirMapping): Future[FhirMapping] = { - // validate that mapping id is unique - mappingDefinitions.get(projectId).flatMap(_.get(mapping.id)).foreach { _ => - throw AlreadyExists( - "Fhir mapping already exists.", - s"A mapping definition with id ${mapping.id} already exists in the mapping repository at ${FileUtils.getPath(mappingRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // validate that mapping url is unique - mappingDefinitions.get(projectId).foreach { mappings => - if (mappings.values.exists(_.url.contentEquals(mapping.url))) { - throw AlreadyExists( - "Fhir mapping already exists.", - s"A mapping definition with url ${mapping.url} already exists in the mapping repository at ${FileUtils.getPath(mappingRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - } - - // Write to the repository as a new file - getFileForMapping(projectId, mapping.id).flatMap(newFile => { - val fw = new FileWriter(newFile) - fw.write(writePretty(mapping)) - fw.close() - - // Add to the project metadata json file - projectRepository.addMapping(projectId, mapping) map { _ => - // Add to the in-memory map - mappingDefinitions.getOrElseUpdate(projectId, mutable.Map.empty).put(mapping.id, mapping) - mapping - } - }) - } - - /** - * Get the mapping by its id - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @return - */ - override def getMapping(projectId: String, mappingId: String): Future[Option[FhirMapping]] = { - Future { - mappingDefinitions(projectId).get(mappingId) - } - } - - /** - * Update the mapping in the repository - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @param mapping mapping to save - * @return - * @throws AlreadyExists if a mapping with the same URL already exists - */ - override def updateMapping(projectId: String, mappingId: String, mapping: FhirMapping): Future[FhirMapping] = { - if (!mappingDefinitions.get(projectId).exists(_.contains(mappingId))) { - throw ResourceNotFound( - "Mapping does not exists.", - s"A mapping with id $mappingId does not exists in the mapping repository at ${FileUtils.getPath(mappingRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // validate that mapping url is unique - mappingDefinitions.get(projectId).foreach { mappings => - if ( - mappings.exists { case (id, definition) => - !id.contentEquals(mappingId) && definition.url.contentEquals(mapping.url) - } - ) { - throw AlreadyExists( - "Fhir mapping already exists.", - s"A mapping definition with url ${mapping.url} already exists in the mapping repository at ${FileUtils.getPath(mappingRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - } - - // update the mapping in the repository - getFileForMapping(projectId, mapping.id).flatMap(file => { - val fw = new FileWriter(file) - fw.write(writePretty(mapping)) - fw.close() - // update the mapping in the in-memory map - mappingDefinitions(projectId).put(mappingId, mapping) - // update the projects metadata json file - projectRepository.updateMapping(projectId, mapping) map { _ => - mapping - } - }) - } - - /** - * Delete the mapping from the repository - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @return - */ - override def deleteMapping(projectId: String, mappingId: String): Future[Unit] = { - if (!mappingDefinitions.get(projectId).exists(_.contains(mappingId))) { - throw ResourceNotFound( - "Mapping does not exists.", - s"A mapping with id $mappingId does not exists in the mapping repository at ${FileUtils.getPath(mappingRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - - // delete the mapping from the repository - getFileForMapping(projectId, mappingDefinitions(projectId)(mappingId).id).flatMap(file => { - file.delete() - mappingDefinitions(projectId).remove(mappingId) - // delete the mapping from projects json file - projectRepository.deleteMapping(projectId, Some(mappingId)) - }) - } - - /** - * Deletes all mappings associated with a specific project. - * - * @param projectId The unique identifier of the project for which mappings should be deleted. - */ - override def deleteAllMappings(projectId: String): Future[Unit] = { - Future { - // delete mapping definitions for the project - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath(mappingRepositoryFolderPath, projectId).toFile) - // remove project from the cache - mappingDefinitions.remove(projectId) - } flatMap { _ => - // delete project mappings - projectRepository.deleteMapping(projectId) - } - } - - /** - * Retrieves the identifiers of mappings referencing the given schema in their definitions. - * - * @param projectId identifier of project whose mappings will be checked - * @param schemaUrl the url of schema - * @return the identifiers of mappings referencing the given schema in their definitions - */ - override def getMappingsReferencingSchema(projectId: String, schemaUrl: String): Future[Seq[String]] = { - Future { - mappingDefinitions - .getOrElse(projectId, Map.empty) // handle the case where project has no mappings by returning an empty Map - .values - .toSeq - .filter(mapping => mapping.source.exists(source => source.url.contentEquals(schemaUrl))) - .map(mapping => mapping.id) - } - } - - /** - * Gets the file for the given mapping definition. - * - * @param fhirMapping - * @return - */ - private def getFileForMapping(projectId: String, fhirMappingId: String): Future[File] = { - projectRepository.getProject(projectId) map { project => - if (project.isEmpty) - throw new IllegalStateException( - s"This should not be possible. ProjectId: $projectId does not exist in the project folder repository." - ) - FileOperations.getFileForEntityWithinProject(mappingRepositoryFolderPath, project.get.id, fhirMappingId) - } - } - - /** - * Initializes the mapping definitions from the repository - * - * @param mappingRepositoryFolderPath path to the mapping repository - * @return - */ - private def initMap(mappingRepositoryFolderPath: String): Unit = { - val mappingRepositoryFolder = FileUtils.getPath(mappingRepositoryFolderPath).toFile - if (!mappingRepositoryFolder.exists()) { - mappingRepositoryFolder.mkdirs() - } - var projectDirectories = Seq.empty[File] - projectDirectories = mappingRepositoryFolder.listFiles.filter(_.isDirectory).toSeq - projectDirectories.foreach { projectDirectory => - // mapping-id -> FhirMapping - val fhirMappingMap: mutable.Map[String, FhirMapping] = mutable.Map.empty - val files = IOUtil.getFilesFromFolder( - projectDirectory, - recursively = true, - ignoreHidden = true, - withExtension = Some(FileExtensions.JSON.toString) - ) - files.foreach { file => - val source = Source.fromFile(file, StandardCharsets.UTF_8.name()) // read the JSON file - val fileContent = - try source.mkString - finally source.close() - try { // Try to parse the file content as FhirMapping - val fhirMapping = JsonMethods.parse(fileContent).extract[FhirMapping] - // discard if the mapping id and file name not match - if (FileOperations.checkFileNameMatchesEntityId(fhirMapping.id, file, "mapping")) { - fhirMappingMap.put(fhirMapping.id, fhirMapping) - } // else case is logged within FileOperations.checkFileNameMatchesEntityId - } catch { - case e: Throwable => - logger.error(s"Failed to parse mapping definition at ${file.getPath}", e) - System.exit(1) - } - } - if (fhirMappingMap.isEmpty) { - // No processable schema files under projectDirectory - logger.warn( - s"There are no processable mapping files under ${projectDirectory.getAbsolutePath}. Skipping ${projectDirectory.getName}." - ) - } else { - this.mappingDefinitions.put(projectDirectory.getName, fhirMappingMap) - } - } - } - - /** - * Returns the Fhir mapping definition by given url - * - * @param mappingUrl Fhir mapping url - * @return - */ - override def getFhirMappingByUrl(mappingUrl: String): FhirMapping = { - mappingDefinitions.values - .flatMap(_.values) // Flatten all the mappings managed for all projects - .find(_.url.contentEquals(mappingUrl)) - .get - } - - /** - * Invalidate the internal cache and refresh the cache with the FhirMappings directly from their source - */ - override def invalidate(): Unit = { - this.mappingDefinitions.clear() - initMap(mappingRepositoryFolderPath) - } - - /** - * Retrieve the projects and FhirMappings within. - * - * @return Map of projectId -> Seq[FhirMapping] - */ - override def getProjectPairs: Map[String, Seq[FhirMapping]] = { - mappingDefinitions.map { case (projectId, mappingPairs) => - projectId -> mappingPairs.values.toSeq - }.toMap - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/IMappingContextRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/IMappingContextRepository.scala deleted file mode 100644 index 80a67fce..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/IMappingContextRepository.scala +++ /dev/null @@ -1,102 +0,0 @@ -package io.ignifyr.server.repository.mappingContext - -import akka.stream.scaladsl.Source -import akka.util.ByteString -import io.ignifyr.engine.repository.ICachedRepository -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.project.IProjectList - -import scala.concurrent.Future - -/** - * Interface for the mapping context repository - */ -trait IMappingContextRepository extends ICachedRepository with IProjectList[String] { - - /** - * Retrieve the metadata of all mapping context ids - * @return - */ - def getAllMappingContext(projectId: String): Future[Seq[String]] - - /** - * Save the mapping context to the repository. - * We only store id of a mapping context in the project metadata json file. - * - * @param projectId project id the mapping context belongs to - * @param id mapping context id to save - * @return - */ - def createMappingContext(projectId: String, id: String): Future[String] - - /** - * Delete the mapping context from the repository - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @return - */ - def deleteMappingContext(projectId: String, id: String): Future[Unit] - - /** - * Deletes all mapping contexts associated with a specific project. - * - * @param projectId The unique identifier of the project for which mapping contexts should be deleted. - */ - def deleteAllMappingContexts(projectId: String): Unit - - /** - * Update the mapping context header by its id - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @param headers mapping context headers - * @return - */ - def updateMappingContextHeader(projectId: String, id: String, headers: Seq[CsvHeader]): Future[Unit] - - /** - * Save the mapping context content to the repository - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @param content mapping context content to save - * @return - */ - def saveMappingContextContent( - projectId: String, - id: String, - content: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] - - /** - * Get the mapping context content by its id - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @return - */ - def getMappingContextContent( - projectId: String, - id: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] - - /** - * Upload the mapping context content to the repository - * - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @param content mapping context content to save - * @return - */ - def uploadMappingContext(projectId: String, id: String, content: Source[ByteString, Any]): Future[Unit] - - /** - * Download the mapping context content by its id - * - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @return - */ - def downloadMappingContext(projectId: String, id: String): Future[Source[ByteString, Any]] -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/MappingContextFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/MappingContextFolderRepository.scala deleted file mode 100644 index 31814ce3..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/mappingContext/MappingContextFolderRepository.scala +++ /dev/null @@ -1,299 +0,0 @@ -package io.ignifyr.server.repository.mappingContext - -import akka.stream.scaladsl.{FileIO, Source} -import akka.util.ByteString -import io.onfhir.api.util.IOUtil -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.common.model.{AlreadyExists, ResourceNotFound} -import io.ignifyr.server.model._ -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.project.IProjectRepository -import io.ignifyr.server.util.CsvUtil - -import java.io.File -import scala.collection.mutable -import scala.concurrent.Future - -/** - * Folder/Directory based mapping context repository implementation. - * - * @param mappingContextRepositoryFolderPath root folder path to the mapping context repository - */ -class MappingContextFolderRepository(mappingContextRepositoryFolderPath: String, projectRepository: IProjectRepository) - extends IMappingContextRepository { - // project id -> mapping context id - private val mappingContextDefinitions: mutable.Map[String, Seq[String]] = mutable.Map.empty[String, Seq[String]] - // Initialize the map for the first time - initMap(mappingContextRepositoryFolderPath) - - /** - * Retrieve the metadata of all mapping context ids - * - * @return - */ - override def getAllMappingContext(projectId: String): Future[Seq[String]] = { - Future { - mappingContextDefinitions.getOrElse(projectId, Seq.empty[String]) - } - } - - /** - * Save the mapping context to the repository. - * We only store id of a mapping context in the project metadata json file. - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id to save - * @return - */ - override def createMappingContext(projectId: String, mappingContextId: String): Future[String] = { - if (mappingContextExists(projectId, mappingContextId)) { - throw AlreadyExists( - "Fhir mapping already exists.", - s"A mapping context definition with id $mappingContextId already exists in the mapping context repository at ${FileUtils.getPath(mappingContextRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // Write to the repository as a new file - getFileForMappingContext(projectId, mappingContextId).flatMap(newFile => { - newFile.createNewFile() - // Add to the project metadata json file - projectRepository.addMappingContext(projectId, mappingContextId) map { _ => - // Add to the in-memory map - mappingContextDefinitions(projectId) = - mappingContextDefinitions.getOrElseUpdate(projectId, Seq.empty) :+ mappingContextId - mappingContextId - } - }) - } - - /** - * Delete the mapping context from the repository - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id - * @return - */ - override def deleteMappingContext(projectId: String, mappingContextId: String): Future[Unit] = { - if (!mappingContextExists(projectId, mappingContextId)) { - throw ResourceNotFound( - "Mapping context does not exists.", - s"A mapping context with id $mappingContextId does not exists in the mapping context repository at ${FileUtils.getPath(mappingContextRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - - // delete the mapping context from the repository - getFileForMappingContext(projectId, mappingContextId).flatMap(file => { - file.delete() - mappingContextDefinitions(projectId) = mappingContextDefinitions(projectId).filterNot(_ == mappingContextId) - // update the projects metadata json file - projectRepository.deleteMappingContext(projectId, Some(mappingContextId)) - }) - } - - /** - * Deletes all mapping contexts associated with a specific project. - * - * @param projectId The unique identifier of the project for which mapping contexts should be deleted. - */ - override def deleteAllMappingContexts(projectId: String): Unit = { - Future { - // delete mapping context definitions for the project - org.apache.commons.io.FileUtils - .deleteDirectory(FileUtils.getPath(mappingContextRepositoryFolderPath, projectId).toFile) - // remove project from the cache - mappingContextDefinitions.remove(projectId) - } flatMap { _ => - // delete project mapping contexts - projectRepository.deleteMappingContext(projectId) - } - } - - /** - * Update the mapping context header by its id - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id - * @param headers mapping context headers - * @return - */ - def updateMappingContextHeader(projectId: String, mappingContextId: String, headers: Seq[CsvHeader]): Future[Unit] = { - if (!mappingContextExists(projectId, mappingContextId)) { - throw ResourceNotFound( - "Mapping context does not exists.", - s"A mapping context with id $mappingContextId does not exists in the mapping context repository at ${FileUtils.getPath(mappingContextRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // Write headers to the first row in the related file in the repository - getFileForMappingContext(projectId, mappingContextId).map(file => { - CsvUtil.writeCsvHeaders(file, headers) - }) - } - - /** - * Save the mapping context content to the repository - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id - * @param content mapping context content to save - * @return - */ - override def saveMappingContextContent( - projectId: String, - mappingContextId: String, - content: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] = { - if (!mappingContextExists(projectId, mappingContextId)) { - throw ResourceNotFound( - "Mapping context does not exists.", - s"A mapping context with id $mappingContextId does not exists in the mapping context repository at ${FileUtils.getPath(mappingContextRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // Write content to the related file in the repository - getFileForMappingContext(projectId, mappingContextId).flatMap(file => { - CsvUtil.writeCsvAndReturnRowNumber(file, content, pageNumber, pageSize) - }) - } - - /** - * Get the mapping context content by its id - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id - * @return - */ - override def getMappingContextContent( - projectId: String, - mappingContextId: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] = { - if (!mappingContextExists(projectId, mappingContextId)) { - throw ResourceNotFound( - "Mapping context does not exists.", - s"A mapping context with id $mappingContextId does not exists in the mapping context repository at ${FileUtils.getPath(mappingContextRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // Read content from the related file in the repository - getFileForMappingContext(projectId, mappingContextId).flatMap(file => { - CsvUtil.getPaginatedCsvContent(file, pageNumber, pageSize) - }) - } - - /** - * Upload the mapping context content to the repository - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id - * @param content mapping context content to save - * @return - */ - override def uploadMappingContext( - projectId: String, - mappingContextId: String, - content: Source[ByteString, Any] - ): Future[Unit] = { - if (!mappingContextExists(projectId, mappingContextId)) { - throw ResourceNotFound( - "Mapping context does not exists.", - s"A mapping context with id $mappingContextId does not exists in the mapping context repository at ${FileUtils.getPath(mappingContextRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // Write content to the related file in the repository - getFileForMappingContext(projectId, mappingContextId).map(file => { - CsvUtil.saveFileContent(file, content) - }) - } - - /** - * Download the mapping context content by its id - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id - * @return - */ - override def downloadMappingContext(projectId: String, mappingContextId: String): Future[Source[ByteString, Any]] = { - if (!mappingContextExists(projectId, mappingContextId)) { - throw ResourceNotFound( - "Mapping context does not exists.", - s"A mapping context with id $mappingContextId does not exists in the mapping context repository at ${FileUtils.getPath(mappingContextRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - // Read content from the related file in the repository - getFileForMappingContext(projectId, mappingContextId).map(file => { - FileIO.fromPath(file.toPath) - }) - } - - /** - * Checks if the mapping context exists in the repository - * - * @param projectId project id the mapping context belongs to - * @param mappingContextId mapping context id - * @return - */ - private def mappingContextExists(projectId: String, mappingContextId: String): Boolean = { - mappingContextDefinitions.get(projectId).exists(_.contains(mappingContextId)) - } - - /** - * Gets the file for the given mapping context id. - * - * @param fhirMappingContextId - * @return - */ - private def getFileForMappingContext(projectId: String, fhirMappingContextId: String): Future[File] = { - val projectFuture: Future[Option[Project]] = projectRepository.getProject(projectId) - projectFuture.map(project => { - val file: File = - FileUtils.getPath(mappingContextRepositoryFolderPath, project.get.id, fhirMappingContextId).toFile - // If the project folder does not exist, create it - if (!file.getParentFile.exists()) { - file.getParentFile.mkdir() - } - file - }) - } - - /** - * Initializes the mapping context map from the repository - * Map of (project id -> mapping context list) - * - * @param mappingContextRepositoryFolderPath path to the mapping context repository - * @return - */ - private def initMap(mappingContextRepositoryFolderPath: String): Unit = { - val folder = FileUtils.getPath(mappingContextRepositoryFolderPath).toFile - if (!folder.exists()) { - folder.mkdirs() - } - var directories = Seq.empty[File] - directories = folder.listFiles.filter(_.isDirectory).toSeq - directories.foreach { projectDirectory => - val files = - IOUtil.getFilesFromFolder(projectDirectory, recursively = true, ignoreHidden = true, withExtension = None) - val fileNameList = files.map(_.getName) - this.mappingContextDefinitions.put(projectDirectory.getName, fileNameList) - } - } - - /** - * Reload the mapping context definitions from the given folder - * - * @return - */ - override def invalidate(): Unit = { - this.mappingContextDefinitions.clear() - initMap(mappingContextRepositoryFolderPath) - } - - /** - * Retrieve the projects and MappingContextIds within. - * - * @return Map of projectId -> Seq[String] - */ - override def getProjectPairs: Map[String, Seq[String]] = { - mappingContextDefinitions.toMap - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectList.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectList.scala deleted file mode 100644 index 70d6e5c9..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectList.scala +++ /dev/null @@ -1,16 +0,0 @@ -package io.ignifyr.server.repository.project - -/** - * Interface to handle the projects referred by an entity. - * This interface is intended to provide the necessary methods to repository implementations (e.g. JobFolderRepository) - * which relates back to Projects. - */ -trait IProjectList[T] { - - /** - * Retrieve the projects and entities within. - * @return Map of projectId -> Seq[T] where T is the entity (e.g. FhirMapping) - */ - def getProjectPairs: Map[String, Seq[T]] - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectRepository.scala deleted file mode 100644 index 038bd3bd..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/IProjectRepository.scala +++ /dev/null @@ -1,151 +0,0 @@ -package io.ignifyr.server.repository.project - -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.model.{FhirMapping, FhirMappingJob} -import io.ignifyr.server.model.Project -import org.json4s.JObject - -import scala.concurrent.Future - -/** - * Interface to manage Projects - */ -trait IProjectRepository { - - /** - * Retrieve all Projects - * - * @return all projects in the repository - */ - def getAllProjects: Future[Seq[Project]] - - /** - * Save project to the repository. - * - * @param project project to be saved - * @return the created project - */ - def createProject(project: Project): Future[Project] - - /** - * Retrieve the project identified by its id. - * - * @param id id of the project - * @return the project - */ - def getProject(id: String): Future[Option[Project]] - - /** - * Update some fields of the project in the repository. - * - * @param id id of the project - * @param patch patch to be applied to the project - * @return - */ - def updateProject(id: String, patch: JObject): Future[Project] - - /** - * Delete the project from the repository. - * - * @param id id of the project - * @return - */ - def deleteProject(id: String): Future[Unit] - - /** - * Methods to manage the reference from a Project to SchemaDefinitions - */ - - /** - * Add the schema definition to the project. - * - * @param projectId - * @param schema - */ - def addSchema(projectId: String, schema: SchemaDefinition): Future[Project] - - /** - * Replace the schema definition of the project. - * - * @param projectId - * @param schemaMetadata - */ - def updateSchema(projectId: String, schemaMetadata: SchemaDefinition): Future[Project] - - /** - * Remove the schema definition from the project - * - * @param projectId - * @param schemaId The unique identifier of the schema to be deleted. If not provided, all schemas will be removed from the project definition. - */ - def deleteSchema(projectId: String, schemaId: Option[String] = None): Future[Unit] - - /** - * Methods to manage the reference from a Project to SchemaDefinitions - */ - - /** - * Add the mapping definition to the project - * - * @param projectId Project id to which the mapping will be added to - * @param mapping Mapping to be added - */ - def addMapping(projectId: String, mapping: FhirMapping): Future[Project] - - /** - * Replaces the mapping definition within the project - * - * @param projectId Project id of which the mapping will be updated - * @param mapping Mapping to be updated - */ - def updateMapping(projectId: String, mapping: FhirMapping): Future[Project] - - /** - * Deletes the mapping definition from the - * - * @param projectId Project id from which the mapping will be deleted - * @param mappingId The unique identifier of the mapping to be deleted. If not provided, all mappings for the project will be deleted. - */ - def deleteMapping(projectId: String, mappingId: Option[String] = None): Future[Unit] - - /** - * Adds the job definition to the project - * - * @param projectId - * @param job - */ - def addJob(projectId: String, job: FhirMappingJob): Future[Project] - - /** - * Replaces the job definition within the project - * - * @param projectId - * @param job - */ - def updateJob(projectId: String, job: FhirMappingJob): Future[Project] - - /** - * Deletes the job definition of the project - * - * @param projectId - * @param jobId The unique identifier of the job to be deleted. If not provided, all jobs for the project will be deleted. - */ - def deleteJob(projectId: String, jobId: Option[String] = None): Future[Unit] - - /** - * Adds the mapping context id to the project - * - * @param projectId Project id the mapping context will be added to - * @param mappingContext Mapping context id to be added - */ - def addMappingContext(projectId: String, mappingContext: String): Future[Project] - - /** - * Deletes the mapping context from the project - * - * @param projectId Project id the mapping context will be deleted - * @param mappingContextId The unique identifier of the mapping context to be deleted. If not provided, all mapping contexts for the project will be deleted. - */ - def deleteMappingContext(projectId: String, mappingContextId: Option[String] = None): Future[Unit] - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/ProjectFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/ProjectFolderRepository.scala deleted file mode 100644 index cd696a08..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/project/ProjectFolderRepository.scala +++ /dev/null @@ -1,373 +0,0 @@ -package io.ignifyr.server.repository.project - -import com.typesafe.scalalogging.Logger -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.engine.model.{FhirMapping, FhirMappingJob} -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.common.model.{AlreadyExists, ResourceNotFound} -import io.ignifyr.server.model._ -import org.json4s._ -import org.json4s.jackson.Serialization - -import java.io.FileWriter -import scala.collection.mutable -import scala.concurrent.Future - -/** - * Folder/Directory based project repository implementation. - * - * @param config Engine configs - */ -class ProjectFolderRepository(config: IgnifyrEngineConfig) extends IProjectRepository { - - private val logger: Logger = Logger(this.getClass) - - // Project cache keeping the up-to-date list of projects - private val projects: mutable.Map[String, Project] = mutable.Map.empty - - /** - * Initializes the projects cache with the given Map of projects and persist to the file (projects.json) - * - * @param projects - */ - def setProjects(vProjects: Map[String, Project]): Unit = { - this.projects.clear() - this.projects ++= vProjects - this.updateProjectsMetadata() - } - - /** - * Retrieve all Projects - * - * @return all projects in the repository - */ - override def getAllProjects: Future[Seq[Project]] = { - Future { - projects.values.toSeq.sortWith(projectComparisonFunc) - } - } - - /** - * Save project to the repository. - * - * @param project project to be saved - * @return the created project - */ - override def createProject(project: Project): Future[Project] = { - Future { - // validate that there is no project with the given id and name - if (projects.contains(project.id)) { - throw AlreadyExists( - "Project already exists.", - s"Project with id ${project.id} already exists in the repository." - ) - } - if (projects.values.exists(p => p.name.contentEquals(project.name))) { - throw AlreadyExists( - "Project already exists.", - s"Project with name ${project.name} already exists in the repository." - ) - } - - // add project to the cache - projects.put(project.id, project) - - // update projects metadata file by adding the new project - updateProjectsMetadata() - project - } - } - - /** - * Retrieve the project identified by its id. - * - * @param id id of the project - * @return the project - */ - override def getProject(id: String): Future[Option[Project]] = { - Future { - projects.get(id) - } - } - - /** - * Update some fields of the project in the repository. - * - * @param id id of the project - * @param patch patch to be applied to the project - * @return - */ - override def updateProject(id: String, patch: JObject): Future[Project] = { - Future { - // validate that the project exists - if (!projects.contains(id)) - throw ResourceNotFound("Project does not exist.", s"Project $id not found") - - // update the editable fields of project with new values - val newDescription = (patch \ ProjectEditableFields.DESCRIPTION).extractOpt[String] - val newSchemaUrlPrefix = (patch \ ProjectEditableFields.SCHEMA_URL_PREFIX).extractOpt[String] - val newMappingUrlPrefix = (patch \ ProjectEditableFields.MAPPING_URL_PREFIX).extractOpt[String] - val updatedProject = projects(id).copy( - description = newDescription, - schemaUrlPrefix = newSchemaUrlPrefix, - mappingUrlPrefix = newMappingUrlPrefix - ) - projects.put(id, updatedProject) - - // update the projects metadata - updateProjectsMetadata() - updatedProject - } - } - - /** - * Delete the project from the repository. - * - * @param id id of the project - * @return - */ - override def deleteProject(id: String): Future[Unit] = { - Future { - // validate that the project exists - if (!projects.contains(id)) - throw ResourceNotFound("Project does not exist.", s"Project $id not found") - // remove the project from the cache - projects.remove(id) - // update projects metadata with the remaining ones - updateProjectsMetadata() - } - } - - /** - * Adds the schema definition to the project - * - * @param projectId - * @param schema - */ - - override def addSchema(projectId: String, schema: SchemaDefinition): Future[Project] = { - Future { - val project: Project = projects(projectId) - val updatedProject = project.copy(schemas = project.schemas :+ schema) - projects.put(projectId, updatedProject) - updateProjectsMetadata() - updatedProject - } - } - - /** - * Replaces the schema definition of the project - * - * @param projectId - * @param schemaMetadata - */ - override def updateSchema(projectId: String, schemaMetadata: SchemaDefinition): Future[Project] = { - deleteSchema(projectId, Some(schemaMetadata.id)) flatMap { _ => - addSchema(projectId, schemaMetadata) - } - } - - /** - * Deletes the schema definition (or all schema definitions) from the project. - * - * @param projectId - * @param schemaId The unique identifier of the schema to be deleted. If not provided, all schemas will be removed from the project definition. - */ - override def deleteSchema(projectId: String, schemaId: Option[String] = None): Future[Unit] = { - Future { - val project: Option[Project] = projects.get(projectId) - project match { - case Some(project) => - schemaId match { - case Some(id) => - projects.put(projectId, project.copy(schemas = project.schemas.filterNot(s => s.id.equals(id)))) - case None => - projects.put(projectId, project.copy(schemas = Seq.empty)) - } - updateProjectsMetadata() - case None => // Do nothing if project is deleted before the schema - } - } - } - - /** - * Add the mapping definition to the project - * - * @param projectId Project id to which the mapping will be added to - * @param mapping Mapping to be added - */ - override def addMapping(projectId: String, mapping: FhirMapping): Future[Project] = { - Future { - val project: Project = projects(projectId) - val updatedProject = project.copy(mappings = project.mappings :+ mapping) - projects.put(projectId, updatedProject) - updateProjectsMetadata() - updatedProject - } - } - - /** - * Replaces the mapping definition within the project - * - * @param projectId Project id of which the mapping will be updated - * @param mapping Mapping to be updated - */ - override def updateMapping(projectId: String, mapping: FhirMapping): Future[Project] = { - deleteMapping(projectId, Some(mapping.id)) flatMap { _ => - addMapping(projectId, mapping) - } - } - - /** - * Deletes the mapping definition from the - * - * @param projectId Project id from which the mapping will be deleted - * @param mappingId The unique identifier of the mapping to be deleted. If not provided, all mappings for the project will be deleted. - */ - override def deleteMapping(projectId: String, mappingId: Option[String] = None): Future[Unit] = { - Future { - val project: Option[Project] = projects.get(projectId) - project match { - case Some(project) => - mappingId match { - case Some(id) => - projects.put(projectId, project.copy(mappings = project.mappings.filterNot(m => m.id.equals(id)))) - case None => - projects.put(projectId, project.copy(mappings = Seq.empty)) - } - updateProjectsMetadata() - case None => // Do nothing if project is deleted before the mapping - } - } - } - - /** - * Adds the job definition to the project - * - * @param projectId - * @param job - */ - override def addJob(projectId: String, job: FhirMappingJob): Future[Project] = { - Future { - val project: Project = projects(projectId) - val updatedProject = project.copy(mappingJobs = project.mappingJobs :+ job) - projects.put(projectId, updatedProject) - updateProjectsMetadata() - updatedProject - } - } - - /** - * Replaces the job definition of the project json file - * - * @param projectId - * @param job - */ - override def updateJob(projectId: String, job: FhirMappingJob): Future[Project] = { - deleteJob(projectId, Some(job.id)) flatMap { _ => - addJob(projectId, job) - } - } - - /** - * Deletes the job definition of the project json file - * - * @param projectId - * @param jobId The unique identifier of the job to be deleted. If not provided, all jobs for the project will be deleted. - */ - override def deleteJob(projectId: String, jobId: Option[String] = None): Future[Unit] = { - Future { - val project: Option[Project] = projects.get(projectId) - project match { - case Some(project) => - jobId match { - case Some(id) => - projects.put(projectId, project.copy(mappingJobs = project.mappingJobs.filterNot(j => j.id.equals(id)))) - case None => - projects.put(projectId, project.copy(mappingJobs = Seq.empty)) - } - updateProjectsMetadata() - case None => // Do nothing if project is deleted before the job - } - } - } - - /** - * Adds the mapping context id to the project json file - * - * @param projectId Project id the mapping context will be added to - * @param mappingContext Mapping context id to be added - */ - override def addMappingContext(projectId: String, mappingContext: String): Future[Project] = { - Future { - val project: Project = projects(projectId) - val updatedProject = project.copy(mappingContexts = project.mappingContexts :+ mappingContext) - projects.put(projectId, updatedProject) - updateProjectsMetadata() - updatedProject - } - } - - /** - * Deletes the mapping context in the project json file - * - * @param projectId Project id the mapping context will be deleted - * @param mappingContextId The unique identifier of the mapping context to be deleted. If not provided, all mapping contexts for the project will be deleted. - */ - override def deleteMappingContext(projectId: String, mappingContextId: Option[String] = None): Future[Unit] = { - Future { - val project: Option[Project] = projects.get(projectId) - project match { - case Some(project) => - mappingContextId match { - case Some(id) => - projects.put( - projectId, - project.copy(mappingContexts = project.mappingContexts.filterNot(m => m.equals(id))) - ) - case None => - projects.put(projectId, project.copy(mappingContexts = Seq.empty)) - } - updateProjectsMetadata() - case None => // Do nothing if project is deleted before the mapping context - } - } - } - - /** - * Updates the projects metadata with project included in the cache. - */ - private def updateProjectsMetadata(): Unit = { - val file = FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile - // when projects metadata file does not exist, create it - if (!file.exists()) { - logger.debug("There does not exist a metadata file for projects to update. Creating it...") - file.createNewFile() - } - // write projects to the file. Only the metadata of the internal resources are written to the file - val fw = new FileWriter(file) - try fw.write(Serialization.writePretty(projects.values.map(_.getMetadata()).toList)) - finally fw.close() - } - - /** - * Comparison function for two Projects. The definitions are compared according to their names with default String comparison. - * - * @param p1 - * @param p2 - * @return - */ - private def projectComparisonFunc(p1: Project, p2: Project): Boolean = { - p1.name.compareTo(p2.name) < 0 - } -} - -/** - * Keeps file/folder names related to the project repository - * */ -object ProjectFolderRepository { - val PROJECTS_JSON = "projects.json" // file keeping the metadata of all projects -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/AbstractSchemaRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/AbstractSchemaRepository.scala deleted file mode 100644 index 06941024..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/AbstractSchemaRepository.scala +++ /dev/null @@ -1,35 +0,0 @@ -package io.ignifyr.server.repository.schema - -import io.onfhir.api.parsers.IFhirFoundationResourceParser -import io.onfhir.config.IFhirVersionConfigurator -import io.onfhir.definitions.resource.fhir.config.{R4Configurator, R5Configurator} -import io.onfhir.r4.parsers.R4Parser -import io.onfhir.r5.parsers.R5Parser -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.util.{FhirVersionUtil, MajorFhirVersion} - -/** - * Abstract class to provide common functionality to the implementations of ISchemaRepository - */ -abstract class AbstractSchemaRepository extends ISchemaRepository { - - /** - * So that its validation function can be used when a new schema needs to be validated. - */ - protected val fhirConfigurator: IFhirVersionConfigurator = - FhirVersionUtil.getMajorFhirVersion(IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion) match { - case MajorFhirVersion.R4 => new R4Configurator() - case MajorFhirVersion.R5 => new R5Configurator() - case _ => throw new NotImplementedError() - } - - /** - * So that a StructureDefinition resource can be parsed into ProfileRestrictions - */ - protected val fhirFoundationResourceParser: IFhirFoundationResourceParser = - FhirVersionUtil.getMajorFhirVersion(IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion) match { - case MajorFhirVersion.R4 => new R4Parser() - case MajorFhirVersion.R5 => new R5Parser() - case _ => throw new NotImplementedError() - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/ISchemaRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/ISchemaRepository.scala deleted file mode 100644 index 68e4828f..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/ISchemaRepository.scala +++ /dev/null @@ -1,97 +0,0 @@ -package io.ignifyr.server.repository.schema - -import io.onfhir.api.Resource -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.mapping.schema.IFhirSchemaLoader -import io.ignifyr.engine.repository.ICachedRepository -import io.ignifyr.server.repository.project.IProjectList - -import scala.concurrent.Future - -/** - * Interface to save and load SchemaDefinitions - * so that the client applications can manage the schemas through CRUD operations - */ -trait ISchemaRepository extends IFhirSchemaLoader with ICachedRepository with IProjectList[SchemaDefinition] { - - /** - * Retrieve the metadata of all SchemaDefinitions (only id, url, type and name fields are populated) - * - * @return - */ - def getAllSchemas(projectId: String): Future[Seq[SchemaDefinition]] - - /** - * Retrieve the schema identified by its id. - * - * @param projectId Project containing the schema definition - * @param schemaId Identifier of the schema definition - * @return - */ - def getSchema(projectId: String, schemaId: String): Future[Option[SchemaDefinition]] - - /** - * Retrieve the schema identified by its url. - * - * @param url URL of the schema definition - * @return - */ - def getSchemaByUrl(url: String): Future[Option[SchemaDefinition]] - - /** - * Save the schema to the repository. - * - * @param projectId Project containing the schema definition - * @param schemaDefinition Content of the schema definition - * @return - */ - def saveSchema(projectId: String, schemaDefinition: SchemaDefinition): Future[SchemaDefinition] - - /** - * Update the schema to the repository. - * - * @param projectId Project containing the schema definition - * @param schemaId Identifier of the schema - * @param schemaDefinition Content of the schema definition - * @return - */ - def updateSchema(projectId: String, schemaId: String, schemaDefinition: SchemaDefinition): Future[SchemaDefinition] - - /** - * Delete the schema from the repository. - * - * @param projectId Project containing the schema definition - * @param schemaId Identifier of the schema definition - * @return - */ - def deleteSchema(projectId: String, schemaId: String): Future[Unit] - - /** - * Deletes all schemas associated with a specific project. - * - * @param projectId The unique identifier of the project for which schemas should be deleted. - */ - def deleteAllSchemas(projectId: String): Future[Unit] - - /** - * Retrieve the Structure Definition of the schema identified by its id. - * - * @param projectId Project containing the schema definition - * @param schemaId Identifier of the schema definition - * @return Structure definition of the schema converted into StructureDefinition Resource - */ - def getSchemaAsStructureDefinition(projectId: String, schemaId: String): Future[Option[Resource]] - - /** - * Saves the schemas by using their Structure Definition resources. - * - * @param projectId The ID of the project that contains the schema definition. - * @param structureDefinitionResources A sequence of resources representing the structure definitions to be saved. - * @return A Future containing a sequence of SchemaDefinition objects for the created schemas. - */ - def saveSchemaByStructureDefinition( - projectId: String, - structureDefinitionResources: Seq[Resource] - ): Future[Seq[SchemaDefinition]] - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaFolderRepository.scala deleted file mode 100644 index a661389c..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaFolderRepository.scala +++ /dev/null @@ -1,683 +0,0 @@ -package io.ignifyr.server.repository.schema - -import com.typesafe.scalalogging.Logger -import io.onfhir.api -import io.onfhir.api.util.IOUtil -import io.onfhir.api.validation.ProfileRestrictions -import io.onfhir.api.{FHIR_FOUNDATION_RESOURCES, FHIR_ROOT_URL_FOR_DEFINITIONS, Resource} -import io.onfhir.config.{BaseFhirConfig, FSConfigReader, IFhirConfigReader} -import io.onfhir.definitions.common.model.SchemaDefinition -import io.onfhir.definitions.common.util.HashUtil -import io.onfhir.definitions.resource.service.SimpleStructureDefinitionService -import io.onfhir.exception.InitializationException -import io.onfhir.util.JsonFormatter._ -import io.ignifyr.common.util.SchemaUtil -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.mapping.schema.SchemaConverter -import io.ignifyr.engine.model.exception.FhirMappingException -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.engine.util.{FhirVersionUtil, FileUtils} -import io.ignifyr.server.common.model.{AlreadyExists, BadRequest, ResourceNotFound} -import io.ignifyr.server.repository.project.IProjectRepository -import io.ignifyr.server.util.FileOperations -import org.apache.spark.sql.types.StructType - -import java.io.{File, FileWriter} -import java.nio.charset.StandardCharsets -import scala.collection.mutable -import scala.concurrent.Future -import scala.io.Source -import scala.language.postfixOps - -/** - * Folder/Directory based schema repository implementation. - * - * @param schemaRepositoryFolderPath - * @param projectRepository - */ -class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectRepository: IProjectRepository) - extends AbstractSchemaRepository { - - private val logger: Logger = Logger(this.getClass) - private var baseFhirConfig: BaseFhirConfig = initBaseFhirConfig() - private var simpleStructureDefinitionService = new SimpleStructureDefinitionService(baseFhirConfig) - - // Schema definition cache: project id -> schema id -> schema definition - private val schemaDefinitions: mutable.Map[String, mutable.Map[String, SchemaDefinition]] = - mutable.Map.empty[String, mutable.Map[String, SchemaDefinition]] - // Initialize the map for the first time - initMap(schemaRepositoryFolderPath) - - /** - * Retrieve all SchemaDefinitions - * - * @return - */ - override def getAllSchemas(projectId: String): Future[Seq[SchemaDefinition]] = { - Future { - schemaDefinitions - .get(projectId) - .map(_.values.toSeq) // If such a project exists, return the schema definitions as a sequence - .getOrElse(Seq.empty[SchemaDefinition]) // Else, return an empty list - } - } - - /** - * Retrieve the schema identified by its project and id. - * - * @param schemaId - * @return - */ - override def getSchema(projectId: String, schemaId: String): Future[Option[SchemaDefinition]] = { - Future { - schemaDefinitions(projectId).get(schemaId) - } - } - - /** - * Retrieve the schema identified by its url. - * - * @param url URL of the schema definition - * @return - */ - override def getSchemaByUrl(url: String): Future[Option[SchemaDefinition]] = { - Future { - schemaDefinitions.values.flatMap(_.values).find(_.url.equals(url)) - } - } - - /** - * Save the schema to the repository. - * - * @param schemaDefinition - * @return - */ - override def saveSchema(projectId: String, schemaDefinition: SchemaDefinition): Future[SchemaDefinition] = { - // Validate the given schema - var structureDefinitionResource: Resource = null - try { - structureDefinitionResource = SchemaUtil.convertToStructureDefinitionResource( - schemaDefinition, - IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion - ) - } catch { - case _: IllegalArgumentException => - throw BadRequest( - "Missing data type.", - s"A field definition must have at least one data type. Element rootPath: ${schemaDefinition.`type`}" - ) - } - try { - fhirConfigurator.validateGivenInfrastructureResources( - baseFhirConfig, - api.FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION, - Seq(structureDefinitionResource) - ) - } catch { - case e: Exception => - throw BadRequest( - "Schema definition is not valid.", - s"Schema definition cannot be validated: ${schemaDefinition.url}", - Some(e) - ) - } - - // Ensure that both the ID and "URL|version" (the canonical URL) of the schema is unique/ - // At this point, also ensure that the name of the schema is unique - // If id field is not provided within the schema object, the schemaID will be unique - // because in its constructor we put the MD5 hash of the URL into the id field for new schema creations. - checkIfSchemaIsUnique( - projectId, - schemaDefinition.id, - s"${schemaDefinition.url}|${schemaDefinition.version}", - Some(schemaDefinition.name) - ) - - // Check SchemaDefinition type is valid (It must start with an uppercase letter) - validateSchemaDefinitionType(schemaDefinition) - - // Write to the repository as a new file and update caches - writeSchemaAndUpdateCaches(projectId, structureDefinitionResource, schemaDefinition) - } - - /** - * Update the schema in the repository. - * - * @param projectId Project containing the schema definition - * @param schemaId Unique ID of the schema - * @param schemaDefinition Schema definition - * @return - */ - override def updateSchema( - projectId: String, - schemaId: String, - schemaDefinition: SchemaDefinition - ): Future[SchemaDefinition] = { - if (!schemaDefinitions.contains(projectId) || !schemaDefinitions(projectId).contains(schemaId)) { - throw ResourceNotFound( - "Schema does not exists.", - s"A schema definition with id $schemaId does not exists in the schema repository at ${FileUtils.getPath(schemaRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - - // Validate the given schema - var structureDefinitionResource: Resource = null - try { - structureDefinitionResource = SchemaUtil.convertToStructureDefinitionResource( - schemaDefinition, - IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion - ) - } catch { - case _: IllegalArgumentException => - throw BadRequest( - "Missing data type.", - s"A field definition must have at least one data type. Element rootPath: ${schemaDefinition.`type`}" - ) - } - try { - fhirConfigurator.validateGivenInfrastructureResources( - baseFhirConfig, - api.FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION, - Seq(structureDefinitionResource) - ) - } catch { - case e: Exception => - throw BadRequest( - "Schema definition is not valid.", - s"Schema definition cannot be validated: ${schemaDefinition.url}", - Some(e) - ) - } - - // Update the file - getFileForSchema(projectId, schemaDefinition.id).flatMap(file => { - val fw = new FileWriter(file) - fw.write(structureDefinitionResource.toPrettyJson) - fw.close() - - // Update cache - baseFhirConfig.profileRestrictions = SchemaManagementUtil.updateProfileRestrictionsMap( - baseFhirConfig.profileRestrictions, - schemaDefinition.url, - schemaDefinition.version, - fhirFoundationResourceParser.parseStructureDefinition( - structureDefinitionResource, - includeElementMetadata = true - ) - ) - schemaDefinitions(projectId).put(schemaId, schemaDefinition) - - // Update the project - projectRepository.updateSchema(projectId, schemaDefinition) map { _ => - schemaDefinition - } - }) - - } - - /** - * Delete the schema from the repository. - * - * @param schemaId Unique ID of the schema - * @return - */ - override def deleteSchema(projectId: String, schemaId: String): Future[Unit] = { - if (!schemaDefinitions(projectId).contains(schemaId)) { - throw ResourceNotFound( - "Schema does not exists.", - s"A schema with id $schemaId does not exists in the schema repository at ${FileUtils.getPath(schemaRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - - // delete schema file from repository - getFileForSchema(projectId, schemaId).flatMap(file => { - file.delete() - val schema: SchemaDefinition = schemaDefinitions(projectId)(schemaId) - // delete the schema from the in-memory map - schemaDefinitions(projectId).remove(schemaId) - // remove the url of the schema - baseFhirConfig.profileRestrictions -= schema.url - // Update project - projectRepository.deleteSchema(projectId, Some(schemaId)) - }) - } - - /** - * Deletes all schemas associated with a specific project. - * - * @param projectId The unique identifier of the project for which schemas should be deleted. - */ - override def deleteAllSchemas(projectId: String): Future[Unit] = { - Future { - // delete schema definitions for the project - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath(schemaRepositoryFolderPath, projectId).toFile) - // remove profile restrictions of project schemas - val schemaUrls: Set[String] = - schemaDefinitions.getOrElse(projectId, mutable.Map.empty).values.map(definition => definition.url).toSet - baseFhirConfig.profileRestrictions --= schemaUrls - // remove project from the cache - schemaDefinitions.remove(projectId) - } flatMap { _ => - // delete project schemas - projectRepository.deleteSchema(projectId) - } - } - - /** - * Gets the file for the given schema ID. - * This function locates the schema file under schemaRepositoryFolder/projectId/ folder assuming the name of the file is schemaId.json - * - * @param schemaDefinition - * @return - */ - private def getFileForSchema(projectId: String, schemaId: String): Future[File] = { - projectRepository.getProject(projectId) map { project => - if (project.isEmpty) - throw new IllegalStateException( - s"This should not be possible. ProjectId: $projectId does not exist in the project folder repository." - ) - FileOperations.getFileForEntityWithinProject(schemaRepositoryFolderPath, project.get.id, schemaId) - } - } - - /** - * Parses the given schema folder and initialize a SchemaDefinition map - * - * @param schemaRepositoryFolderPath - * @return - */ - private def initMap(schemaRepositoryFolderPath: String): Unit = { - val schemaFolder = FileUtils.getPath(schemaRepositoryFolderPath).toFile - logger.info(s"Initializing the Schema Repository from path ${schemaFolder.getAbsolutePath}.") - if (!schemaFolder.exists()) { - schemaFolder.mkdirs() - } - // We may need to give a warning if there are non-directories inside the scheme repository folder. - schemaFolder - .listFiles() - .foreach(projectFolder => { - var schemaFiles = Seq.empty[File] - try { - // We may need to give a warning if there are non-json files or other directories inside the project folders. - schemaFiles = IOUtil.getFilesFromFolder( - projectFolder, - recursively = true, - ignoreHidden = true, - withExtension = Some(FileExtensions.JSON.toString) - ) - } catch { - case e: Throwable => - throw FhirMappingException( - s"Given folder for the schema repository is not valid at path ${projectFolder.getAbsolutePath}", - e - ) - } - - // Read each file containing ProfileRestrictions and convert them to SchemaDefinitions - val projectSchemas: mutable.Map[String, SchemaDefinition] = mutable.Map.empty - schemaFiles.foreach { schemaFile => - val source = Source.fromFile(schemaFile, StandardCharsets.UTF_8.name()) // read the JSON file - val fileContent = - try source.mkString - finally source.close() - try { - val profileRestrictions: ProfileRestrictions = - fhirFoundationResourceParser.parseStructureDefinition(fileContent.parseJson) - - // We send the filename (stripped from its .json extension) as an id to the SchemaDefinition constructor because at this point we know that the file name is unique within the projectFolder - // It is not possible to have two files with the same name within a folder. - val schema: SchemaDefinition = simpleStructureDefinitionService.convertToSchemaDefinition( - profileRestrictions.id.getOrElse(IOUtil.removeFileExtension(schemaFile.getName)), - profileRestrictions - ) - // validate the 'type' field of schema definition - validateSchemaDefinitionType(schema) - if (FileOperations.checkFileNameMatchesEntityId(schema.id, schemaFile, "schema")) { - projectSchemas.put(schema.id, schema) - } // else case is logged within FileOperations.checkFileNameMatchesEntityId - } catch { - case e: Throwable => - logger.error(s"Failed to parse schema definition at ${schemaFile.getPath}", e) - System.exit(1) - } - } - if (projectSchemas.isEmpty) { - // No processable schema files under projectFolder - logger.warn( - s"There are no processable schema files under ${projectFolder.getAbsolutePath}. Skipping ${projectFolder.getName}." - ) - } else { - this.schemaDefinitions.put(projectFolder.getName, projectSchemas) - } - }) - } - - /** - * Comparison function for two SchemaDefinitions. The definitions are compared according to their names - * - * @param s1 - * @param s2 - * @return - */ - private def schemaComparisonFunc(s1: SchemaDefinition, s2: SchemaDefinition): Boolean = { - s1.name.compareTo(s2.name) < 0 - } - - /** - * Read the schema given with the url and convert it to the Spark schema - * - * @param schemaUrl URL of the schema - * @return - */ - override def getSchema(schemaUrl: String): Option[StructType] = { - schemaDefinitions.values - .flatMap(_.values) // Flatten all the schemas managed for all projects - .find(_.url.contentEquals(schemaUrl)) // Find the desired url - .map(s => { - val decomposedSchema: Resource = SchemaUtil.convertToStructureDefinitionResource( - s, - IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion - ) // Schema definition in the FHIR Resource representation - new SchemaConverter(FhirVersionUtil.getMajorFhirVersion(IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion)) - .convertSchema(decomposedSchema) - }) - } - - /** - * Try to initialize the BaseFhirConfig. Otherwise print the error message and halt. - * - * @return - */ - private def initBaseFhirConfig(): BaseFhirConfig = { - // BaseFhirConfig will act as a validator for the schema definitions by holding the ProfileDefinitions in memory - val fhirConfigReader: IFhirConfigReader = new FSConfigReader( - fhirVersion = FhirVersionUtil.getMajorFhirVersion(IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion), - profilesPath = Some(FileUtils.getPath(schemaRepositoryFolderPath).toString) - ) - - val folder = FileUtils.getPath(schemaRepositoryFolderPath).toFile - if (!folder.exists()) { - folder.mkdirs() - } - var baseFhirConfig: BaseFhirConfig = null - try { - baseFhirConfig = fhirConfigurator.initializePlatform(fhirConfigReader) - } catch { - case error: InitializationException => - logger.error(error.getMessage) - System.exit(1) - } - baseFhirConfig - } - - /** - * Retrieve the Structure Definition of the schema identified by its id. - * - * @param projectId Project containing the schema definition - * @param schemaId Identifier of the schema definition - * @return Structure definition of the schema converted into StructureDefinition Resource - */ - override def getSchemaAsStructureDefinition(projectId: String, schemaId: String): Future[Option[Resource]] = { - getSchema(projectId, schemaId).map { - case Some(schemaStructureDefinition) => - Some( - SchemaUtil.convertToStructureDefinitionResource( - schemaStructureDefinition, - IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion - ) - ) - case None => - None - } - } - - /** - * Saves the schemas by using their Structure Definition resources. - * - * This method validates the given structure definition resources, processes them to create schema definitions, - * and saves these definitions to the repository. It also ensures that the schema definitions are unique within - * the specified project and updates the cache accordingly. - * - * Throws: - * - BadRequest: If the schema resource cannot be validated. - * - * @param projectId The identifier of the project in which the schemas will be created. - * @param structureDefinitionResources A sequence of structure definition resources for the schemas. - * @return A Future containing a sequence of SchemaDefinition objects representing the created schemas. - */ - override def saveSchemaByStructureDefinition( - projectId: String, - structureDefinitionResources: Seq[Resource] - ): Future[Seq[SchemaDefinition]] = { - // Extract the URLs of the schemas that are about to be saved. These URLs will be used later to validate - // any referenced schemas within the current schema, ensuring they exist either as base FHIR definitions, - // already present schemas, or as part of this batch. - val schemaUrls = structureDefinitionResources.map(resource => (resource \ "url").extract[String]) - // convert each Resource to a SchemaDefinition - val schemaDefinitions: Seq[SchemaDefinition] = structureDefinitionResources.map(structureDefinitionResource => { - // Validate the resource - try { - fhirConfigurator.validateGivenInfrastructureResources( - baseFhirConfig, - api.FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION, - Seq(structureDefinitionResource) - ) - } catch { - case e: Exception => - throw BadRequest("Schema resource is not valid.", s"Schema resource cannot be validated.", Some(e)) - } - // Validate that all referenced schemas in the current schema exist or are part of the schemas to be saved. - validateReferencedSchemas(projectId, structureDefinitionResource, schemaUrls) - - // Create structureDefinition from the resource - val structureDefinition: ProfileRestrictions = fhirFoundationResourceParser.parseStructureDefinition( - structureDefinitionResource, - includeElementMetadata = true - ) - // Generate an Id if id is missing - val schemaId = structureDefinition.id.getOrElse(HashUtil.md5Hash(structureDefinition.url)) - val version = structureDefinition.version.getOrElse( - SchemaDefinition.VERSION_LATEST - ) // We always use the "latest" version if there is no version! - checkIfSchemaIsUnique(projectId, schemaId, s"${structureDefinition.url}|$version", None) - - // To use convertToSchemaDefinition, profileRestrictions sequence must include the structure definition. Add it before conversion - baseFhirConfig.profileRestrictions = SchemaManagementUtil.updateProfileRestrictionsMap( - baseFhirConfig.profileRestrictions, - structureDefinition.url, - version, - structureDefinition - ) - // This part is a little ugly because we update baseFhirConfig.profileRestrictions with the version that we evaluate above and then - // let concertToSchemaDefinition put a version to the created schemaDefinition. We rely on that both our above version assignment - // and convertToSchema uses the same method to evaluate the value for the version. - val schemaDefinition = simpleStructureDefinitionService.convertToSchemaDefinition(schemaId, structureDefinition) - // Remove structure definition from the cache and add it after file writing is done to ensure files and cache are the same - baseFhirConfig.profileRestrictions -= structureDefinition.url - - // Check SchemaDefinition type is valid. - this.validateSchemaDefinitionType(schemaDefinition) - schemaDefinition - }) - // write the schemas to the repository as a new file and update caches - val futures: Seq[Future[SchemaDefinition]] = schemaDefinitions.zipWithIndex.map { case (schemaDefinition, index) => - writeSchemaAndUpdateCaches(projectId, structureDefinitionResources.lift(index).get, schemaDefinition) - } - Future.sequence(futures) - } - - /** - * Checks if ID of the schema is unique in the project and schema URL (including the version) is unique among all schemas (within all projects) - * Throws: - * {@link AlreadyExists} with the code 409 if the schema ID is not unique in the project or the schema URL is not unique - * - * @param projectId Identifier of the project to check schema ID's in it - * @param schemaId Identifier of the schema - * @param schemaCanonicalUrl Canonical Url of the schema with its version - * @param schemaName Name of the schema. If provided its uniqueness is checked within the given projectId. - */ - private def checkIfSchemaIsUnique( - projectId: String, - schemaId: String, - schemaUrl: String, - schemaName: Option[String] - ): Unit = { - // Check the uniqueness of the schemaId - schemaDefinitions.get(projectId).flatMap(_.get(schemaId)).foreach { _ => - throw AlreadyExists( - "Schema already exists.", - s"A schema definition with id $schemaId already exists in the schema repository at ${FileUtils.getPath(schemaRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - - // Collect the Canonical URLs of all schema definitions and check the uniqueness of the schemaUrl - val schemaCanonicalUrls: Map[String, String] = - schemaDefinitions.values.flatMap(_.values).map(schema => s"${schema.url}|${schema.version}" -> schema.name).toMap - schemaCanonicalUrls.get(schemaUrl).foreach { schemaName => - throw AlreadyExists( - "Schema already exists.", - s"A schema definition with url $schemaUrl already exists. Check the schema '$schemaName'" - ) - } - - // If a schemaName exists, check the uniqueness of the schemaName - schemaName.foreach { name => - schemaDefinitions.get(projectId).foreach { schemas => - if (schemas.values.exists(_.name == name)) { - throw AlreadyExists( - "Schema already exists.", - s"A schema definition with name $name already exists in the schema repository at ${FileUtils.getPath(schemaRepositoryFolderPath).toAbsolutePath.toString}" - ) - } - } - } - - } - - /** - * Write the schema file and update the caches accordingly - * - * @param projectId Id of the project that will include the schema - * @param structureDefinitionResource Schema resource that will be written - * @param schemaDefinition Definition of the schema - * @return - */ - private def writeSchemaAndUpdateCaches( - projectId: String, - structureDefinitionResource: Resource, - schemaDefinition: SchemaDefinition - ): Future[SchemaDefinition] = { - getFileForSchema(projectId, schemaDefinition.id).flatMap(newFile => { - val fw = new FileWriter(newFile) - fw.write(structureDefinitionResource.toPrettyJson) - fw.close() - - // Update the caches with the new schema - baseFhirConfig.profileRestrictions = SchemaManagementUtil.updateProfileRestrictionsMap( - baseFhirConfig.profileRestrictions, - schemaDefinition.url, - schemaDefinition.version, - fhirFoundationResourceParser.parseStructureDefinition( - structureDefinitionResource, - includeElementMetadata = true - ) - ) - schemaDefinitions.getOrElseUpdate(projectId, mutable.Map.empty).put(schemaDefinition.id, schemaDefinition) - - // Update the project with the schema - projectRepository.addSchema(projectId, schemaDefinition) map { _ => - schemaDefinition - } - }) - } - - /** - * Check SchemaDefinition type starts with uppercase. - * - * @param schemaDefinition Definition of the schema - * @return - */ - private def validateSchemaDefinitionType(schemaDefinition: SchemaDefinition): Unit = { - val schemaDefinitionType: String = schemaDefinition.`type` - if (schemaDefinitionType.isEmpty || schemaDefinitionType.apply(0).isLower) { - throw BadRequest( - "Schema definition is not valid.", - s"Schema definition type must start with an uppercase letter!" - ) - } - } - - /** - * Validates that all referenced profiles in a given schema either exist as base FHIR definitions, - * are already present in the system, or are included in the list of schemas to be created. - * - * @param projectId The ID of the project in which the schemas are being validated. - * @param schemaResource The FHIR StructureDefinition resource representing the schema to be validated. - * @param schemaUrls The list of URLs of the schemas that are included in the current batch to be created. - * @throws BadRequest if any referenced profile is missing. - */ - private def validateReferencedSchemas(projectId: String, schemaResource: Resource, schemaUrls: Seq[String]): Unit = { - - /** - * Validates a single profile URL to ensure it either exists as a base FHIR definition, - * is already present in the system, or is included in the provided schema URLs list. - * - * @param profile The URL of the profile to be validated. - * @param schemaUrl The URL of the schema that references this profile, used for error reporting. - * @throws BadRequest if the profile is missing. - */ - def validateProfile(profile: String, schemaUrl: String): Unit = { - if ( - !profile.startsWith(s"$FHIR_ROOT_URL_FOR_DEFINITIONS/${FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION}") && - !schemaUrls.contains(profile) && - !schemaDefinitions(projectId).values.exists(s => s.url.contentEquals(profile)) - ) { - throw BadRequest( - "Invalid Schema Reference !", - s"The schema with URL '$schemaUrl' references a non-existent schema: '$profile'. Ensure all referenced schemas exist." - ) - } - } - - // Validate the base definition of the schema - val baseDefinitionUrl: String = (schemaResource \ "baseDefinition").extract[String] - val schemaUrl: String = (schemaResource \ "url").extract[String] - validateProfile(baseDefinitionUrl, schemaUrl) - - // Validate the profiles associated with each element in the schema - val elements = (schemaResource \ "differential" \ "element").extract[Seq[Resource]] - elements.foreach { element => - val types = (element \ "type").extract[Seq[Resource]] - types.foreach { elementType => - val profiles = (elementType \ "profile").extract[Seq[String]] - val targetProfiles = (elementType \ "targetProfile").extract[Seq[String]] - val allProfiles = profiles ++ targetProfiles - allProfiles.foreach(profile => validateProfile(profile, schemaUrl)) - } - } - } - - /** - * Reload the schema definitions from the given folder - * - * @return - */ - override def invalidate(): Unit = { - this.schemaDefinitions.clear() - this.baseFhirConfig = initBaseFhirConfig() - this.simpleStructureDefinitionService = new SimpleStructureDefinitionService(this.baseFhirConfig) - initMap(schemaRepositoryFolderPath) - } - - /** - * Retrieve the projects and SchemaDefinitions within. - * - * @return Map of projectId -> Seq[SchemaDefinition] - */ - override def getProjectPairs: Map[String, Seq[SchemaDefinition]] = { - schemaDefinitions.map { case (projectId, schemaPairs) => - projectId -> schemaPairs.values.toSeq - }.toMap - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaManagementUtil.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaManagementUtil.scala deleted file mode 100644 index e5d6f04e..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/schema/SchemaManagementUtil.scala +++ /dev/null @@ -1,37 +0,0 @@ -package io.ignifyr.server.repository.schema - -import io.onfhir.api.validation.ProfileRestrictions - -/** - * Utility functions to help the management of the schema within the ISchemaRepository and relations with onFHIR ProfileRestrictions - */ -object SchemaManagementUtil { - - /** - * Return an updated map for the ProfileRestrictions (of a StructureDefinition resource) - * - * @param profileRestrictionsMap The ProfileRestrictions map to be updated - * @param schemaUrl URL of the schema to be updated (or to be added to the map) - * @param schemaVersion Version of the schema to be updated (or to be added). A Schema can have multiple versions (so a schema URL can have multiple versions) - * @param newProfileRestrictions The new ProfileRestrictions to be put into the map. - * @return - */ - def updateProfileRestrictionsMap( - profileRestrictionsMap: Map[String, Map[String, ProfileRestrictions]], - schemaUrl: String, - schemaVersion: String, - newProfileRestrictions: ProfileRestrictions - ): Map[String, Map[String, ProfileRestrictions]] = { - profileRestrictionsMap.updated( - schemaUrl, - profileRestrictionsMap.get(schemaUrl) match { - case Some(versionMap) => - // Update the existing inner map for the URL - versionMap.updated(schemaVersion, newProfileRestrictions) - case None => - // If no map exists for the URL, create a new inner map with the version - Map(schemaVersion -> newProfileRestrictions) - } - ) - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/ITerminologySystemRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/ITerminologySystemRepository.scala deleted file mode 100644 index a516eaf9..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/ITerminologySystemRepository.scala +++ /dev/null @@ -1,52 +0,0 @@ -package io.ignifyr.server.repository.terminology - -import io.ignifyr.engine.repository.ICachedRepository -import io.ignifyr.server.model.TerminologySystem - -import scala.concurrent.Future - -/** - * Interface to save and load TerminologySystem so that the client applications can manage the TerminologySystem - * through CRUD operations - */ -trait ITerminologySystemRepository extends ICachedRepository { - - /** - * Retrieve the metadata of all TerminologySystems - * - * @return Seq[TerminologySystem] - */ - def getTerminologySystemsMetadata: Future[Seq[TerminologySystem]] - - /** - * Create a new TerminologySystem - * - * @return TerminologySystem - */ - def createTerminologySystem(terminology: TerminologySystem): Future[TerminologySystem] - - /** - * Retrieve a TerminologySystem - * - * @param id id of the TerminologySystem - * @return TerminologySystem if found - */ - def getTerminologySystem(id: String): Future[Option[TerminologySystem]] - - /** - * Update a TerminologySystem - * - * @param id id of the TerminologySystem to be updated - * @param terminology TerminologySystem to update - * @return updated TerminologySystem - */ - def updateTerminologySystem(id: String, terminology: TerminologySystem): Future[TerminologySystem] - - /** - * Remove a TerminologySystem - * - * @param id id of the TerminologySystem to be removed - * @return - */ - def deleteTerminologySystem(id: String): Future[Unit] -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/TerminologySystemFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/TerminologySystemFolderRepository.scala deleted file mode 100644 index e702abf1..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/TerminologySystemFolderRepository.scala +++ /dev/null @@ -1,294 +0,0 @@ -package io.ignifyr.server.repository.terminology - -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.repository.ICachedRepository -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.common.model.{AlreadyExists, BadRequest, ResourceNotFound} -import io.ignifyr.server.model.TerminologySystem -import io.ignifyr.server.repository.terminology.TerminologySystemFolderRepository.getTerminologySystemsJsonPath -import io.ignifyr.server.util.FileOperations - -import java.io.File -import scala.collection.mutable -import scala.concurrent.Future - -/** - * Folder/Directory based terminology system repository implementation. - */ -class TerminologySystemFolderRepository(terminologySystemsFolderPath: String) - extends ITerminologySystemRepository - with ICachedRepository { - - // terminology system id -> TerminologySystem - private val terminologySystemMap: mutable.Map[String, TerminologySystem] = - mutable.Map.empty[String, TerminologySystem] - // Initialize the map for the first time - initMap() - - /** - * Retrieve the metadata of all TerminologySystems - * - * @return Seq[TerminologySystem] - */ - override def getTerminologySystemsMetadata: Future[Seq[TerminologySystem]] = { - Future { - this.terminologySystemMap.values.map(_.copy(conceptMaps = Seq.empty, codeSystems = Seq.empty)).toSeq - } - } - - /** - * Create a new TerminologySystem by creating its folder and files in it. - * - * @return TerminologySystem - */ - override def createTerminologySystem(terminologySystem: TerminologySystem): Future[TerminologySystem] = { - Future { - this.validate(terminologySystem) - // check if id exists - if (this.terminologySystemMap.contains(terminologySystem.id)) { - throw AlreadyExists("TerminologySystem already exists.", s"Id ${terminologySystem.id} already exists.") - } - val terminologySystemFolder = FileUtils.getPath(terminologySystemsFolderPath, terminologySystem.id).toFile - // Update the internal JSON file that we use as our database - val terminologySystemSeq = this.terminologySystemMap.values.toSeq :+ terminologySystem - this.updateTerminologySystemsDBFile(terminologySystemSeq) - - terminologySystemFolder.mkdirs() // create its folder - this.createConceptMapAndCodeSystemFiles(terminologySystem) // create concept maps/code systems files - // Update the map that we use as a cache - this.terminologySystemMap.put(terminologySystem.id, terminologySystem) - terminologySystem - } - } - - /** - * Retrieve a TerminologySystem - * - * @param id id of the TerminologySystem - * @return TerminologySystem - */ - override def getTerminologySystem(id: String): Future[Option[TerminologySystem]] = { - Future { - this.terminologySystemMap.get(id) - } - } - - /** - * Update a TerminologySystem - * - * @param id id of the TerminologySystem - * @param terminologySystem TerminologySystem to update - * @return updated TerminologySystem - */ - override def updateTerminologySystem(id: String, terminologySystem: TerminologySystem): Future[TerminologySystem] = { - // cross check ids - if (id != terminologySystem.id) { - throw BadRequest( - "Terminology System IDs do not match.", - s"Id $id does not match with the id in the body ${terminologySystem.id}." - ) - } - this.validate(terminologySystem) - // find the terminology system - this.terminologySystemMap.get(id) match { - case Some(foundTerminology) => - // update the terminology system metadata in the database - val updatedLocalTerminologies = this.terminologySystemMap.values.map { - case t if t.id == id => terminologySystem - case t => t - }.toSeq - this.updateTerminologySystemsDBFile(updatedLocalTerminologies) - // update concept maps/code systems files - this.updateConceptMapAndCodeSystemFiles(foundTerminology, terminologySystem) map (_ => { - // update the terminology service in the map - this.terminologySystemMap.put(id, terminologySystem) - terminologySystem - }) - case None => throw ResourceNotFound("Terminology system not found.", s"Terminology system with id $id not found.") - } - } - - /** - * Remove a TerminologySystem - * - * @param id id of the TerminologySystem - * @return - */ - override def deleteTerminologySystem(id: String): Future[Unit] = { - Future { - // find the terminology system from the cache - this.terminologySystemMap.get(id) match { - case Some(foundTerminology) => - // delete the terminology metadata - val updatedTerminologies = this.terminologySystemMap.values.toSeq.filterNot(_.id == id) - this.updateTerminologySystemsDBFile(updatedTerminologies) - // delete the terminology folder - val terminologyFolder = FileUtils.getPath(terminologySystemsFolderPath, foundTerminology.id).toFile - org.apache.commons.io.FileUtils.deleteDirectory(terminologyFolder) - // remove the terminology from the map - this.terminologySystemMap.remove(id) - case None => - throw ResourceNotFound("Terminology system not found.", s"Terminology system with id $id not found.") - } - } - } - - /** - * Create concept map and code system files in the terminology system folder. - * - * @param terminologySystem TerminologySystem - * @return - */ - private def createConceptMapAndCodeSystemFiles(terminologySystem: TerminologySystem): Future[Unit] = { - Future { - terminologySystem.conceptMaps.foreach(conceptMap => { - val file = FileUtils.getPath(terminologySystemsFolderPath, terminologySystem.id, conceptMap.id).toFile - FileOperations.writeStringContentToFile(file, TerminologySystemFolderRepository.DEFAULT_CONCEPT_MAP_COLUMNS) - }) - terminologySystem.codeSystems.foreach(codeSystem => { - val file = FileUtils.getPath(terminologySystemsFolderPath, terminologySystem.id, codeSystem.id).toFile - FileOperations.writeStringContentToFile(file, TerminologySystemFolderRepository.DEFAULT_CODE_SYSTEM_COLUMNS) - }) - } - } - - /** - * Create/delete concept map and code system files in the terminology folder according to the new TerminologySystem - * - * @param existingTerminologySystem - * @param newTerminologySystem - * @return - */ - private def updateConceptMapAndCodeSystemFiles( - existingTerminologySystem: TerminologySystem, - newTerminologySystem: TerminologySystem - ): Future[Unit] = { - Future { - // find intersection ids of concept map and code system in local terminologies - val currentConceptMapIds = existingTerminologySystem.conceptMaps.map(_.id) - val currentCodeSystemIds = existingTerminologySystem.codeSystems.map(_.id) - val newConceptMapIds = newTerminologySystem.conceptMaps.map(_.id) - val newCodeSystemIds = newTerminologySystem.codeSystems.map(_.id) - // find ids to delete and add - val conceptMapIdsToDelete = currentConceptMapIds.diff(newConceptMapIds) - val conceptMapIdsToAdd = newConceptMapIds.diff(currentConceptMapIds) - val codeSystemIdsToDelete = currentCodeSystemIds.diff(newCodeSystemIds) - val codeSystemIdsToAdd = newCodeSystemIds.diff(currentCodeSystemIds) - // delete concept maps - conceptMapIdsToDelete.foreach { id => - val conceptMap = existingTerminologySystem.conceptMaps.find(_.id == id).get - val file = FileUtils.getPath(terminologySystemsFolderPath, existingTerminologySystem.id, conceptMap.id).toFile - file.delete() - } - // delete code systems - codeSystemIdsToDelete.foreach { id => - val codeSystem = existingTerminologySystem.codeSystems.find(_.id == id).get - val file = FileUtils.getPath(terminologySystemsFolderPath, existingTerminologySystem.id, codeSystem.id).toFile - file.delete() - } - // add concept maps - conceptMapIdsToAdd.foreach { id => - val conceptMap = newTerminologySystem.conceptMaps.find(_.id == id).get - val file = FileUtils.getPath(terminologySystemsFolderPath, existingTerminologySystem.id, conceptMap.id).toFile - FileOperations.writeStringContentToFile(file, TerminologySystemFolderRepository.DEFAULT_CONCEPT_MAP_COLUMNS) - } - // add code systems - codeSystemIdsToAdd.foreach { id => - val codeSystem = newTerminologySystem.codeSystems.find(_.id == id).get - val file = FileUtils.getPath(terminologySystemsFolderPath, existingTerminologySystem.id, codeSystem.id).toFile - FileOperations.writeStringContentToFile(file, TerminologySystemFolderRepository.DEFAULT_CODE_SYSTEM_COLUMNS) - } - } - } - - /** - * Read local terminology file/folder if exists, otherwise create a new one - * - * @return - */ - private def readTerminologySystemsDBFile(): Seq[TerminologySystem] = { - val terminologySystemsFolder = FileUtils.getPath(terminologySystemsFolderPath).toFile - if (!terminologySystemsFolder.exists()) { - terminologySystemsFolder.mkdirs() - } - val terminologySystemsJsonFile = - FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemsFolderPath)).toFile - if (!terminologySystemsJsonFile.exists()) { - terminologySystemsJsonFile.createNewFile() - FileOperations.writeJsonContent(terminologySystemsJsonFile, Seq.empty[TerminologySystem]) - } - val availableTerminologySystems = FileOperations.readJsonContent[TerminologySystem](terminologySystemsJsonFile) - availableTerminologySystems - } - - /** - * Initialize the map that we use as cache. - * - * @return - */ - private def initMap(): Unit = { - val termonologySystems = readTerminologySystemsDBFile() - termonologySystems.foreach(terminology => { - this.terminologySystemMap.put(terminology.id, terminology) - }) - } - - /** - * Validate the given TerminologySystem - * - * @param terminologySystem - */ - private def validate(terminologySystem: TerminologySystem): Unit = { - if (terminologySystem.name == null || terminologySystem.name.isEmpty) { - throw BadRequest("TerminologySystem name cannot be empty", s"TerminologySystem id: ${terminologySystem.id}") - } - terminologySystem.conceptMaps.foreach(conceptMap => { - if (conceptMap.name == null || conceptMap.name.isEmpty) { - throw BadRequest("Concept map name cannot be empty", s"TerminologySystem id: ${terminologySystem.id}") - } - }) - terminologySystem.codeSystems.foreach(codeSystem => { - if (codeSystem.name == null || codeSystem.name.isEmpty) { - throw BadRequest("Code system name cannot be empty", s"TerminologySystem id: ${terminologySystem.id}") - } - }) - // combine and check if concept map/code system name is unique - val names = terminologySystem.conceptMaps.map(_.name) ++ terminologySystem.codeSystems.map(_.name) - if (names.distinct.length != names.length) { - throw BadRequest( - "A name cannot be used more than once across concept maps and code systems within a TerminologySystem", - s"TerminologySystem id: ${terminologySystem.id}." - ) - } - } - - private def updateTerminologySystemsDBFile(terminologySystems: Seq[TerminologySystem]): Unit = { - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemsFolderPath)).toFile - FileOperations.writeJsonContent(localTerminologyFile, terminologySystems) - } - - /** - * Reload the terminology systems from the given folder - * - * @return - */ - def invalidate(): Unit = { - this.terminologySystemMap.clear() - initMap() - } -} - -object TerminologySystemFolderRepository { - private val DEFAULT_CONCEPT_MAP_COLUMNS = "source_system,source_code,target_system,target_code" - private val DEFAULT_CODE_SYSTEM_COLUMNS = "code,display" - - /** - * Returns the path for the JSON file keeping the terminology systems - * - * @param terminologySystemFolder - * @return - */ - def getTerminologySystemsJsonPath(terminologySystemFolder: String): String = { - terminologySystemFolder + File.separator + "terminology-systems.json" - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/CodeSystemFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/CodeSystemFolderRepository.scala deleted file mode 100644 index bb293513..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/CodeSystemFolderRepository.scala +++ /dev/null @@ -1,313 +0,0 @@ -package io.ignifyr.server.repository.terminology.codesystem - -import akka.stream.scaladsl.{FileIO, Source} -import akka.util.ByteString -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.common.model.{AlreadyExists, BadRequest, ResourceNotFound} -import io.ignifyr.server.model.TerminologySystem.TerminologyCodeSystem -import io.ignifyr.server.model._ -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.terminology.TerminologySystemFolderRepository.getTerminologySystemsJsonPath -import io.ignifyr.server.util.{CsvUtil, FileOperations} -import org.json4s.jackson.Serialization.writePretty - -import java.io.FileWriter -import scala.concurrent.Future - -class CodeSystemFolderRepository(terminologySystemFolderPath: String) extends ICodeSystemRepository { - - /** - * Retrieve the code system within a terminology - * - * @return Seq[TerminologyCodeSystem] - */ - override def getCodeSystems(terminologyId: String): Future[Seq[TerminologyCodeSystem]] = { - Future { - findLocalTerminologyById(terminologyId).codeSystems - } - } - - /** - * Create a new code system within a terminology - * - * @return TerminologyCodeSystem - */ - override def createCodeSystem( - terminologyId: String, - codeSystem: TerminologyCodeSystem - ): Future[TerminologyCodeSystem] = { - Future { - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - val localTerminology = FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - // check if code system id exists - localTerminology.find(_.id == terminologyId) match { - case Some(t) => - if (t.codeSystems.exists(_.id == codeSystem.id)) { - throw AlreadyExists( - "Local terminology code system id already exists.", - s"Id ${codeSystem.id} already exists." - ) - } - // update local terminology file - val newConceptMaps = t.codeSystems :+ codeSystem - val newLocalTerminology = localTerminology.map(t => - if (t.id == terminologyId) t.copy(codeSystems = newConceptMaps) - else t - ) - val writer = new FileWriter(localTerminologyFile) - try { - writer.write(writePretty(newLocalTerminology)) - } finally { - writer.close() - } - // create code system file - val codeSystemFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, codeSystem.id).toFile - codeSystemFile.createNewFile() - codeSystem - - case None => - throw BadRequest("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - } - - /** - * Retrieve a code system within a terminology - * - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return TerminologyCodeSystem if found - */ - override def getCodeSystem(terminologyId: String, codeSystemId: String): Future[TerminologyCodeSystem] = { - Future { - findCodeSystemById(terminologyId, codeSystemId) - } - } - - /** - * Update a code system within a terminology - * - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param codeSystem TerminologyCodeSystem to update - * @return updated TerminologyCodeSystem - */ - override def updateCodeSystem( - terminologyId: String, - codeSystemId: String, - codeSystem: TerminologyCodeSystem - ): Future[TerminologyCodeSystem] = { - Future { - // cross check ids - if (codeSystem.id != codeSystemId) { - throw BadRequest( - "Code system id does not match.", - s"Code system id ${codeSystem.id} does not match $codeSystemId." - ) - } - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - val localTerminology = FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - // check if code system id exists in json file - localTerminology.find(_.id == terminologyId) match { - case Some(t) => - if (!t.codeSystems.exists(_.id == codeSystemId)) { - throw ResourceNotFound( - "Local terminology code system not found.", - s"Local terminology code system with id $codeSystemId not found." - ) - } - val newCodeSystems = t.codeSystems.map(cm => - if (cm.id == codeSystemId) codeSystem - else cm - ) - val newLocalTerminology = localTerminology.map(t => - if (t.id == terminologyId) t.copy(codeSystems = newCodeSystems) - else t - ) - val writer = new FileWriter(localTerminologyFile) - try { - writer.write(writePretty(newLocalTerminology)) - } finally { - writer.close() - } - codeSystem - case None => - throw BadRequest("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - } - - /** - * Remove a code system within a terminology - * - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return - */ - override def removeCodeSystem(terminologyId: String, codeSystemId: String): Future[Unit] = { - Future { - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - val localTerminology = FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - localTerminology.find(_.id == terminologyId) match { - case Some(t) => - // check if code system id exists in json file - if (!t.codeSystems.exists(_.id == codeSystemId)) { - throw ResourceNotFound( - "Local terminology code system not found.", - s"Local terminology code system with id $codeSystemId not found." - ) - } - // remove code system from json file - val newCodeSystems = t.codeSystems.filterNot(_.id == codeSystemId) - val newLocalTerminology = localTerminology.map(t => - if (t.id == terminologyId) t.copy(codeSystems = newCodeSystems) - else t - ) - val writer = new FileWriter(localTerminologyFile) - try { - writer.write(writePretty(newLocalTerminology)) - } finally { - writer.close() - } - // remove concept map file - val codeSystemFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, codeSystemId).toFile - codeSystemFile.delete() - case None => - throw BadRequest("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - } - - /** - * Update the code system header by its id - * @param terminologyId terminology id the code system belongs to - * @param codeSystemId code system id - * @param headers new headers to update - * @return - */ - def updateCodeSystemHeader(terminologyId: String, codeSystemId: String, headers: Seq[CsvHeader]): Future[Unit] = { - val codeSystem = findCodeSystemById(terminologyId, codeSystemId) - // get file and update headers - val codeSystemFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, codeSystem.id).toFile - CsvUtil.writeCsvHeaders(codeSystemFile, headers) - } - - /** - * Retrieve and save the content of a code system csv file within a terminology - * - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param content Source of the csv file - * @return - */ - override def saveCodeSystemContent( - terminologyId: String, - codeSystemId: String, - content: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] = { - val codeSystem = findCodeSystemById(terminologyId, codeSystemId) - // save code system file - val codeSystemFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, codeSystem.id).toFile - CsvUtil.writeCsvAndReturnRowNumber(codeSystemFile, content, pageNumber, pageSize) - } - - /** - * Retrieve the content of a code system csv file within a terminology - * - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return Source of the csv file - */ - override def getCodeSystemContent( - terminologyId: String, - codeSystemId: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] = { - val codeSystem = findCodeSystemById(terminologyId, codeSystemId) - // get code system file - val codeSystemFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, codeSystem.id).toFile - CsvUtil.getPaginatedCsvContent(codeSystemFile, pageNumber, pageSize) - } - - /** - * Upload the code system to the repository - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param content Source of the csv file - * @return - */ - override def uploadCodeSystem( - terminologyId: String, - codeSystemId: String, - content: Source[ByteString, Any] - ): Future[Unit] = { - val codeSystem = findCodeSystemById(terminologyId, codeSystemId) - // save code system file - val codeSystemFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, codeSystem.id).toFile - CsvUtil.saveFileContent(codeSystemFile, content) - } - - /** - * Download the content of a code system csv file within a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return Source of the csv file - */ - override def downloadCodeSystem(terminologyId: String, codeSystemId: String): Future[Source[ByteString, Any]] = { - Future { - val codeSystem = findCodeSystemById(terminologyId, codeSystemId) - // get code system file - val codeSystemFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, codeSystem.id) - FileIO.fromPath(codeSystemFile) - } - } - - /** - * Find local terminology by id from json file - * JSON file contains a list of [[TerminologySystem]] - * If not found, throw ResourceNotFound - * @param terminologyId id of the terminology - * @return - */ - private def findLocalTerminologyById(terminologyId: String): TerminologySystem = { - val localTerminologies = getLocalTerminologiesFromFile - localTerminologies.find(_.id == terminologyId) match { - case Some(t) => t - case None => throw ResourceNotFound("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - - /** - * Find code system by id from a local terminology - * If either terminology or code system not found, throw ResourceNotFound - * @param terminologyId id of the terminology - * @param codeSystemId code system id to find inside the terminology - * @return - */ - private def findCodeSystemById(terminologyId: String, codeSystemId: String): TerminologyCodeSystem = { - val localTerminology = findLocalTerminologyById(terminologyId) - localTerminology.codeSystems.find(_.id == codeSystemId) match { - case Some(cs) => cs - case None => - throw ResourceNotFound( - "Local terminology code system not found.", - s"Local terminology code system with id $codeSystemId not found." - ) - } - } - - /** - * Read terminology systems JSON and return as a list of [[TerminologySystem]] - * @return - */ - private def getLocalTerminologiesFromFile: Seq[TerminologySystem] = { - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/ICodeSystemRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/ICodeSystemRepository.scala deleted file mode 100644 index 2cae097e..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/codesystem/ICodeSystemRepository.scala +++ /dev/null @@ -1,108 +0,0 @@ -package io.ignifyr.server.repository.terminology.codesystem - -import akka.stream.scaladsl.Source -import akka.util.ByteString -import io.ignifyr.server.model.TerminologySystem.TerminologyCodeSystem -import io.ignifyr.server.model.csv.CsvHeader - -import scala.concurrent.Future - -trait ICodeSystemRepository { - - /** - * Retrieve the code systems within a terminology - * - * @return Seq[TerminologyCodeSystem] - */ - def getCodeSystems(terminologyId: String): Future[Seq[TerminologyCodeSystem]] - - /** - * Create a new code system within a terminology - * @return TerminologyCodeSystem - */ - def createCodeSystem(terminologyId: String, codeSystem: TerminologyCodeSystem): Future[TerminologyCodeSystem] - - /** - * Retrieve a code system within a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return TerminologyCodeSystem if found - */ - def getCodeSystem(terminologyId: String, codeSystemId: String): Future[TerminologyCodeSystem] - - /** - * Update a code system within a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param codeSystem TerminologyCodeSystem to update - * @return updated TerminologyCodeSystem - */ - def updateCodeSystem( - terminologyId: String, - codeSystemId: String, - codeSystem: TerminologyCodeSystem - ): Future[TerminologyCodeSystem] - - /** - * Remove a code system within a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return - */ - def removeCodeSystem(terminologyId: String, codeSystemId: String): Future[Unit] - - /** - * Update the code system header by its id - * @param terminologyId terminology id the code system belongs to - * @param codeSystemId code system id e.g. icd9-to-icd10.csv - * @param headers new headers to update - * @return - */ - def updateCodeSystemHeader(terminologyId: String, codeSystemId: String, headers: Seq[CsvHeader]): Future[Unit] - - /** - * Retrieve and save the content of a code system csv file within a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param content Source of the csv file - * @return - */ - def saveCodeSystemContent( - terminologyId: String, - codeSystemId: String, - content: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] - - /** - * Retrieve the content of a code system csv file within a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return Source of the csv file - */ - def getCodeSystemContent( - terminologyId: String, - codeSystemId: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] - - /** - * Upload the code system to the repository - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param content Source of the csv file - * @return - */ - def uploadCodeSystem(terminologyId: String, codeSystemId: String, content: Source[ByteString, Any]): Future[Unit] - - /** - * Download the content of a code system csv file within a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return Source of the csv file - */ - def downloadCodeSystem(terminologyId: String, codeSystemId: String): Future[Source[ByteString, Any]] - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/ConceptMapFolderRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/ConceptMapFolderRepository.scala deleted file mode 100644 index 852ad1c1..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/ConceptMapFolderRepository.scala +++ /dev/null @@ -1,313 +0,0 @@ -package io.ignifyr.server.repository.terminology.conceptmap - -import akka.stream.scaladsl.{Concat, FileIO, Framing, Source} -import akka.util.ByteString -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.engine.Execution.actorSystem -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.common.model.{AlreadyExists, BadRequest, InternalError, ResourceNotFound} -import io.ignifyr.server.model.TerminologySystem.TerminologyConceptMap -import io.ignifyr.server.model._ -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.terminology.TerminologySystemFolderRepository.getTerminologySystemsJsonPath -import io.ignifyr.server.util.{CsvUtil, FileOperations} -import org.json4s.jackson.Serialization.writePretty - -import java.io.{File, FileWriter} -import scala.concurrent.Future - -class ConceptMapFolderRepository(terminologySystemFolderPath: String) extends IConceptMapRepository { - - /** - * Retrieve the concept maps within a terminology - * @return Seq[TerminologyConceptMap] - */ - override def getConceptMaps(terminologyId: String): Future[Seq[TerminologyConceptMap]] = { - Future { - findLocalTerminologyById(terminologyId).conceptMaps - } - } - - /** - * Create a new concept map within a terminology - * - * @return TerminologyConceptMap - */ - override def createConceptMap( - terminologyId: String, - conceptMap: TerminologyConceptMap - ): Future[TerminologyConceptMap] = { - Future { - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - val localTerminology = FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - // check if concept map id exists - localTerminology.find(_.id == terminologyId) match { - case Some(t) => - if (t.conceptMaps.exists(_.id == conceptMap.id)) { - throw AlreadyExists( - "Local terminology concept map id already exists.", - s"Id ${conceptMap.id} already exists." - ) - } - // update local terminology file - val newConceptMaps = t.conceptMaps :+ conceptMap - val newLocalTerminology = localTerminology.map(t => - if (t.id == terminologyId) t.copy(conceptMaps = newConceptMaps) - else t - ) - val writer = new FileWriter(localTerminologyFile) - try { - writer.write(writePretty(newLocalTerminology)) - } finally { - writer.close() - } - // create concept map file - val conceptMapFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, conceptMap.id).toFile - conceptMapFile.createNewFile() - conceptMap - - case None => - throw BadRequest("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - } - - /** - * Retrieve a concept map within a terminology - * - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return TerminologyConceptMap if found - */ - override def getConceptMap(terminologyId: String, conceptMapId: String): Future[TerminologyConceptMap] = { - Future { - findConceptMapById(terminologyId, conceptMapId) - } - } - - /** - * Update a concept map within a terminology - * - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param conceptMap TerminologyConceptMap to update - * @return updated TerminologyConceptMap - */ - override def updateConceptMap( - terminologyId: String, - conceptMapId: String, - conceptMap: TerminologyConceptMap - ): Future[TerminologyConceptMap] = { - Future { - // cross check ids - if (conceptMap.id != conceptMapId) { - throw BadRequest( - "Concept map id does not match.", - s"Concept map id ${conceptMap.id} does not match $conceptMapId." - ) - } - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - val localTerminology = FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - // check if concept map id exists in json file - localTerminology.find(_.id == terminologyId) match { - case Some(t) => - if (!t.conceptMaps.exists(_.id == conceptMapId)) { - throw ResourceNotFound( - "Local terminology concept map not found.", - s"Local terminology concept map with id $conceptMapId not found." - ) - } - val newConceptMaps = t.conceptMaps.map(cm => - if (cm.id == conceptMapId) conceptMap - else cm - ) - val newLocalTerminology = localTerminology.map(t => - if (t.id == terminologyId) t.copy(conceptMaps = newConceptMaps) - else t - ) - val writer = new FileWriter(localTerminologyFile) - try { - writer.write(writePretty(newLocalTerminology)) - } finally { - writer.close() - } - conceptMap - case None => - throw BadRequest("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - } - - /** - * Remove a concept map within a terminology - * - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return - */ - override def removeConceptMap(terminologyId: String, conceptMapId: String): Future[Unit] = { - Future { - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - val localTerminology = FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - - localTerminology.find(_.id == terminologyId) match { - case Some(t) => - // check if concept map id exists in json file - if (!t.conceptMaps.exists(_.id == conceptMapId)) { - throw ResourceNotFound( - "Local terminology concept map not found.", - s"Local terminology concept map with id $conceptMapId not found." - ) - } - // remove concept map from json file - val newConceptMaps = t.conceptMaps.filterNot(_.id == conceptMapId) - val newLocalTerminology = localTerminology.map(t => - if (t.id == terminologyId) t.copy(conceptMaps = newConceptMaps) - else t - ) - val writer = new FileWriter(localTerminologyFile) - try { - writer.write(writePretty(newLocalTerminology)) - } finally { - writer.close() - } - // remove concept map file - val conceptMapFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, conceptMapId).toFile - conceptMapFile.delete() - case None => - throw BadRequest("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - } - - /** - * Update the concept map header by its id - * @param terminologyId terminology id the concept map belongs to - * @param conceptMapId concept map id e.g. icd9-to-icd10.csv - * @param headers new headers to update - * @return - */ - def updateConceptMapHeader(terminologyId: String, conceptMapId: String, headers: Seq[CsvHeader]): Future[Unit] = { - val conceptMap = findConceptMapById(terminologyId, conceptMapId) - // get file and update headers - val conceptMapFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, conceptMap.id).toFile - CsvUtil.writeCsvHeaders(conceptMapFile, headers) - } - - /** - * Retrieve and save the content of a concept map csv file within a terminology - * - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param content content of the csv file - * @return - */ - override def saveConceptMapContent( - terminologyId: String, - conceptMapId: String, - content: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] = { - val conceptMap = findConceptMapById(terminologyId, conceptMapId) - // save concept map file - val conceptMapFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, conceptMap.id).toFile - CsvUtil.writeCsvAndReturnRowNumber(conceptMapFile, content, pageNumber, pageSize) - } - - /** - * Retrieve the content of a concept map csv file within a terminology - * - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return content of the csv file - */ - override def getConceptMapContent( - terminologyId: String, - conceptMapId: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] = { - val conceptMap = findConceptMapById(terminologyId, conceptMapId) - // get concept map file - val file = FileUtils.getPath(terminologySystemFolderPath, terminologyId, conceptMap.id).toFile - CsvUtil.getPaginatedCsvContent(file, pageNumber, pageSize) - } - - /** - * Upload the concept map to the repository - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param content content of the csv file - * @return - */ - override def uploadConceptMap( - terminologyId: String, - conceptMapId: String, - content: Source[ByteString, Any] - ): Future[Unit] = { - val conceptMap = findConceptMapById(terminologyId, conceptMapId) - // save concept map file - val conceptMapFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, conceptMap.id).toFile - CsvUtil.saveFileContent(conceptMapFile, content) - } - - /** - * Download the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return content of the csv file - */ - override def downloadConceptMap(terminologyId: String, conceptMapId: String): Future[Source[ByteString, Any]] = { - Future { - val conceptMap = findConceptMapById(terminologyId, conceptMapId) - // get concept map file - val conceptMapFile = FileUtils.getPath(terminologySystemFolderPath, terminologyId, conceptMap.id) - FileIO.fromPath(conceptMapFile) - } - } - - /** - * Find local terminology by id from json file - * JSON file contains a list of [[TerminologySystem]] - * If not found, throw ResourceNotFound - * @param terminologyId id of the terminology - * @return - */ - private def findLocalTerminologyById(terminologyId: String): TerminologySystem = { - val localTerminologies = getLocalTerminologiesFromFile - localTerminologies.find(_.id == terminologyId) match { - case Some(t) => t - case None => throw ResourceNotFound("Local terminology id does not exist.", s"Id $terminologyId does not exist.") - } - } - - /** - * Find concept map by id from a local terminology - * If either terminology or concept map not found, throw ResourceNotFound - * @param terminologyId id of the terminology - * @param conceptMapId concept map id to find inside the terminology - * @return - */ - private def findConceptMapById(terminologyId: String, conceptMapId: String): TerminologyConceptMap = { - val localTerminology = findLocalTerminologyById(terminologyId) - localTerminology.conceptMaps.find(_.id == conceptMapId) match { - case Some(cs) => cs - case None => - throw ResourceNotFound( - "Local terminology concept map not found.", - s"Local terminology concept map with id $conceptMapId not found." - ) - } - } - - /** - * Read terminology systems JSON and return as a list of [[TerminologySystem]] - * @return - */ - private def getLocalTerminologiesFromFile: Seq[TerminologySystem] = { - val localTerminologyFile = FileUtils.getPath(getTerminologySystemsJsonPath(terminologySystemFolderPath)).toFile - FileOperations.readJsonContent[TerminologySystem](localTerminologyFile) - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/IConceptMapRepository.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/IConceptMapRepository.scala deleted file mode 100644 index 986d4c34..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/repository/terminology/conceptmap/IConceptMapRepository.scala +++ /dev/null @@ -1,108 +0,0 @@ -package io.ignifyr.server.repository.terminology.conceptmap - -import akka.stream.scaladsl.Source -import akka.util.ByteString -import io.ignifyr.server.model.TerminologySystem.TerminologyConceptMap -import io.ignifyr.server.model.csv.CsvHeader - -import scala.concurrent.Future - -trait IConceptMapRepository { - - /** - * Retrieve the concept maps within a terminology - * - * @return Seq[TerminologyConceptMap] - */ - def getConceptMaps(terminologyId: String): Future[Seq[TerminologyConceptMap]] - - /** - * Create a new concept map within a terminology - * @return TerminologyConceptMap - */ - def createConceptMap(terminologyId: String, conceptMap: TerminologyConceptMap): Future[TerminologyConceptMap] - - /** - * Retrieve a concept map within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return TerminologyConceptMap if found - */ - def getConceptMap(terminologyId: String, conceptMapId: String): Future[TerminologyConceptMap] - - /** - * Update a concept map within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param conceptMap TerminologyConceptMap to update - * @return updated TerminologyConceptMap - */ - def updateConceptMap( - terminologyId: String, - conceptMapId: String, - conceptMap: TerminologyConceptMap - ): Future[TerminologyConceptMap] - - /** - * Remove a concept map within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return - */ - def removeConceptMap(terminologyId: String, conceptMapId: String): Future[Unit] - - /** - * Update the concept map header by its id - * @param terminologyId terminology id the concept map belongs to - * @param conceptMapId concept map id e.g. icd9-to-icd10.csv - * @param headers new headers to update - * @return - */ - def updateConceptMapHeader(terminologyId: String, conceptMapId: String, headers: Seq[CsvHeader]): Future[Unit] - - /** - * Retrieve and save the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param content content of the csv file - * @return - */ - def saveConceptMapContent( - terminologyId: String, - conceptMapId: String, - content: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] - - /** - * Retrieve the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return content of the csv file - */ - def getConceptMapContent( - terminologyId: String, - conceptMapId: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] - - /** - * Upload the concept map to the repository - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param content content of the csv file - * @return - */ - def uploadConceptMap(terminologyId: String, conceptMapId: String, content: Source[ByteString, Any]): Future[Unit] - - /** - * Download the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return content of the csv file - */ - def downloadConceptMap(terminologyId: String, conceptMapId: String): Future[Source[ByteString, Any]] - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/ExecutionService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/ExecutionService.scala deleted file mode 100644 index 680e9295..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/ExecutionService.scala +++ /dev/null @@ -1,425 +0,0 @@ -package io.ignifyr.server.service - -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.env.EnvironmentVariableResolver -import io.ignifyr.engine.execution.MappingJobLauncher -import io.ignifyr.engine.mapping.context.MappingContextLoader -import io.ignifyr.engine.model._ -import io.ignifyr.engine.spi.ExtensionRegistry -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.engine.{Execution, IgnifyrEngine} -import io.ignifyr.server.common.model.{BadRequest, ResourceNotFound} -import io.ignifyr.server.model.{ExecuteJobTask, TestResourceCreationRequest} -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.repository.schema.ISchemaRepository -import io.ignifyr.server.util.DataFrameUtil -import org.apache.spark.SparkException -import org.apache.spark.sql.{AnalysisException, KeyValueGroupedDataset} -import org.json4s.jackson.JsonMethods -import org.json4s.{JArray, JBool, JObject, JString, JValue} - -import java.io.File -import java.util.UUID -import scala.concurrent.{ExecutionContext, ExecutionException, Future} - -/** - * Service to handle all execution related operations - * E.g. Run a mapping job, run a mapping task, run a test resource creation, get execution logs - * - * @param jobRepository - * @param mappingRepository - * @param schemaRepository - */ -class ExecutionService( - jobRepository: IJobRepository, - mappingRepository: IMappingRepository, - schemaRepository: ISchemaRepository -) extends LazyLogging { - - // TODO do not define engine and client as a global variable inside the class. (Testing becomes impossible) - val ignifyrEngine = new IgnifyrEngine(Some(mappingRepository), Some(schemaRepository)) - - import Execution.actorSystem - - implicit val ec: ExecutionContext = actorSystem.dispatcher - - // The single home of the batch/stream/scheduled dispatch (shared with the CLI). - private val mappingJobLauncher = new MappingJobLauncher(ignifyrEngine) - - /** - * Run the job for the given execute job tasks - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param executeJobTask execute job task instance contains mapping urls and error handling type - * @return - */ - def runJob( - projectId: String, - jobId: String, - executionId: Option[String], - executeJobTask: Option[ExecuteJobTask] - ): Future[Unit] = { - jobRepository.getJob(projectId, jobId) map { - case None => - throw ResourceNotFound( - "Mapping job does not exists.", - s"A mapping job with id $jobId does not exists in the mapping job repository" - ) - case Some(mj) => - val mappingJob = EnvironmentVariableResolver.resolveFhirMappingJob(mj) - - // get the list of mapping task to be executed - val mappingTasks = executeJobTask.flatMap(_.mappingTaskNames) match { - case Some(names) => names.flatMap(name => mappingJob.mappings.find(p => p.name.contentEquals(name))) - case None => mappingJob.mappings - } - - if (mappingTasks.isEmpty) { - throw BadRequest("No mapping task to execute!", "No mapping task to execute!") - } - - // all the mappings in mappingTasks should not running, if any of them is running, give already running response - val JobExecutionMap = ignifyrEngine.runningJobRegistry.getRunningExecutions() - val jobExecution = JobExecutionMap.get(jobId) - if (jobExecution.isDefined) { - val runningMappingTaskNames = jobExecution.get.flatMap(_._2) - val runningMappingTaskNamesSet = runningMappingTaskNames.toSet - val mappingTaskNamesSet = mappingTasks.map(_.name).toSet - val intersection = runningMappingTaskNamesSet.intersect(mappingTaskNamesSet) - if (intersection.nonEmpty) { - // create jvalue json response with already running mappingTask names as list string in values and execution ids in keys - // find execution ids for the intersection mappingTask names - val executionIds = jobExecution.get - .filter(p => p._2.exists(name => intersection.contains(name))) - .map(x => { - (x._1, x._2.filter(name => intersection.contains(name))) - }) - // convert execution ids to json response - val jValueResponse = JArray( - executionIds - .map(x => { - JObject( - "executionId" -> JString(x._1), - "mappingTaskNames" -> JArray(x._2.map(JString(_)).toList) - ) - }) - .toList - ) - // use it in the response message - throw BadRequest( - "Mapping tasks are already running!", - JsonMethods.compact(JsonMethods.render(jValueResponse)) - ) - } - } - - // create execution - val mappingJobExecution = FhirMappingJobExecution( - executionId.getOrElse(UUID.randomUUID().toString), - job = mappingJob, - projectId = projectId, - mappingTasks = mappingTasks - ) - - // check whether the job execution is already scheduled (vacuously false without a scheduling provider; - // the launcher raises the missing-capability error when it actually tries to schedule) - if ( - executionId.nonEmpty && mappingJob.schedulingSettings.nonEmpty && - ExtensionRegistry.scheduler.exists(_.isScheduled(mappingJob.id, executionId.get)) - ) { - throw BadRequest( - "The mapping job execution is already scheduled!", - s"The mapping job execution is already scheduled!" - ) - } - - // dispatch to batch / streaming / scheduled execution - mappingJobLauncher.launch( - mappingJob, - mappingJobExecution, - clearCheckpoints = executeJobTask.exists(_.clearCheckpoints) - ) - () - } - } - - /** - * Tests the given mapping task by running it with mapping job configurations (i.e. source data configurations) and - * returns its results - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param testResourceCreationRequest test resource creation request to be executed - * @return - */ - def testMappingWithJob( - projectId: String, - jobId: String, - testResourceCreationRequest: TestResourceCreationRequest - ): Future[Seq[FhirMappingResultsForInput]] = { - jobRepository.getJob(projectId, jobId) flatMap { - case None => - throw ResourceNotFound( - "Mapping job does not exists.", - s"A mapping job with id $jobId does not exists in the mapping job repository" - ) - case Some(mj) => - val mappingJob = EnvironmentVariableResolver.resolveFhirMappingJob(mj) - - logger.debug( - s"Testing the mapping ${testResourceCreationRequest.fhirMappingTask.mappingRef} inside the job $jobId by selecting ${testResourceCreationRequest.resourceFilter.numberOfRows} ${testResourceCreationRequest.resourceFilter.order} records." - ) - - // If an unmanaged mapping is provided within the mapping task, normalize the context urls - val mappingTaskWithNormalizedUrls: FhirMappingTask = - testResourceCreationRequest.fhirMappingTask.mapping match { - case None => testResourceCreationRequest.fhirMappingTask - case _ => - // get the path of mapping file which will be used to normalize mapping context urls - val pathToMappingFile: File = FileUtils - .getPath( - IgnifyrConfig.engineConfig.mappingRepositoryFolderPath, - projectId, - s"${testResourceCreationRequest.fhirMappingTask.mapping.get.id}${FileExtensions.JSON}" - ) - .toFile - // normalize the mapping context urls - val mappingWithNormalizedContextUrls: FhirMapping = MappingContextLoader - .normalizeContextURLs(Seq((testResourceCreationRequest.fhirMappingTask.mapping.get, pathToMappingFile))) - .head - // Copy the mapping with the normalized urls - testResourceCreationRequest.fhirMappingTask.copy(mapping = Some(mappingWithNormalizedContextUrls)) - } - - // If the mapping task has a batching strategy, substitute the first batch parameter set - // This allows testing of parameterized preprocessSql queries with sample parameters - val mappingTask: FhirMappingTask = mappingTaskWithNormalizedUrls.batchingStrategy match { - case Some(strategy) if strategy.batchParameterSets.nonEmpty => - val firstBatchParams = strategy.batchParameterSets.head - logger.debug(s"Testing with first batch parameters: $firstBatchParams") - mappingTaskWithNormalizedUrls.substituteBatchParameters(firstBatchParams) - case _ => mappingTaskWithNormalizedUrls - } - - val fhirMappingJobManager = mappingJobLauncher.jobManager - // Force batch reading for the test run: streaming settings (e.g. Kafka) are copied with - // asStream=false so the test reads a bounded sample instead of opening a stream. - val jobSourceSettings: Map[String, MappingJobSourceSettings] = - mappingJob.sourceSettings.map { case (key, settings) => key -> settings.withAsStream(false) } - - val (fhirMapping, mappingJobSourceSettings, dataFrame) = - try { - fhirMappingJobManager.readJoinSourceData( - mappingTask, - jobSourceSettings, - jobId = Some(jobId), - isTestExecution = true - ) - } catch { - // Check if the root cause of the exception is an AnalysisException due to a missing file path. - // If the error class is PATH_NOT_FOUND, extract and clean up the message by removing the prefix, - // then throw a user-friendly BadRequest error with the relevant details. - // Otherwise, rethrow the original exception. - case e: Exception => - val PATH_NOT_FOUND = "PATH_NOT_FOUND" - Option(e.getCause) match { - case Some(ae: AnalysisException) if ae.getErrorClass.contentEquals(PATH_NOT_FOUND) => - val cleanMessage = ae.getMessage.stripPrefix(s"[$PATH_NOT_FOUND] ").trim - throw BadRequest("File Not Found", cleanMessage) - case _ => - throw e - } - } - val selectedDataFrame = DataFrameUtil - .applyResourceFilter(dataFrame, testResourceCreationRequest.resourceFilter) - .distinct() // Remove duplicate rows to ensure each FHIR Resource is represented only once per source. - // This prevents confusion for users in the UI, as displaying the same resource multiple times could lead to misunderstandings. - fhirMappingJobManager - .executeTask( - mappingJob.id, - mappingTask.name, - fhirMapping, - selectedDataFrame, - mappingJobSourceSettings, - mappingJob.terminologyServiceSettings, - mappingJob.getIdentityServiceSettings(), - projectId = Some(projectId) - ) - .map { resultingDataFrame => - // Import implicits for the Spark session - import ignifyrEngine.sparkSession.implicits._ - // Group by the 'source' column - val grouped: KeyValueGroupedDataset[String, FhirMappingResult] = resultingDataFrame - .groupByKey((result: FhirMappingResult) => result.source) - // Map each group to FhirMappingResultForInput - grouped.mapGroups(FhirMappingResultConverter.convertToFhirMappingResultsForInput).collect().toSeq - } - .recover { - case ee: ExecutionException => - // Ask connector-provided descriptors to translate the failure into a clearer message - // (e.g. Kafka's "unknown topic" error naming the missing topics) without the server - // importing any connector client types. - ExtensionRegistry.sourceFailureDescriptors - .flatMap(_.describeBatchTaskFailure(ee, testResourceCreationRequest.fhirMappingTask)) - .headOption match { - case Some(described) => - throw BadRequest("Invalid source for the mapping test execution", described.getMessage) - case None => - throw ee - } - case se: SparkException => - throw BadRequest( - "Invalid Source File", - se.getCause.getMessage.replace("\n", " "), - Some(se) - ) - case e: Exception => - throw e - } - } - } - - /** - * Returns the ongoing (i.e. running or scheduled) mapping job executions. - * - * @param projectId project id the job belongs to - * @param jobId job id - * @return a list of JSONs indicating the execution id and its status i.e. runningStatus or scheduled - * @throws ResourceNotFound when mapping job does not exist - */ - def getExecutions(projectId: String, jobId: String): Future[Seq[JValue]] = { - // retrieve the job to validate its existence - jobRepository.getJob(projectId, jobId).flatMap { - case Some(_) => - // Retrieve the running executions for the given job - val runningExecutionsJson: Seq[JValue] = ignifyrEngine.runningJobRegistry - .getRunningExecutions(jobId) - .map(id => - JObject( - List( - "id" -> JString(id), - "runningStatus" -> JBool(true) - ) - ) - ) - .toSeq - // Retrieve the scheduled executions for the given job (empty unless a scheduling provider is installed) - val scheduledExecutionsJson: Seq[JValue] = ExtensionRegistry.scheduler - .map(_.getScheduledExecutions(jobId)) - .getOrElse(Set.empty) - .map(id => - JObject( - List( - "id" -> JString(id), - "scheduled" -> JBool(true) - ) - ) - ) - .toSeq - - Future.successful(runningExecutionsJson ++ scheduledExecutionsJson) - - case None => - throw ResourceNotFound("Mapping job does not exists.", s"A mapping job with id $jobId does not exists") - } - } - - /** - * Stops the specified job execution. This means that all running executions regarding the mappings included in this job will be stopped. - * - * @param jobId Identifier of the job - * @return - */ - def stopJobExecution(jobId: String, executionId: String): Future[Unit] = { - Future { - if (ignifyrEngine.runningJobRegistry.executionExists(jobId, executionId, None)) { - ignifyrEngine.runningJobRegistry.stopJobExecution(jobId, executionId) - logger.debug(s"Job execution stopped. jobId: $jobId, execution: $executionId") - } else { - throw ResourceNotFound( - "Job execution does not exists.", - s"A job execution with jobId: $jobId, executionId: $executionId does not exists." - ) - } - - } - } - - /** - * Deschedules a job execution. - * - * @param jobId The ID of the job. - * @param executionId The ID of the job execution. - * @return A Future[Unit] representing the descheduling process. - * @throws ResourceNotFound if the specified job execution is not scheduled. - */ - def descheduleJobExecution(jobId: String, executionId: String): Future[Unit] = { - Future { - // Scheduled executions are owned by the installable scheduling provider; if none is installed - // there is nothing scheduled to deschedule. - ExtensionRegistry.scheduler.filter(_.isScheduled(jobId, executionId)) match { - case Some(scheduler) => - scheduler.descheduleJobExecution(ignifyrEngine.runningJobRegistry, jobId, executionId) - logger.debug(s"Job execution descheduled. jobId: $jobId, execution: $executionId") - case None => - throw ResourceNotFound( - "Job is not scheduled.", - s"There is no scheduled job execution with jobId: $jobId, executionId: $executionId." - ) - } - } - } - - /** - * Stops all executions associated with a mapping job specified by the jobId. - * - * @param jobId The identifier of the mapping job for which executions should be stopped. - * @return A Future[Unit] representing the completion of the stop operations. - */ - def stopJobExecutions(jobId: String): Future[Unit] = { - Future { - ignifyrEngine.runningJobRegistry.stopJobExecutions(jobId) - } - } - - /** - * Stops the execution of the specific mapping task - * - * @param executionId Execution in which the mapping is run - * @param mappingTaskName Mapping to be stopped - * @return - */ - def stopMappingExecution(jobId: String, executionId: String, mappingTaskName: String): Future[Unit] = { - Future { - if (ignifyrEngine.runningJobRegistry.executionExists(jobId, executionId, Some(mappingTaskName))) { - ignifyrEngine.runningJobRegistry.stopMappingExecution(jobId, executionId, mappingTaskName) - logger.debug( - s"Mapping execution stopped. jobId: $jobId, executionId: $executionId, mappingTaskName: $mappingTaskName" - ) - } else { - throw ResourceNotFound( - "Mapping execution does not exists.", - s"A mapping execution with jobId: $jobId, executionId: $executionId, mappingTaskName: $mappingTaskName does not exists." - ) - } - } - } - - /** - * Checks whether a mapping job with the specified jobId is currently running. - * - * @param jobId The identifier of the mapping job to be checked for running status. - * @return A Future[Boolean] indicating whether the specified mapping job is running (true) or not (false). - */ - def isJobRunning(jobId: String): Future[Boolean] = { - Future { - ignifyrEngine.runningJobRegistry.isJobRunning(jobId) - } - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/JobService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/JobService.scala deleted file mode 100644 index b5e63bf8..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/JobService.scala +++ /dev/null @@ -1,89 +0,0 @@ -package io.ignifyr.server.service - -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.model.FhirMappingJob -import io.ignifyr.server.common.model.BadRequest -import io.ignifyr.server.repository.job.IJobRepository - -import javax.ws.rs.BadRequestException -import scala.concurrent.Future - -class JobService(jobRepository: IJobRepository) extends LazyLogging { - - /** - * Get all mapping metadata from the mapping repository - * - * @param projectId project id the mappings belong to - * @return - */ - def getAllMetadata(projectId: String): Future[Seq[FhirMappingJob]] = { - jobRepository.getAllJobs(projectId) - } - - /** - * Create a new job - * - * @param projectId project id the job will belong to - * @param job job to create - * @return - * @throws BadRequest if the mapping job is not valid - */ - def createJob(projectId: String, job: FhirMappingJob): Future[FhirMappingJob] = { - try { - // validate the mapping job definition - job.validate() - // create the job - jobRepository.saveJob(projectId, job) - } catch { - case e: BadRequestException => throw BadRequest("Invalid mapping job!", e.getMessage) - } - } - - /** - * Get the job by its id - * - * @param projectId project id the job belongs to - * @param jobId job id - * @return - */ - def getJob(projectId: String, jobId: String): Future[Option[FhirMappingJob]] = { - jobRepository.getJob(projectId, jobId) - } - - /** - * Update the job - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param job job to update - * @return - * @throws BadRequest when the mapping job is not valid - */ - def updateJob(projectId: String, jobId: String, job: FhirMappingJob): Future[FhirMappingJob] = { - if (!jobId.equals(job.id)) { - throw BadRequest( - "Job definition is not valid.", - s"Identifier of the job definition: ${job.id} does not match with the provided jobId: $jobId in the path!" - ) - } - try { - // validate the mapping job definition - job.validate() - // update the job - jobRepository.updateJob(projectId, jobId, job) - } catch { - case e: BadRequestException => throw BadRequest("Invalid mapping job!", e.getMessage) - } - } - - /** - * Delete the job - * - * @param projectId project id the job belongs to - * @param jobId job id - * @return - */ - def deleteJob(projectId: String, jobId: String): Future[Unit] = { - jobRepository.deleteJob(projectId, jobId) - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingContextService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingContextService.scala deleted file mode 100644 index 8ed1a815..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingContextService.scala +++ /dev/null @@ -1,111 +0,0 @@ -package io.ignifyr.server.service - -import akka.stream.scaladsl.Source -import akka.util.ByteString -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.mappingContext.IMappingContextRepository - -import scala.concurrent.Future - -/** - * Service for mapping context - * @param mappingContextRepository - */ -class MappingContextService(mappingContextRepository: IMappingContextRepository) extends LazyLogging { - - /** - * Get all mapping context ids from the mapping context repository - * @param projectId return the mapping contexts for the project - * @return Seq[String] - */ - def getAllMappingContext(projectId: String): Future[Seq[String]] = { - mappingContextRepository.getAllMappingContext(projectId) - } - - /** - * Save the mapping context to the repository - * @param projectId project id the mapping context belongs to - * @param mappingContext mapping context id to save - * @return String mapping context id - */ - def createMappingContext(projectId: String, mappingContext: String): Future[String] = { - mappingContextRepository.createMappingContext(projectId, mappingContext) - } - - /** - * Delete the mapping context from the repository - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @return - */ - def deleteMappingContext(projectId: String, id: String): Future[Unit] = { - mappingContextRepository.deleteMappingContext(projectId, id) - } - - /** - * Update the mapping context header by its id - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @param headers mapping context headers - * @return - */ - def updateMappingContextHeader(projectId: String, id: String, headers: Seq[CsvHeader]): Future[Unit] = { - mappingContextRepository.updateMappingContextHeader(projectId, id, headers) - } - - /** - * Save the mapping context content to the repository - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @param byteSource mapping context content to save - * @return - */ - def saveMappingContextContent( - projectId: String, - id: String, - byteSource: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] = { - mappingContextRepository.saveMappingContextContent(projectId, id, byteSource, pageNumber, pageSize) - } - - /** - * Get the mapping context content by its id - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @return - */ - def getMappingContextContent( - projectId: String, - id: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] = { - mappingContextRepository.getMappingContextContent(projectId, id, pageNumber, pageSize) - } - - /** - * Upload the mapping context content to the repository - * - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @param content mapping context content to save - * @return - */ - def uploadMappingContextFile(projectId: String, id: String, byteSource: Source[ByteString, Any]): Future[Unit] = { - mappingContextRepository.uploadMappingContext(projectId, id, byteSource) - } - - /** - * Download the mapping context content by its id - * @param projectId project id the mapping context belongs to - * @param id mapping context id - * @return - */ - def downloadMappingContextFile(projectId: String, id: String): Future[Source[ByteString, Any]] = { - mappingContextRepository.downloadMappingContext(projectId, id) - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingService.scala deleted file mode 100644 index c5d0d292..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/MappingService.scala +++ /dev/null @@ -1,97 +0,0 @@ -package io.ignifyr.server.service - -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.model.FhirMapping -import io.ignifyr.server.common.model.{BadRequest, ResourceNotFound} -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.mapping.IMappingRepository - -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import scala.concurrent.Future - -class MappingService(mappingRepository: IMappingRepository, jobRepository: IJobRepository) extends LazyLogging { - - /** - * Get all mapping metadata from the mapping repository - * - * @param projectId if given, only return the mappings in the given sub-folder - * @return Seq[MappingFile] - */ - def getAllMappings(projectId: String): Future[Seq[FhirMapping]] = { - mappingRepository.getAllMappings(projectId) - } - - /** - * Save the mapping to the repository - * - * @param projectId sub-folder of the mapping - * @param mapping mapping to save - * @return FhirMapping - */ - def createMapping(projectId: String, mapping: FhirMapping): Future[FhirMapping] = { - mappingRepository.saveMapping(projectId, mapping) - } - - /** - * Get the mapping by its id - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @return - */ - def getMapping(projectId: String, mappingId: String): Future[Option[FhirMapping]] = { - mappingRepository.getMapping(projectId, mappingId) - } - - /** - * Update the mapping from the repository - * If the mapping url is changed, update the mapping url in all jobs that reference this mapping - * - * @param projectId Project id the mapping belongs to - * @param mappingId Unique identifier of the FhirMapping - * @param mapping The updated FhirMapping - * @return - */ - def updateMapping(projectId: String, mappingId: String, mapping: FhirMapping): Future[FhirMapping] = { - // Ensure that the provided schemaId matches the SchemaDefinition's schemaId - if (!mappingId.equals(mapping.id)) { - throw BadRequest( - "Mapping definition is not valid.", - s"Identifier of the mapping definition: ${mapping.id} does not match with the provided mappingId: $mappingId in the path!" - ) - } - mappingRepository.updateMapping(projectId, mappingId, mapping) - } - - /** - * Delete the mapping from the repository if it is not referenced by any job - * - * @param projectId project id the mapping belongs to - * @param mappingId mapping id - * @return - */ - def deleteMapping(projectId: String, mappingId: String): Future[Unit] = { - // find mapping url from mapping id - mappingRepository.getMapping(projectId, mappingId).flatMap { mapping => - // find all jobs that reference this mapping - val mappingUrl = mapping - .getOrElse( - throw ResourceNotFound( - "Mapping does not exists.", - s"A mapping with id $mappingId does not exists in the mapping repository." - ) - ) - .url - jobRepository.getJobsReferencingMapping(projectId, mappingUrl).flatMap { jobs => - if (jobs.isEmpty) - mappingRepository.deleteMapping(projectId, mappingId) - else - throw BadRequest( - "Mapping is referenced by jobs.", - s"The mapping with URL $mappingUrl is referenced by jobs: ${jobs.map(_.id).mkString(", ")}." - ) - } - } - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/MetadataService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/MetadataService.scala deleted file mode 100644 index 69e99e33..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/MetadataService.scala +++ /dev/null @@ -1,107 +0,0 @@ -package io.ignifyr.server.service - -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.server.common.config.WebServerConfig -import io.ignifyr.server.common.spi.IgnifyrServerExtension -import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.env.EnvironmentVariableResolver -import io.ignifyr.server.model.{Archiving, MappingExecutionConfiguration, Metadata, RepositoryNames} - -import java.util.Properties -import scala.concurrent.Await -import scala.concurrent.duration.DurationInt -import scala.util.Try - -/** - * Service for retrieving metadata about the Ignifyr server. - * @param ignifyrEngineConfig engine related configurations - * @param webServerConfig web server related configurations - * @param fhirDefinitionsConfig fhir related configurations - * @param serverExtensions installed server extension modules (queried for external component versions) - */ -class MetadataService( - ignifyrEngineConfig: IgnifyrEngineConfig, - webServerConfig: WebServerConfig, - fhirDefinitionsConfig: FhirDefinitionsConfig, - serverExtensions: Seq[IgnifyrServerExtension] -) { - - /** - * Use configurations to create a Metadata object along with the version set in pom.xml. - * @return - */ - def getMetadata: Metadata = { - val properties: Properties = new Properties() - properties.load(getClass.getClassLoader.getResourceAsStream("version.properties")) - val ignifyrRedCapVersion = getIgnifyrRedCapVersion - // fetch the mapping executions' configurations - val configurations: Seq[MappingExecutionConfiguration] = getMappingExecutionConfigurations - Metadata( - name = "Ignifyr", - description = "Ignifyr is a tool for mapping data from various sources to FHIR resources.", - version = properties.getProperty("application.version"), - fhirDefinitionsVersion = fhirDefinitionsConfig.majorFhirVersion, - ignifyrRedcapVersion = ignifyrRedCapVersion, - definitionsRootUrls = fhirDefinitionsConfig.definitionsRootURLs, - schemasFhirVersion = ignifyrEngineConfig.schemaRepositoryFhirVersion, - repositoryNames = RepositoryNames( - mappings = ignifyrEngineConfig.mappingRepositoryFolderPath, - schemas = ignifyrEngineConfig.schemaRepositoryFolderPath, - contexts = ignifyrEngineConfig.mappingContextRepositoryFolderPath, - jobs = ignifyrEngineConfig.jobRepositoryFolderPath, - terminologySystems = ignifyrEngineConfig.terminologySystemFolderPath - ), - archiving = Archiving( - erroneousRecordsFolder = ignifyrEngineConfig.erroneousRecordsFolder, - archiveFolder = ignifyrEngineConfig.archiveFolder, - streamArchivingFrequency = ignifyrEngineConfig.streamArchivingFrequency - ), - environmentVariables = EnvironmentVariableResolver.getEnvironmentVariables, - executionConfigurations = configurations - ) - } - - /** - * Ask the installed REDCap server extension (if any) for the version of the connected - * ignifyr-redcap service. If no response is received, return None. - * @return - */ - private def getIgnifyrRedCapVersion: Option[String] = { - serverExtensions.find(_.id == "redcap").flatMap { extension => - Try( - Await.result( - extension.externalComponentVersion(), - 1.seconds // increasing this leads to increase initial loading time of the Ignifyr frontend - ) - ).toOption.flatten - } - } - - /** - * Retrieves a sequence of predefined configurations used during the execution of mapping jobs. - * - * @return A sequence of `Configuration` objects, each representing a specific setting. - */ - private def getMappingExecutionConfigurations: Seq[MappingExecutionConfiguration] = { - Seq( - MappingExecutionConfiguration( - name = "Mapping Timeout", - description = "Timeout for each mapping execution on an individual input record", - value = ignifyrEngineConfig.mappingTimeout.toString - ), - MappingExecutionConfiguration( - name = "Maximum Chunk Size", - description = - "Max chunk size to execute for batch executions, if number of records exceed this, the source data will be divided into chunks", - value = ignifyrEngineConfig.maxChunkSizeForMappingJobs.getOrElse("Not Set").toString - ), - MappingExecutionConfiguration( - name = "Batch Group Size", - description = - "The number of FHIR resources in the group while executing (create/update) a FHIR batch operation.", - value = ignifyrEngineConfig.fhirWriterBatchGroupSize.toString - ) - ) - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/ProjectService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/ProjectService.scala deleted file mode 100644 index 2a261fb2..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/ProjectService.scala +++ /dev/null @@ -1,111 +0,0 @@ -package io.ignifyr.server.service - -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.BadRequest -import io.ignifyr.server.model.{Project, ProjectEditableFields} -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.repository.mappingContext.IMappingContextRepository -import io.ignifyr.server.repository.project.IProjectRepository -import io.ignifyr.server.repository.schema.ISchemaRepository -import org.json4s.JObject -import org.json4s.JsonAST.JString - -import scala.concurrent.Future - -class ProjectService( - projectRepository: IProjectRepository, - jobRepository: IJobRepository, - mappingRepository: IMappingRepository, - mappingContextRepository: IMappingContextRepository, - schemaRepository: ISchemaRepository -) extends LazyLogging { - - /** - * Retrieve all Projects - * - * @return all projects in the repository - */ - def getAllProjects: Future[Seq[Project]] = { - projectRepository.getAllProjects - } - - /** - * Save project to the repository. - * - * @param project project to be saved - * @return the created project - */ - def createProject(project: Project): Future[Project] = { - projectRepository.createProject(project) - } - - /** - * Retrieve the project identified by its id. - * - * @param id id of the project - * @return the project - */ - def getProject(id: String): Future[Option[Project]] = { - projectRepository.getProject(id) - } - - /** - * Update some fields of a project in the repository. - * - * @param id id of the project - * @param project patch to be applied to the project - * @return - */ - def updateProject(id: String, project: JObject): Future[Project] = { - // validate patch - validateProjectPatch(project) - projectRepository.updateProject(id, project) - } - - /** - * Removes a project and associated resources. - * - * @param projectId The unique identifier of the project to be removed. - */ - def removeProject(projectId: String): Future[Unit] = { - // first delete the project from repository - projectRepository - .deleteProject(projectId) - // if project deletion is failed throw the error - .recover { case e: Throwable => throw e } - // else delete jobs, mappings, mapping contexts and schemas as well - .flatMap { _ => - jobRepository.deleteAllJobs(projectId) map { _ => - mappingRepository.deleteAllMappings(projectId) - mappingContextRepository.deleteAllMappingContexts(projectId) - schemaRepository.deleteAllSchemas(projectId) - } - } - } - - /** - * Validates the given project patch. It ensures that the patch includes updates only for the editable fields of a project. - * - * @param patch the patch - * @throws BadRequest when the given patch is invalid - */ - private def validateProjectPatch(patch: JObject): Unit = { - // Define the allowed fields set from ProjectEditableFields - val allowedFields = Set( - ProjectEditableFields.DESCRIPTION, - ProjectEditableFields.SCHEMA_URL_PREFIX, - ProjectEditableFields.MAPPING_URL_PREFIX - ) - - // Extract the keys from the patch - val patchKeys = patch.obj.map(_._1).toSet - - // Check for invalid fields - val invalidFields = patchKeys.diff(allowedFields) - if (invalidFields.nonEmpty) { - throw BadRequest("Invalid Patch!", "Invalid project patch!") - } - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/ReloadService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/ReloadService.scala deleted file mode 100644 index 171e77ff..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/ReloadService.scala +++ /dev/null @@ -1,34 +0,0 @@ -package io.ignifyr.server.service - -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.repository.{FolderDBInitializer, IRepositoryManager} -import io.ignifyr.server.repository.job.JobFolderRepository -import io.ignifyr.server.repository.mapping.ProjectMappingFolderRepository -import io.ignifyr.server.repository.mappingContext.MappingContextFolderRepository -import io.ignifyr.server.repository.schema.SchemaFolderRepository -import io.ignifyr.server.repository.terminology.TerminologySystemFolderRepository - -import scala.concurrent.Future - -/** - * Service for reloading resources from the file system. - */ -class ReloadService(repositoryManager: IRepositoryManager) { - - /** - * Reload all resources. - * - * @return - */ - def reloadResources(): Future[Unit] = { - Future { - repositoryManager.mappingRepository.invalidate() - repositoryManager.schemaRepository.invalidate() - repositoryManager.mappingJobRepository.invalidate() - repositoryManager.mappingContextRepository.invalidate() - repositoryManager.terminologySystemRepository.invalidate() - repositoryManager.clear() - repositoryManager.init() - } - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/SchemaDefinitionService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/SchemaDefinitionService.scala deleted file mode 100644 index f7b00eb3..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/SchemaDefinitionService.scala +++ /dev/null @@ -1,283 +0,0 @@ -package io.ignifyr.server.service - -import com.typesafe.scalalogging.LazyLogging -import io.onfhir.api.Resource -import io.onfhir.client.util.FhirClientUtil -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.Execution.actorSystem -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.data.read.SourceHandler -import io.ignifyr.engine.mapping.schema.SchemaConverter -import io.ignifyr.engine.model.exception.FhirMappingException -import io.ignifyr.engine.spi.ExtensionRegistry -import io.ignifyr.engine.util.FhirVersionUtil -import io.ignifyr.server.common.model.{BadRequest, ResourceNotFound} -import io.ignifyr.server.model.{ImportSchemaSettings, InferTask} -import io.ignifyr.server.repository.mapping.IMappingRepository -import io.ignifyr.server.repository.schema.ISchemaRepository -import org.apache.hadoop.shaded.org.apache.http.HttpStatus -import org.apache.spark.sql.types.StructType - -import scala.concurrent.Future - -class SchemaDefinitionService(schemaRepository: ISchemaRepository, mappingRepository: IMappingRepository) - extends LazyLogging { - - /** - * Get all schema definition metadata (not populated with field definitions) from the schema repository - * - * @return A map of URL -> Seq[SimpleStructureDefinition] - */ - def getAllSchemas(projectId: String): Future[Seq[SchemaDefinition]] = { - schemaRepository.getAllSchemas(projectId) - } - - /** - * Get a schema definition from the schema repository - * - * @param projectId - * @param id - * @return - */ - def getSchema(projectId: String, id: String): Future[Option[SchemaDefinition]] = { - schemaRepository.getSchema(projectId, id) - } - - /** - * Get a schema definition by its URL from the schema repository - * - * @param url URL of the schema definition - * @return - */ - def getSchemaByUrl(url: String): Future[Option[SchemaDefinition]] = { - schemaRepository.getSchemaByUrl(url) - } - - /** - * Create and save the schema definition to the schema repository. - * - * @param projectId - * @param simpleStructureDefinition - * @return - */ - def createSchema(projectId: String, schemaDefinition: SchemaDefinition): Future[SchemaDefinition] = { - schemaRepository.saveSchema(projectId, schemaDefinition) - } - - /** - * Update the schema definition in the schema repository. - * - * @param projectId - * @param schemaId - * @param schemaDefinition - * @return - */ - def putSchema(projectId: String, schemaId: String, schemaDefinition: SchemaDefinition): Future[SchemaDefinition] = { // TODO: HTTP PUT methods should return the updated object w.r.t REST principles - // Ensure that the provided schemaId matches the SchemaDefinition's schemaId - if (!schemaId.equals(schemaDefinition.id)) { - throw BadRequest( - "Schema definition is not valid.", - s"Identifier of the schema definition: ${schemaDefinition.id} does not match with the provided schemaId: $schemaId in the path!" - ) - } - schemaRepository.updateSchema(projectId, schemaId, schemaDefinition) - } - - /** - * Delete the schema definition from the schema repository. - * - * @param projectId - * @param schemaId - * @throws ResourceNotFound when the schema does not exist - * @throws BadRequest when the schema is in use by some mappings - * @return - */ - def deleteSchema(projectId: String, schemaId: String): Future[Unit] = { - schemaRepository - .getSchema(projectId, schemaId) - .flatMap(schema => { - if (schema.isEmpty) - throw ResourceNotFound( - "Schema does not exists.", - s"A schema definition with id $schemaId does not exists in the schema repository." - ) - mappingRepository - .getMappingsReferencingSchema(projectId, schema.get.url) - .flatMap(mappingIds => { - if (mappingIds.isEmpty) - schemaRepository.deleteSchema(projectId, schemaId) - else - throw BadRequest( - "Schema is referenced by some mappings.", - s"Schema definition with id $schemaId is referenced by the following mappings:${mappingIds.mkString(",")}" - ) - }) - }) - } - - /** - * It takes inferTask object, connects to database, and executes the query. Infer a SchemaDefinition object from the result of the query. - * - * @param inferTask The object that contains database connection information and sql query - * @return SchemaDefinition object containing the field type information - */ - def inferSchema(inferTask: InferTask): Future[Option[SchemaDefinition]] = { - - // Initialize a SchemaConverter for our FHIR version - val schemaConverter = new SchemaConverter(majorFhirVersion = - FhirVersionUtil.getMajorFhirVersion(IgnifyrConfig.engineConfig.schemaRepositoryFhirVersion) - ) - - // Extract the first mapping job source setting - val sourceSettings = - inferTask.mappingJobSourceSettings.head // We are sure that we only have a single sourceSetting since this is an infer task - ._2 // Get the MappingJobSourceSettings object - - // Inner function to call from multiple lines below so that the schema is inferred after read by spark. - def inferSchemaFromSparkRead(): StructType = { - // Execute SQL and get the dataFrame - val dataFrame = - try { - SourceHandler - .readSource( - inferTask.name, - IgnifyrConfig.sparkSession, - inferTask.sourceBinding, - sourceSettings, - schema = None - ) - .limit(1) // It is enough to take the first row to infer the schema. - } catch { - case e: FhirMappingException => - val simplifiedCause = Option(e.getCause).map(_.getMessage).map(_.replace("\n", " ")).getOrElse("") - val detail = - if ( - simplifiedCause.toLowerCase - .contains("parquet") && simplifiedCause.toLowerCase.contains("not a parquet file") - ) - "Content type mismatch: the input file is not a valid Parquet file." - else - simplifiedCause.capitalize - throw BadRequest(e.getMessage, detail) - } - dataFrame.schema // Get the schema inferred by Spark - } - - // Prefer a connector-native inferrer (e.g. SQL reads JDBC metadata without pulling data); - // fall back to the generic one-row Spark read when no inferrer is registered for the settings - // type, or when the registered inferrer defers to Spark (e.g. a SQL binding with preprocessSql). - val effectiveSchema = ExtensionRegistry.schemaInferrers.get(sourceSettings.getClass) match { - case Some(inferrer) => - val inferred = - try { - inferrer.inferSchema(inferTask.sourceBinding, sourceSettings) - } catch { - case e: Throwable => - throw BadRequest(e.getMessage, Option(e.getCause).map(_.toString).getOrElse(e.getMessage), Some(e)) - } - inferred.getOrElse(inferSchemaFromSparkRead()) - case None => - inferSchemaFromSparkRead() - } - - // Default name for undefined information - val defaultName: String = "Unnamed" - // Create unnamed Schema definition by infer the schema from DataFrame - val unnamedSchema = { - // Map SQL DataTypes to Fhir DataTypes - var fieldDefinitions = - effectiveSchema.fields.map(structField => schemaConverter.fieldsToSchema(structField, defaultName)) - // Remove INPUT_VALIDITY_ERROR fieldDefinition that is added by SourceHandler - fieldDefinitions = - fieldDefinitions.filter(fieldDefinition => fieldDefinition.id != SourceHandler.INPUT_VALIDITY_ERROR) - SchemaDefinition( - url = defaultName, - version = SchemaDefinition.VERSION_LATEST, - `type` = defaultName, - name = defaultName, - description = Option.empty, - rootDefinition = Option.empty, - fieldDefinitions = Some(fieldDefinitions) - ) - } - Future.apply(Some(unnamedSchema)) - } - - /** - * Imports a schema from the specified FHIR server using the provided settings. - * - * @param projectId The ID of the project. - * @param settings The settings for importing the schema. - * @return A Future containing the imported SchemaDefinition. - * @throws BadRequest if the resource identifier is invalid or does not exist. - */ - @throws[BadRequest] - def importSchema(projectId: String, settings: ImportSchemaSettings): Future[SchemaDefinition] = { - // get the OnFhir Client using the given settings - val onFhirClient = FhirClientUtil.createOnFhirClient(settings.baseUrl, settings.securitySettings) - // fetch the structure definition - onFhirClient.read("StructureDefinition", settings.resourceId).execute() flatMap { response => - // the requested resource does not exist - if (response.httpStatus.intValue() == HttpStatus.SC_NOT_FOUND) - throw BadRequest( - "Invalid resource identifier!", - s"Structure Definition with id '${settings.resourceId}' does not exist." - ) - // save the schema - schemaRepository - .saveSchemaByStructureDefinition(projectId, Seq(response.responseBody.get)) - .map(definitions => definitions.head) - } - } - - /** - * Creates schema definitions for the given FHIR resources. - * - * @param projectId The ID of the project in which the schema definitions will be created. - * @param resources A sequence of FHIR resources representing the schema definitions to be created. - * @return A Future containing a sequence of `SchemaDefinition` objects for the created schemas. - */ - def createSchemas(projectId: String, resources: Seq[Resource]): Future[Seq[SchemaDefinition]] = { - schemaRepository.saveSchemaByStructureDefinition(projectId, resources) - } - - /** - * Saves schema definitions imported by a server extension module (e.g. from a REDCap data - * dictionary) — the persistence half of the extensions' schema-import routes. - * - * @param projectId project id for which the schemas will be created - * @param definitions the imported schema definitions - * @return the saved schema definitions - */ - def saveImportedSchemas(projectId: String, definitions: Seq[SchemaDefinition]): Future[Seq[SchemaDefinition]] = { - Future.sequence(definitions.map(definition => schemaRepository.saveSchema(projectId, definition))) - } - - /** - * Get structure definition resource of the schema - * - * @param projectId project containing the schema definition - * @param id id of the requested schema - * @return Structure definition of the schema converted into StructureDefinition Resource - */ - def getSchemaAsStructureDefinition(projectId: String, id: String): Future[Option[Resource]] = { - schemaRepository.getSchemaAsStructureDefinition(projectId, id) - } - - /** - * Save the schema by using its StructureDefinition - * - * @param projectId Identifier of the project in which the schema will be created - * @param structureDefinitionResource schema definition in the form of Structure Definition resource - * @return the SchemaDefinition of the created schema - */ - def createSchemaFromStructureDefinition( - projectId: String, - structureDefinitionResource: Resource - ): Future[SchemaDefinition] = { - schemaRepository - .saveSchemaByStructureDefinition(projectId, Seq(structureDefinitionResource)) - .map(definitions => definitions.head) - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/fhir/FileSystemTreeStructureService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/fhir/FileSystemTreeStructureService.scala deleted file mode 100644 index 52418edd..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/fhir/FileSystemTreeStructureService.scala +++ /dev/null @@ -1,74 +0,0 @@ -package io.ignifyr.server.service.fhir - -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.model.FilePathNode -import io.ignifyr.server.service.fhir.FileSystemTreeStructureService.FILE_LIMIT - -import java.nio.file.{Files, Path} -import scala.jdk.CollectionConverters.IteratorHasAsScala - -class FileSystemTreeStructureService { - - /** - * Check if the given path should be excluded or not - * @param path The path to check - * @return - */ - private def shouldBeExcluded(path: Path): Boolean = { - val fileName = path.getFileName.toString - Files.isHidden(path) || fileName.startsWith(".") // Exclude hidden files and files starting with . (dot) - } - - /** - * Get the folder tree structure of the given base path - * @param basePathString The base path to start with - * @param includeFiles if false, only folders will be included in the tree - * @return - */ - def getFolderTreeStructure(basePathString: String, includeFiles: Boolean): FilePathNode = { - var fileCount = 0 // Track total number of files processed - - val basePath = FileUtils.getPath(basePathString) - if (!basePath.toAbsolutePath.normalize().startsWith(FileUtils.getPath("").toAbsolutePath)) { - throw new IllegalArgumentException("The given path is outside the root context path.") - } - - def buildNode(path: Path): FilePathNode = { - if (fileCount >= FILE_LIMIT) { - throw new IllegalStateException(s"File limit exceeded. The maximum number of files to process is $FILE_LIMIT.") - } - - val children = if (Files.isDirectory(path)) { - val dirStream = Files.list(path) - try { - dirStream - .iterator() - .asScala - .filter(p => includeFiles || Files.isDirectory(p)) - .filterNot(shouldBeExcluded) // Filter out hidden files, folders - .map { p => - fileCount += 1 - buildNode(p) - } - .toList - } finally { - dirStream.close() // Ensures the stream is closed after usage - } - } else { - List.empty[FilePathNode] - } - FilePathNode( - path.getFileName.toString, - Files.isDirectory(path), - basePath.relativize(path).toString.replaceAll("\\\\", "/"), - children - ) - } - buildNode(basePath) - } - -} - -object FileSystemTreeStructureService { - val FILE_LIMIT = 10000 // Maximum number of files to process -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/CodeSystemService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/CodeSystemService.scala deleted file mode 100644 index 13b9397e..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/CodeSystemService.scala +++ /dev/null @@ -1,136 +0,0 @@ -package io.ignifyr.server.service.terminology - -import akka.stream.scaladsl.Source -import akka.util.ByteString -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.server.model.TerminologySystem.TerminologyCodeSystem -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.terminology.codesystem.ICodeSystemRepository - -import scala.concurrent.Future - -class CodeSystemService(codeSystemRepository: ICodeSystemRepository) extends LazyLogging { - - /** - * Get all code systems for a terminology - * @param terminologyId id of the terminology - * @return Seq[TerminologyCodeSystem] - */ - def getCodeSystems(terminologyId: String): Future[Seq[TerminologyCodeSystem]] = { - codeSystemRepository.getCodeSystems(terminologyId) - } - - /** - * Create a new CodeSystem for a terminology - * @param terminologyId id of the terminology - * @param codeSystem TerminologyCodeSystem to create - * @return created TerminologyCodeSystem - */ - def createCodeSystem(terminologyId: String, codeSystem: TerminologyCodeSystem): Future[TerminologyCodeSystem] = { - codeSystemRepository.createCodeSystem(terminologyId, codeSystem) - } - - /** - * Get a CodeSystem for a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return TerminologyCodeSystem if found - */ - def getCodeSystem(terminologyId: String, codeSystemId: String): Future[TerminologyCodeSystem] = { - codeSystemRepository.getCodeSystem(terminologyId, codeSystemId) - } - - /** - * Update a CodeSystem for a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param codeSystem TerminologyCodeSystem to update - * @return updated TerminologyCodeSystem - */ - def updateCodeSystem( - terminologyId: String, - codeSystemId: String, - codeSystem: TerminologyCodeSystem - ): Future[TerminologyCodeSystem] = { - codeSystemRepository.updateCodeSystem(terminologyId, codeSystemId, codeSystem) - } - - /** - * Remove a CodeSystem for a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return - */ - def removeCodeSystem(terminologyId: String, codeSystemId: String): Future[Unit] = { - codeSystemRepository.removeCodeSystem(terminologyId, codeSystemId) - } - - /** - * Update the code system header by its id - * @param terminologyId terminology id the code system belongs to - * @param codeSystemId code system id e.g. icd9-to-icd10.csv - * @param headers new headers to update - * @return - */ - def updateCodeSystemHeader(terminologyId: String, codeSystemId: String, headers: Seq[CsvHeader]): Future[Unit] = { - codeSystemRepository.updateCodeSystemHeader(terminologyId, codeSystemId, headers) - } - - /** - * Upload a CodeSystem csv file for a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param byteSource Source of the csv file - * @return - */ - def saveCodeSystemContent( - terminologyId: String, - codeSystemId: String, - byteSource: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] = { - codeSystemRepository.saveCodeSystemContent(terminologyId, codeSystemId, byteSource, pageNumber, pageSize) - } - - /** - * Download a CodeSystem csv file for a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return Source of the csv file - */ - def getCodeSystemContent( - terminologyId: String, - codeSystemId: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] = { - codeSystemRepository.getCodeSystemContent(terminologyId, codeSystemId, pageNumber, pageSize) - } - - /** - * Upload a CodeSystem csv file for a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @param byteSource Source of the csv file - * @return - */ - def uploadCodeSystemFile( - terminologyId: String, - codeSystemId: String, - byteSource: Source[ByteString, Any] - ): Future[Unit] = { - codeSystemRepository.uploadCodeSystem(terminologyId, codeSystemId, byteSource) - } - - /** - * Download a CodeSystem csv file for a terminology - * @param terminologyId id of the terminology - * @param codeSystemId id of the code system - * @return Source of the csv file - */ - def downloadCodeSystemFile(terminologyId: String, codeSystemId: String): Future[Source[ByteString, Any]] = { - codeSystemRepository.downloadCodeSystem(terminologyId, codeSystemId) - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/ConceptMapService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/ConceptMapService.scala deleted file mode 100644 index 0271b81c..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/ConceptMapService.scala +++ /dev/null @@ -1,136 +0,0 @@ -package io.ignifyr.server.service.terminology - -import akka.stream.scaladsl.Source -import akka.util.ByteString -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.server.model.TerminologySystem.TerminologyConceptMap -import io.ignifyr.server.model.csv.CsvHeader -import io.ignifyr.server.repository.terminology.conceptmap.IConceptMapRepository - -import scala.concurrent.Future - -class ConceptMapService(conceptMapRepository: IConceptMapRepository) extends LazyLogging { - - /** - * Get all ConceptMaps for a terminology - * @param terminologyId id of the terminology - * @return Seq[TerminologyConceptMap] - */ - def getConceptMaps(terminologyId: String): Future[Seq[TerminologyConceptMap]] = { - conceptMapRepository.getConceptMaps(terminologyId) - } - - /** - * Create a new ConceptMap for a terminology - * @param terminologyId id of the terminology - * @param conceptMap TerminologyConceptMap to create - * @return created TerminologyConceptMap - */ - def createConceptMap(terminologyId: String, conceptMap: TerminologyConceptMap): Future[TerminologyConceptMap] = { - conceptMapRepository.createConceptMap(terminologyId, conceptMap) - } - - /** - * Get a ConceptMap for a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return TerminologyConceptMap if found - */ - def getConceptMap(terminologyId: String, conceptMapId: String): Future[TerminologyConceptMap] = { - conceptMapRepository.getConceptMap(terminologyId, conceptMapId) - } - - /** - * Update a ConceptMap for a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param conceptMap TerminologyConceptMap to update - * @return updated TerminologyConceptMap - */ - def updateConceptMap( - terminologyId: String, - conceptMapId: String, - conceptMap: TerminologyConceptMap - ): Future[TerminologyConceptMap] = { - conceptMapRepository.updateConceptMap(terminologyId, conceptMapId, conceptMap) - } - - /** - * Remove a ConceptMap for a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return - */ - def removeConceptMap(terminologyId: String, conceptMapId: String): Future[Unit] = { - conceptMapRepository.removeConceptMap(terminologyId, conceptMapId) - } - - /** - * Update the concept map header by its id - * @param terminologyId terminology id the concept map belongs to - * @param conceptMapId concept map id e.g. icd9-to-icd10.csv - * @param headers new headers to update - * @return - */ - def updateConceptMapHeader(terminologyId: String, conceptMapId: String, headers: Seq[CsvHeader]): Future[Unit] = { - conceptMapRepository.updateConceptMapHeader(terminologyId, conceptMapId, headers) - } - - /** - * Retrieve and save the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param byteSource content of the csv file - * @return - */ - def saveConceptMapContent( - terminologyId: String, - conceptMapId: String, - byteSource: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] = { - conceptMapRepository.saveConceptMapContent(terminologyId, conceptMapId, byteSource, pageNumber, pageSize) - } - - /** - * Retrieve the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return content of the csv file - */ - def getConceptMapContent( - terminologyId: String, - conceptMapId: String, - pageNumber: Int, - pageSize: Int - ): Future[(Source[ByteString, Any], Long)] = { - conceptMapRepository.getConceptMapContent(terminologyId, conceptMapId, pageNumber, pageSize) - } - - /** - * Retrieve and save the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @param byteSource content of the csv file - * @return - */ - def uploadConceptMapFile( - terminologyId: String, - conceptMapId: String, - byteSource: Source[ByteString, Any] - ): Future[Unit] = { - conceptMapRepository.uploadConceptMap(terminologyId, conceptMapId, byteSource) - } - - /** - * Retrieve the content of a concept map csv file within a terminology - * @param terminologyId id of the terminology - * @param conceptMapId id of the concept map - * @return content of the csv file - */ - def downloadConceptMapFile(terminologyId: String, conceptMapId: String): Future[Source[ByteString, Any]] = { - conceptMapRepository.downloadConceptMap(terminologyId, conceptMapId) - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/TerminologySystemService.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/TerminologySystemService.scala deleted file mode 100644 index ad180c98..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/service/terminology/TerminologySystemService.scala +++ /dev/null @@ -1,120 +0,0 @@ -package io.ignifyr.server.service.terminology - -import com.typesafe.scalalogging.LazyLogging -import io.ignifyr.engine.model.{FhirMappingJob, LocalFhirTerminologyServiceSettings} -import io.ignifyr.server.model.TerminologySystem -import io.ignifyr.server.repository.job.IJobRepository -import io.ignifyr.server.repository.terminology.ITerminologySystemRepository - -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import scala.concurrent.Future - -class TerminologySystemService( - terminologySystemRepository: ITerminologySystemRepository, - mappingJobRepository: IJobRepository -) extends LazyLogging { - - /** - * Get all TerminologySystem metadata from the TerminologySystem repository - * - * @return List of TerminologySystem metadata - */ - def getAllMetadata: Future[Seq[TerminologySystem]] = { - terminologySystemRepository.getTerminologySystemsMetadata - } - - /** - * Create a new TerminologySystem - * - * @param terminologySystem The TerminologySystem will be created - * @return Created TerminologySystem - */ - def createTerminologySystem(terminologySystem: TerminologySystem): Future[TerminologySystem] = { - terminologySystemRepository.createTerminologySystem(terminologySystem) - } - - /** - * Get the TerminologySystem by id - * - * @param id id of the TerminologySystem - * @return The TerminologySystem with given id - */ - def getTerminologySystem(id: String): Future[Option[TerminologySystem]] = { - terminologySystemRepository.getTerminologySystem(id) - } - - /** - * Update the TerminologySystem - * - * @param id TerminologySystem id - * @param terminologySystem TerminologySystem to update - * @return Updated TerminologySystem - */ - def updateTerminologySystem(id: String, terminologySystem: TerminologySystem): Future[TerminologySystem] = { - // Update TerminologySystem first, then update Jobs Terminology service setting fields - terminologySystemRepository.updateTerminologySystem(id, terminologySystem).flatMap { terminologySystem => - updateJobTerminologyServiceSettings(id, Some(terminologySystem)).map(_ => terminologySystem) - } - } - - /** - * Delete the TerminologySystem - * - * @param id TerminologySystem id - * @return - */ - def deleteTerminologySystem(id: String): Future[Unit] = { - // Delete terminology system first, then update Jobs Terminology fields - terminologySystemRepository.deleteTerminologySystem(id).flatMap { terminologySystem => - // update Jobs Terminology fields if it is successful - updateJobTerminologyServiceSettings(id).map(_ => terminologySystem) - } - } - - /** - * Find the mappings jobs which refer to the given terminologySystemId and update their terminology service settings - * with the provided terminologySystem. If the provided terminologySystem is None, remove Job's terminology service settings. - * - * @param terminologySystemId id of TerminologySystem - * @param terminologySystem TerminologySystem - * @return - */ - private def updateJobTerminologyServiceSettings( - terminologySystemId: String, - terminologySystem: Option[TerminologySystem] = Option.empty - ): Future[Unit] = { - Future - .sequence(mappingJobRepository.getProjectPairs.flatMap { // Get all mapping jobs grouped under their projects. - case (projectId, mappingJobs) => - mappingJobs.filter { job => // Find the mapping jobs which refer to the given terminologySystemId - job.terminologyServiceSettings match { - case Some(settings: LocalFhirTerminologyServiceSettings) => - settings.folderPath.split('/').lastOption.contains(terminologySystemId) - case _ => false - } - } map { jobToBeUpdated => - // Create updated mapping job object - val updatedJob = terminologySystem match { - // Update terminology service case - case Some(ts) => - val terminologyServiceSettings: LocalFhirTerminologyServiceSettings = - jobToBeUpdated.terminologyServiceSettings.get.asInstanceOf[LocalFhirTerminologyServiceSettings] - // Update concept maps - val updatedConceptMaps = - terminologyServiceSettings.conceptMapFiles.flatMap(cm => ts.conceptMaps.find(_.id == cm.id)) - // Update code systems - val updatedCodeSystems = - terminologyServiceSettings.codeSystemFiles.flatMap(cs => ts.codeSystems.find(_.id == cs.id)) - val updatedTerminologyServiceSettings: LocalFhirTerminologyServiceSettings = terminologyServiceSettings - .copy(conceptMapFiles = updatedConceptMaps, codeSystemFiles = updatedCodeSystems) - jobToBeUpdated.copy(terminologyServiceSettings = Some(updatedTerminologyServiceSettings)) - // Delete terminology service case - case None => jobToBeUpdated.copy(terminologyServiceSettings = None) - } - // Update job in the repository - mappingJobRepository.updateJob(projectId, jobToBeUpdated.id, updatedJob) - } - }.toSeq) - .map { _ => () } - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/util/CsvUtil.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/util/CsvUtil.scala deleted file mode 100644 index f7e5ee1e..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/util/CsvUtil.scala +++ /dev/null @@ -1,231 +0,0 @@ -package io.ignifyr.server.util - -import akka.stream.scaladsl.{Concat, FileIO, Flow, Framing, Sink, Source} -import akka.util.ByteString -import com.opencsv.CSVParserBuilder -import io.ignifyr.engine.Execution.actorSystem -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.server.common.model.InternalError -import io.ignifyr.server.model.csv.CsvHeader - -import java.io.File -import java.nio.file.StandardOpenOption -import scala.concurrent.Future - -object CsvUtil { - - /** - * Flow to split a ByteString into lines - * The maximum length of a line is set to 1024 characters - * We cannot set an unlimited frame length because it could potentially lead to memory issues. - * If the data stream contains a very long line (or a line without a delimiter), - * it could cause the application to run out of memory while trying to buffer the entire line. - * allowTruncation indicates that we don't require an explicit line ending even for the last message - */ - private val lineDelimiterFlow: Flow[ByteString, ByteString, _] = - Framing.delimiter(ByteString("\n"), maximumFrameLength = 1024, allowTruncation = true) - - /** - * Flow to remove carriage returns from a ByteString - * \r characters are removed to avoid issues with Windows line endings - */ - private val removeCarriageReturns: Flow[ByteString, ByteString, _] = Flow[ByteString].map { line => - val lineStr = line.utf8String // convert ByteString to String - val modifiedLineStr = lineStr.replaceAll("\r", "") // remove \r characters - ByteString(modifiedLineStr) // convert the modified String back to ByteString - } - - /** - * Get the total number of rows in a file - * - * @param file file to count the rows - * @return Future[Long] total number of rows in the file - */ - private def getTotalRows(file: File): Future[Long] = { - FileIO - .fromPath(file.toPath) - .via(lineDelimiterFlow) // split the file content by lines - .filterNot(_.isEmpty) // filter out empty lines - .drop(1) // skip first line which is the header - .runFold(0L)((count, _) => count + 1) // count the number of lines - } - - /** - * Overwrite headers string to the first line of the CSV file - * Adjust the rows to match the new headers if necessary e.g. - * - if a column name is changed, change headers of the rows with new name - * - if a column removed, remove the column from the rows - * - if a new column name recognized, add the column to the rows with a default value () - * - order of the columns are preserved according to the newHHeaders - * @param file CSV file - * @param newHeaders Headers to write to the file - * @return - */ - def writeCsvHeaders(file: File, newHeaders: Seq[CsvHeader]): Future[Unit] = { - // Used external CSV Parser library to handle double quotes in the CSV file - val parser = new CSVParserBuilder().withSeparator(',').withQuoteChar('"').build() - - // Read the existing CSV file into a list of list where each list represents a row - val existingContentFuture = FileIO - .fromPath(file.toPath) - .via(lineDelimiterFlow) // split the file content by lines - .map(_.utf8String) // convert ByteString to String - .filterNot(_.isEmpty) // filter out empty lines - .runWith(Sink.seq) - .map { lines => - if (lines.nonEmpty) { // check if csv is empty - val oldHeaders = parser.parseLine(lines.head.trim) // parse the first line as headers - lines.tail.map { line => // parse the rest of the lines as rows - // for each row this returns a list of tuples where the first element in tuple is the header - oldHeaders.zip(parser.parseLine(line.trim)).toSeq // e.g. Seq(("header1", "value1"), ("header2", "value2")) - } - } else { - Seq.empty - } - } - - // here an example existingContent looks like: - // header1, header2, header3 - // value1, value2, value3 - // value4, value5, value6 - // - // an example newHeaders looks like: - // Seq( - // CsvHeader(currentName = "header2", previousName = "header2"), - // CsvHeader(currentName = "headerChanged", previousName = "header3"), - // CsvHeader(currentName = "header4", previousName = "header4") // previousName may be any placeholder - // ) - // "header1" is deleted, - // "header3" changed to "headerChanged" - // "header4" is added - // flatMap, not map: the write below returns its own Future. Mapping over it would let the returned - // Future complete while the file was still being written, so a caller (the mapping-context header - // endpoint) could answer OK and then read back the old content. - existingContentFuture.flatMap { existingContent => - // Create a new list of lists where each list is a row and the first element in the tuples is the header - val updatedContent = existingContent.map { row => // iterate each row - newHeaders.map { header => // iterate each new header - // If the current name is different than the previously saved name, update the header of tuples - val key = if (header.currentName != header.previousName) header.currentName else header.previousName - // if the header already exists or header name is changed, use its value (header2 -> value2, headerChanged -> value3) - // otherwise, use a default value (header4 -> ) - key -> row.find(_._1 == header.previousName).map(_._2).getOrElse(s"<${header.currentName}>") - } - } - - // here an example updatedContent looks like: - // header2, headerChanged, header4 - // value2, value3, - // value5, value6, - - // Extract the new names from CsvHeader array - val newHeaderNames = newHeaders.map(_.currentName) - // create a header line by joining the new headers with a comma - // create row content by using the second element of the tuple and joining them with a comma - // finally merge the header and row content with a comma - val csvContent = (newHeaderNames.mkString(",") +: updatedContent.map(_.map(x => s"\"${x._2}\"").mkString(","))) - .map(ByteString(_)) - // Write the updated CSV content back to the file - Source(csvContent) - .intersperse(ByteString("\n")) - .runWith(FileIO.toPath(file.toPath, Set(StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING))) - .map(_ => ()) - } - } - - /** - * Get the content of a CSV file as a Source of ByteString - * @param file CSV file - * @param pageNumber number of the page to read - * @param pageSize size of the page - * @return Future[(Source[ByteString, Any], Long)] where the first element is the Source of ByteString and the second element is the total number of rows in the CSV file - */ - def getPaginatedCsvContent(file: File, pageNumber: Int, pageSize: Int): Future[(Source[ByteString, Any], Long)] = { - val totalRecordsFuture: Future[Long] = this.getTotalRows(file) - - totalRecordsFuture.map { totalRecords => - // calculate the start and end index of the CSV file - val start = (pageNumber - 1) * pageSize - val csvFile = FileIO.fromPath(file.toPath) - - val headerSource = csvFile - .via(removeCarriageReturns) // remove out \r characters to avoid issues with Windows line endings - .via(lineDelimiterFlow) // split the file content by lines - .take(1) // Take only the first line for header - - val content = csvFile - .via(removeCarriageReturns) // remove out \r characters to avoid issues with Windows line endings - .via(lineDelimiterFlow) // split the file content by lines - .drop(start + 1) // start from "start" value and +1 to skip the header - .take(pageSize) // get page size number of lines - - // Combine the header and paginated content and return together with the total number of records - val source = Source.combine(headerSource, content)(Concat(_)).intersperse(ByteString("\n")) - (source, totalRecords) - } - } - - /** - * Given a CSV file and content, write the content to the file starting from the specified page number - * After writing, return the total number of rows in the file - * @param file CSV file to write to - * @param content Source of ByteString to write - * @param pageNumber Page number to start writing from - * @param pageSize Page size - * @return Future[Long] Future of total number of rows in the CSV file - */ - def writeCsvAndReturnRowNumber( - file: File, - content: Source[ByteString, Any], - pageNumber: Int, - pageSize: Int - ): Future[Long] = { - val start = (pageNumber - 1) * pageSize + 1 // +1 to skip the header - - // Convert the CSV content to a list of strings - val contentFuture: Future[Seq[String]] = content - .via(removeCarriageReturns) // remove out \r chars to avoid issues with Windows line endings - .map(_.utf8String) // convert ByteString to String - .filterNot(_.isEmpty) // filter out empty lines - // collect values emitted from the stream into a collection, the collection is available through a Future - // when the stream is run with Sink.seq (materialized), it will consume all the elements of the stream and return a Future[Seq[T]] - .runWith(Sink.seq[String]) - - // Convert the CSV content to a list of strings - val existingContentFuture: Future[Seq[String]] = FileIO - .fromPath(file.toPath) - .via(removeCarriageReturns) // remove out \r chars to avoid issues with Windows line endings - .via(lineDelimiterFlow) // split the file content by lines - .map(_.utf8String) // convert ByteString to String - .filterNot(_.isEmpty) // filter out empty lines - .runWith(Sink.seq[String]) - - Future.sequence(Seq(contentFuture, existingContentFuture)).flatMap { case Seq(newContent, existingContent) => - // Replace the rows in the existing CSV file with the new content starting from 'start' and number of 'pageSize' rows - val updatedContent = existingContent.patch(start, newContent, pageSize) - // Write the updated list back to the CSV file by - // converting each line to ByteString and adding a newline character at the end - // and then write line by line to the file - val byteSource = Source(updatedContent).map(s => ByteString(s + "\n")).filterNot(_.isEmpty) - byteSource - .runWith(FileIO.toPath(file.toPath, Set(StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING))) - // flatMap is applied to the Future returned by runWith to get the total number of rows in the file in the end - .flatMap(_ => getTotalRows(file)) - } - - } - - /** - * Overwrite the content of a CSV file with the given content - * @param file file to be write - * @param content content to write - */ - def saveFileContent(file: File, content: akka.stream.scaladsl.Source[ByteString, Any]): Future[Unit] = { - content - .via(removeCarriageReturns) // remove out \r chars to avoid issues with Windows line endings - .runWith(FileIO.toPath(file.toPath)) // use FileIO as a Sink to write the content to the file - .map(_ => ()) // map the result of Future to Unit - .recover(e => throw InternalError("Error while writing file.", e.getMessage)) - - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/util/DataFrameUtil.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/util/DataFrameUtil.scala deleted file mode 100644 index 22fbf9d2..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/util/DataFrameUtil.scala +++ /dev/null @@ -1,27 +0,0 @@ -package io.ignifyr.server.util - -import io.ignifyr.server.model.{ResourceFilter, RowSelectionOrder} -import org.apache.spark.sql.DataFrame - -object DataFrameUtil { - - /** - * Apply the row selection to the dataframe - * - * @param df dataframe to select rows from - * @param resourceFilter filter to apply - * @return dataframe with selected rows - */ - def applyResourceFilter(df: DataFrame, resourceFilter: ResourceFilter): DataFrame = { - resourceFilter.order match { - case RowSelectionOrder.START => - df.limit(resourceFilter.numberOfRows) - case RowSelectionOrder.RANDOM => - // Using a sampling fraction to reduce the cost of ordering - // sampleFraction cannot be higher than 1.0 - val sampleFraction = 1.0.min(resourceFilter.numberOfRows.toDouble / df.count()) - df.sample(sampleFraction).limit(resourceFilter.numberOfRows) - } - } - -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/util/FileOperations.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/util/FileOperations.scala deleted file mode 100644 index 2740b226..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/util/FileOperations.scala +++ /dev/null @@ -1,220 +0,0 @@ -package io.ignifyr.server.util - -import com.typesafe.scalalogging.Logger -import io.onfhir.api.Resource -import io.onfhir.api.util.IOUtil -import io.onfhir.util.OnFhirZipInputStream -import io.ignifyr.engine.Execution.actorSystem.dispatcher -import io.ignifyr.engine.util.FileUtils.FileExtensions -import org.json4s._ -import org.json4s.jackson.JsonMethods._ -import org.json4s.jackson.Serialization.writePretty -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.common.model.{BadRequest, InternalError} -import org.apache.commons.io.input.BOMInputStream -import org.json4s.JsonAST.JObject -import org.json4s.jackson.JsonMethods - -import java.io.{File, FileWriter, InputStreamReader, Reader} -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path} -import java.util.zip.ZipEntry -import scala.collection.mutable -import scala.concurrent.Future -import scala.io.Source - -object FileOperations { - - private val logger: Logger = Logger(this.getClass) - - /** - * Read the content of a json file and parse it to a sequence of objects - * - * @param f File to be read - * @param c class type that can be parsed from JSON - * @tparam K class that created from JSON - * @return Sequence of objects - */ - def readJsonContent[K](f: File)(implicit ev: scala.reflect.Manifest[K]): Seq[K] = { - val source = Source.fromFile(f, StandardCharsets.UTF_8.name()) - val fileContent = - try source.mkString - finally source.close() - val parsed = parse(fileContent).extract[Seq[K]] - parsed - } - - /** - * Reads the given file into a JSON object - * - * @param f - * @return - */ - def readFileIntoJson(f: File): JValue = { - val source = Source.fromFile(f, StandardCharsets.UTF_8.name()) - val fileContent = - try source.mkString - finally source.close() - parse(fileContent) - } - - /** - * Write seq content to a json file - * - * @param f File to be written - * @param content array of objects to be saved - * @tparam K class that can be parsed to JSON and saved - */ - def writeJsonContent[K](f: File, content: Seq[K]): Unit = { - val writer = new FileWriter(f) - try { - writer.write(writePretty(content)) - } finally { - writer.close() - } - } - - /** - * Read the content of a json file and parse it to a sequence of objects - * - * @param f File to be read - * @param c class type that can be parsed from JSON - * @tparam K class that created from JSON - * @return Sequence of objects - */ - def readJsonContentAsObject[K](f: File)(implicit ev: scala.reflect.Manifest[K]): K = { - val source = Source.fromFile(f, StandardCharsets.UTF_8.name()) - val fileContent = - try source.mkString - finally source.close() - val parsed = parse(fileContent).extract[K] - parsed - } - - /** - * Checks whether the given CSV file is a unit conversion file. A CSV file is a unit conversion file - * if it includes a column called "conversion_function" in its first line. - * - * @param f CSV file - * @return true if the file is a unit conversion file, false otherwise - * */ - def isUnitConversionFile(f: File): Boolean = { - val source = Source.fromFile(f, StandardCharsets.UTF_8.name()) - val firstLine = - try source.getLines().toSeq.head - finally source.close() - firstLine.contains("conversion_function") - } - - /** - * Get folder if exists - * - * @param path path of the folder - * @return File object - */ - def getFileIfExists(path: String): File = { - val folder = new File(path) - if (folder.exists()) { - folder - } else { - throw InternalError("File not found.", s"$path file should exists.") - } - } - - /** - * Check whether the file name matches with the entity id - * - * @param entityId id of the entity e.g. job id, mapping id - * @param file file to whose name will be checked - * @param entityType type of the entity e.g. job, mapping - * @return true if the file name matches with the entity id, false otherwise - */ - def checkFileNameMatchesEntityId(entityId: String, file: File, entityType: String): Boolean = { - if (!entityId.equals(IOUtil.removeFileExtension(file.getName))) { - logger.warn( - s"Discarding $entityType definition with id $entityId as it does not match with the file name ${file.getName}" - ) - false - } else { - true - } - } - - /** - * Processes a ZIP file containing FHIR structure definitions and extracts the resources. - * - * This method reads the ZIP file, parses each JSON resource, and collects them into a sequence of - * `Resource` objects. - * - * @param zipFilePath The path to the ZIP file to be processed. - * @return A Future containing a sequence of `Resource` objects extracted from the ZIP file. - */ - def processZipFile(zipFilePath: Path): Future[Seq[Resource]] = Future { - - /** - * Parses a JSON resource from the provided reader. - * - * This method attempts to parse a JSON resource from the given `Reader`. If the path does not end with ".json", - * or if the JSON parsing fails, appropriate exceptions are thrown. - * - * @param reader The reader to read the JSON data from. - * @param path The file path from which the resource is being read. - * @return The parsed `Resource` object from the JSON data. - * @throws InternalError If there is a problem parsing the JSON data from the path. - * @throws BadRequest If the file path does not end with ".json". - */ - def parseResource(reader: Reader, path: String): Resource = { - if (path.endsWith(".json")) - try { - JsonMethods.parse(reader).asInstanceOf[JObject] - } catch { - case e: Exception => - throw InternalError("JSON parsing problem", s"Cannot parse resource from path $path!", Some(e)) - } - else - throw BadRequest("Invalid JSON!", s"Cannot read resource from path $path, it should be JSON file!") - } - - val zipStream = new OnFhirZipInputStream(Files.newInputStream(zipFilePath)) - val resources: mutable.ListBuffer[Resource] = new mutable.ListBuffer[Resource] - var zipEntry: ZipEntry = zipStream.getNextEntry - - while (zipEntry != null) { - val reader = new InputStreamReader(BOMInputStream.builder.setInputStream(zipStream).get(), "UTF-8") - resources.append(parseResource(reader, zipEntry.getName)) - zipStream.closeEntry() - zipEntry = zipStream.getNextEntry - } - resources.toSeq - } - - /** - * Write string content to a file - * - * @param file file to be written - * @param content content to be written - */ - def writeStringContentToFile(file: File, content: String): Unit = { - val fileWriter = new FileWriter(file) - fileWriter.write(content) - fileWriter.close() - } - - /** - * Gets the File object for the given entityId (jobId, mappingId, schemaId). - * - * @param repositoryPath (Path to the job repository, schema repository or mapping repository) - * @param projectId The id of the Ignifyr project. - * @param entityId The id of the job, mapping or schema. - * @return - */ - def getFileForEntityWithinProject(repositoryPath: String, projectId: String, entityId: String): File = { - val file: File = FileUtils.getPath(repositoryPath, projectId, s"$entityId${FileExtensions.JSON}").toFile - // If the project folder does not exist, create it - if (!file.getParentFile.exists()) { - file.getParentFile.mkdir() - } - file - } -} diff --git a/ignifyr-server/src/main/scala/io/ignifyr/server/util/IgnifyrRejectionHandler.scala b/ignifyr-server/src/main/scala/io/ignifyr/server/util/IgnifyrRejectionHandler.scala deleted file mode 100644 index 9d9ef896..00000000 --- a/ignifyr-server/src/main/scala/io/ignifyr/server/util/IgnifyrRejectionHandler.scala +++ /dev/null @@ -1,106 +0,0 @@ -package io.ignifyr.server.util - -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server._ -import io.ignifyr.server.common.interceptor.ICORSHandler -import io.ignifyr.server.common.model.{ - BadRequest, - MethodForbidden, - RequestTimeout, - ResourceNotFound, - UnsupportedMediaType -} - -object IgnifyrRejectionHandler extends ICORSHandler { - - /** - * Custom rejection handler to send proper error message to front-end on rejections. - */ - private val rejectionHandler: RejectionHandler = - RejectionHandler - .newBuilder() - - /** - * Handles the cases where the type of the request data is wrong. Ex: String instead of a terminology system model. - */ - .handle { case UnsupportedRequestContentTypeRejection(supportedTypes) => - complete( - StatusCodes.UnsupportedMediaType -> UnsupportedMediaType( - "Unsupported payload format", - s"Server refuses to accept the request because the type of the request data is not in a supported format" - ).toString - ) - } - - /** - * Handles the malformed content exception. Some examples are: - * - No id field in an imported file - * - No usable value for id cause is a brief description of the exception - * - Any malformed content for service - */ - .handle { - case MalformedRequestContentRejection(message, cause) => - complete(StatusCodes.BadRequest -> BadRequest("Malformed request content", message).toString) - case ValidationRejection(message, _) => - complete(StatusCodes.BadRequest -> BadRequest("Request is not valid", message).toString) - } - - /** - * Handles all types of method rejections, i.e the REST method is not applicable on the given URL. - */ - .handleAll[MethodRejection] { methodRejections => - // supportedMethods are the applicable methods on the given URL - val supportedMethods = methodRejections.map(_.supported.name) - complete( - StatusCodes.MethodNotAllowed -> MethodForbidden( - "Method not allowed", - s"Server refuses to accept the request because request method is not allowed in this URL. Supported methods: ${supportedMethods.mkString(", ")}." - ).toString - ) - } - - /** - * Handles when the given URL is not mapped to anywhere in API - * This case has a special representation as handleNotFound - */ - .handleNotFound { - extractUri { requestUrl => - complete( - StatusCodes.NotFound -> ResourceNotFound( - "Url not found", - s"${requestUrl} does not exist in this API." - ).toString - ) - } - } - - /** - * Handle the rest of the possible rejection types. Send the name of the rejections. - */ - .handleAll[Rejection] { rejections => - val rejectionMessages = rejections.map(_.toString).mkString(", ") - complete(StatusCodes.BadRequest -> BadRequest("Request rejected.", s"Rejection reason: $rejectionMessages")) - } - .result() - - def getRejectionHandler(): RejectionHandler = { - rejectionHandler - } - - /** - * Handles the timeout exception. This is a custom response to the timeout exception. - */ - def timeoutResponseHandler(): HttpResponse = { - val error = RequestTimeout( - title = "Request Timeout", - detail = "The server could not complete the request in time. Try increasing the timeout config and try again." - ) - addCORSHeaders( - HttpResponse( - status = StatusCodes.RequestTimeout, - entity = HttpEntity(ContentTypes.`application/json`, error.toString) - ) - ) - } -} diff --git a/ignifyr-server/src/test/resources/application.conf b/ignifyr-server/src/test/resources/application.conf deleted file mode 100644 index 8d6ecd87..00000000 --- a/ignifyr-server/src/test/resources/application.conf +++ /dev/null @@ -1,138 +0,0 @@ -# Ignifyr configurations -ignifyr { - - # A path to a file/directory from where any File system readings should use within the mappingjob definition. - # e.g., FileSystemSourceSettings.dataFolderPath or LocalFhirTerminologyServiceSettings.folderPath - context-path = "test-context-conf" - - mappings = { - repository = { # The repository where the mapping definition are kept. - folder-path = "mappings" - } - - # Configuration of the schemas used in the mapping definitions. - schemas = { - repository = { # The repository where the schema definitions are kept. - folder-path = "schemas" - } - } - - contexts = { - # The repository where the context definitions are kept. - repository = { - folder-path = "mapping-contexts" - } - } - - - # Timeout for each mapping execution on an individual input record - timeout = 5 seconds - } - - mapping-jobs = { - repository = { # The repository where the schema definitions are kept. - folder-path = "mapping-jobs" - } - # Absolute path to the JSON file for the MappingJob definition to load at the beginning - # initial-job-file-path = "mapping-jobs/pilot1-mappingjob.json" - - # Number of partitions to repartition the source data before executing the mappings for the mapping jobs - # numOfPartitions = 10 - } - - terminology-systems = { - # The path to the folder where Terminology System files (config files, CodeSystems, ConceptMaps etc.) are kept. - folder-path = "terminology-systems" - } - - archiving = { - # Folder to keep erroneous records - erroneous-records-folder = "erroneous-records-folder" - - # Folder to keep archived files - archive-folder = "archive-folder" - - # Frequency in milliseconds to run the archiving task for file streaming jobs - stream-archiving-frequency = 5000 - } - - # Settings for FHIR repository writer - fhir-server-writer { - # The # of FHIR resources in the group while executing (create/update) a FHIR batch operation. - batch-group-size = 50 - } - - # Database folder of Ignifyr repository - db-path = "ignifyr-db-test" -} - -fhir = { - # major FHIR version, currently R4 and R5 is supported - fhir-version = "R4" - - # List of root URLs while retrieving the definitions (profiles, valuesets, codesystems). - # The definitions below the given root URLs will be retrieved from the configured paths or FHIR endpoints. - # All definitions will be retrieved if no root URLs are provided. - # e.g. ["https://aiccelerate.eu", "https://fair4health.eu"] - definitions-root-urls = [] - - # FHIR URL to retrieve resource definitions (profiles, valuesets and codesystems). - # If this URL is defined, file paths (profiles-path, valuesets-path, codesystems-path) will be ignored even if they are also provided. - # For now, Ignifyr can read definitions from a single FHIR endpoint. - definitions-fhir-endpoint = null - fhir-endpoint-auth = { - # basic | token | fixed-token - # If one of the auth methods is selected, its configurations must be provided as shown below. - method = null - -# # basic configurations are used if the auth method is basic -# basic = { -# username = "user" -# password = "pass" -# } -# -# # token configurations are used if the auth method is token -# token = { -# client-id = "id" -# client-secret = "secret" -# scopes = [] -# token-endpoint = "https://onauth.srdc.com.tr" -# } - -# # fixed token configurations are used if the auth method is fixed-token -# fixed-token = "XXX" - } - - # A path to a context file/directory from where profiles, value sets and code systems reading should start. - context-path = "test-context-conf" - # Path to the zip file or folder that includes the FHIR resource and data type profile definitions (FHIR StructureDefinition) to be served by Ignifyr webserver so that mappings can be performed accordingly. - profiles-path = "profiles" - - # Path to the zip file or folder that includes the FHIR Value Set definitions (FHIR ValueSet) that are referenced by your FHIR profiles. - valuesets-path = "valuesets" - - # Path to the zip file or folder that includes the FHIR Code system definitions (FHIR CodeSystem) that are referenced by your FHIR value sets. - codesystems-path = "codesystems" -} - -# Configuration for the ignifyr-redcap service -ignifyr-redcap { - # Base URL of the ignifyr-redcap service - endpoint = "http://localhost:8095/tofhir-redcap" - - # Paths for the endpoints provided by ignifyr-redcap - paths { - # Specifies the path to the notification endpoint relative to the endpoint URL i.e. http://localhost:8095/tofhir-redcap/notification - notification = "notification" - - # Specifies the path to the projects endpoint relative to the endpoint URL i.e. http://localhost:8095/tofhir-redcap/projects - projects = "projects" - - # Specifies the path to the projects' data endpoint relative to the endpoint URL - projectData = "data" - } - # Parameters for the endpoints provided by ignifyr-redcap - parameters { - reload = "reload" - } -} \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/fhir-resources/condition-resource.json b/ignifyr-server/src/test/resources/fhir-resources/condition-resource.json deleted file mode 100644 index ff975a69..00000000 --- a/ignifyr-server/src/test/resources/fhir-resources/condition-resource.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "code": { - "coding": [ - { - "system": "http://hl7.org/fhir/sid/icd-10", - "code": "I21" - } - ] - }, - "subject": { - "reference": "Patient/90bf438cf407677de3b7ee1f7c5567b7" - }, - "onsetDateTime": "2019-10-04T00:00:00Z", - "encounter": { - "reference": "Encounter/d7c7ca69ad2bf6b357a1c2ab2a331544" - }, - "id": "745d5452af9baed370610eaf90be307d", - "meta": { - "profile": [ - "https://datatools4heart.eu/fhir/StructureDefinition/DT4H-Condition" - ] - }, - "resourceType": "Condition" -} diff --git a/ignifyr-server/src/test/resources/fhir-resources/custom-patient-profile.json b/ignifyr-server/src/test/resources/fhir-resources/custom-patient-profile.json deleted file mode 100644 index 7208746a..00000000 --- a/ignifyr-server/src/test/resources/fhir-resources/custom-patient-profile.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "resourceType": "StructureDefinition", - "id": "CustomPatient", - "url": "http://example.org/fhir/StructureDefinition/CustomPatient", - "version": "1.0.0", - "name": "CustomPatient", - "status": "draft", - "date": "2024-07-25", - "publisher": "Example Organization", - "description": "Custom profile for Patient resource", - "fhirVersion": "4.0.1", - "kind": "resource", - "abstract": false, - "type": "Patient", - "baseDefinition": "http://hl7.org/fhir/StructureDefinition/Patient", - "derivation": "constraint", - "differential": { - "element": [ - { - "id": "Patient.birthDate", - "path": "Patient.birthDate", - "min": 1, - "max": "1", - "type": [ - { - "code": "date" - } - ], - "mustSupport": true - }, - { - "id": "Patient.extension", - "path": "Patient.extension", - "slicing": { - "discriminator": [ - { - "type": "value", - "path": "url" - } - ], - "rules": "open" - } - }, - { - "id": "Patient.extension:nationality", - "path": "Patient.extension", - "sliceName": "nationality", - "min": 0, - "max": "1", - "type": [ - { - "code": "Extension" - } - ] - }, - { - "id": "Patient.extension:nationality.url", - "path": "Patient.extension.url", - "fixedUri": "http://example.org/fhir/StructureDefinition/nationality" - }, - { - "id": "Patient.extension:nationality.valueCodeableConcept", - "path": "Patient.extension.valueCodeableConcept", - "min": 1, - "type": [ - { - "code": "CodeableConcept" - } - ] - } - ] - } -} diff --git a/ignifyr-server/src/test/resources/fhir-resources/custom-profiles.zip b/ignifyr-server/src/test/resources/fhir-resources/custom-profiles.zip deleted file mode 100644 index dcde8352698d29b08afb550190c5c903a3ee2af1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1347 zcmWIWW@Zs#U|`^2a1Zzy@nIXs86GAEh9EWu23eqJa%pi%ey(nEeqKsuNoIbYZb4Cg zT4qkFURH5_-rSIbdAALCY?nLycl@xkckTk`#+b`-+nsmEw#kXi?(4icN#SDTCZ+#& zUekotPfuFbkm#=By6?Nkv%lZmI*v45UVHXoX|lr1&L^!ovxBoAa(ab$tOzX5bIHq>F4`N;&9}IjGN*TQM%bpt z|Ka5-D@*UiM??l~5u0>qZl+o7j`MQwm&Y$Uyro+zmi1coeCsujX2*K9)<03mWz^fA zq4DTt7I$)KXU)PZVQ(5b;@mz;uN3MHiHW`4d*KwLt>L8b9ES~&772YG(u~c0gu9)!Jz~Q1e>&MpLrwL7&Vcz=8-iy~Br^PBb+`l(BvEag$ z$uIhqV|p!DrHW*UeED2_yyp+c*8}dd?;PvHH{~hU=D*Ya8OpOQcltWz$L4$2D(ZBa zEpR&VjmvIxUEX>nw)n@7qM$R=p(7F_hRg`KQFZ3 zCHvKXAy4c)c$tV$+jO1or zQWXRyqZPnpB#%f&`ANm8MP-Q?327-(LOQY2cMd2a?Uvh}^kUoT3(ZNFHwHutX=r(e zJ-P3G((BO`?+FVU5~EdI?e@99{QJwRL$S$NuK35sZ3c5XA961|e^R-oH7L{d)Qg9{ zJt3|i>?_=tRnD_Mu=GuZ?X`okf;Vqg@Mzz>A?3y<{&-W6#XtXBA2)`0JBWwP$um0T z7HO1y^MEExpGu{;64w+})oex0ceBzi@4o%!U)e8J%Qf?_p9*a_`1{5q(epbdME!lc zO!vUk6*{UexuuC0r*xT5VRL2gx^YSKpT;T|Wz*~CJH)&rtAD&(s^*$~FfH-h?s*Mb z!5k5S8Qa&(bo}1>{88V|87kLjUkdEa-I+42*j=MIDO@v~eeKjqTWU`|yHr|w?{)aq z@5{Hoey+t3YE&@q&I<9`G z6Z#jgS@LSm4mVbt$y+9@wg^2ry@T`U_N(kYdpdI7e%f-+_s5=PCN|9a_oA8^tv53Z z{(JT;MV$SU+{*{aJ*z|4eNhrL>#%7ne6sFPOzuJcs5i1}l|^$WCEUBlwcO;ES?;cF z3k!IENIkk~C|1IAGv>3dIV_VGWOy(zQXWt+Hnb!8_&ue|B9*#Y$Jr2HI)^ahX zCyyyqD}-gef^#?HT-n$w(?n-xY~Wk?<@3vsML`J-DGi(ZpiOxal~x3@sWOLD@~ zrUfM;4%}Cdh!;Iwb>Q1c6<0lNt-0$L-dShq=Bek*%v77}<+#JA#k63-q6ZpLY?@rl z4|1@z?tChf!8*TiW6kD0b(On&6zv+g9^IU%WG8+p>!FUx!thJZlW%NxX`4|u`$CS( z)9sc^4A-b6M4RaSuxGIo5C0f^=GY^xxi-e{CTlF2YAN#7K0Gy&;c3EEw;1d2D=+i4 zP6c`@Yo%TA*JF9t^jGaz(nhuPK+cug!Yz|n9=)Ev--5H&<@MaZznJ!+I5@eo7QJls%7Wa&9^w!zx`L83U3Slrp`lu z#S$N^J7|(7DKK9kbp4ej+)_T5wM8sC&R}MGAU0Zg zSKHRY?0jBD!>RG_RUWrI%8iKy8Ki{=QGR&s?V5dEZ(e zt4Ces56x%K{k-9l@%n;Q!ry#_K0a5Mx>4z_v)A$JaT9%^xc59>Z0^oC@|NvdwzPd6 z+obzH7>-Il2{@{4QMF4bXZGa(o2!aLs>*gs9J{^I>d@_yz2DeBXYc#J=-1arS1Zai zlUDrF>kvQp?$nzdFV0!Xy!)OXopNH%cJ`(R=eambre8Lx6X@&x@q5YqB14I<5u*Qd z=kaySTH+^lCValL|3af0mcM-;ZXC3ERkEyDyl87z|AgS*JC19e4NGVHu2vADKc%v! z<7)YAs~HCaC;WND5q{hK>_d<4_ti#dZph4^YTxLSy>6M|9ZT=e{J54 zlkKAOop+yNTOj=O{r2B8IV0W){GV{><73W4cOU=l-1T+V++ef1qlb0HE?6G@et4R8 zpZI;@ - - - - - - - - - - - - %d{yyyy-MM-dd_HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n - - - - - - ${LOG_FOLDER}/ignifyr-server.log - - %d{yyyy-MM-dd_HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - ${LOG_FOLDER}/ignifyr-server.%i.log.zip - 1 - 10 - - - - 10MB - - - - - - - - - ${LOG_FOLDER}/ignifyr-mappings.log - - INFO - - - false - - [ignore] - [ignore] - [ignore] - [ignore] - - - - - - - ${LOG_FOLDER}/ignifyr-mappings.%i.log.zip - 1 - 10 - - - - 10MB - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ignifyr-server/src/test/resources/redcap/instrument.csv b/ignifyr-server/src/test/resources/redcap/instrument.csv deleted file mode 100644 index d3b34b23..00000000 --- a/ignifyr-server/src/test/resources/redcap/instrument.csv +++ /dev/null @@ -1,10 +0,0 @@ -"Variable / Field Name","Form Name","Section Header","Field Type","Field Label","Choices, Calculations, OR Slider Labels","Field Note","Text Validation Type OR Show Slider Number","Text Validation Min","Text Validation Max",Identifier?,"Branching Logic (Show field only if...)","Required Field?","Custom Alignment","Question Number (surveys only)","Matrix Group Name","Matrix Ranking?","Field Annotation" -record_id,test,,text,"Record ID",,,,,,,,,,,,, -name,test,"1. General",text,"1.1 Name",,"Name",,,,,,,,,,, -gender,test,,dropdown,"2.Gender","male, Male | female, Female","Enter your gender",,,,y,,y,,,,, -age,test,,checkbox,"3.Age Category","baby, Baby | young, Young | adult, Adult | old, Old",,,,,,,,,,,, -explanation,test,,descriptive,"The following question is not mandatory:",,,,,,,,,,,,, -children,test,,radio,"How many children do you have ?","zero, 0 | one, 1 | two, 2 | more_than_two More than 2",,,,,,,,,,,, -occupation,additional_information,,text,"Occupation","Enter your primary occupation",,,,,,,,,,,, -marital_status,additional_information,,dropdown,"Marital Status","single, Single | married, Married | divorced, Divorced | widowed, Widowed",,,,,,,,,,,, -hobbies,additional_information,,checkbox,"Hobbies","reading, Reading | sports, Sports | traveling, Traveling | music, Music | other, Other",,,,,,,,,,,, \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/sql/sql-source-drop.sql b/ignifyr-server/src/test/resources/sql/sql-source-drop.sql deleted file mode 100644 index e428bc2d..00000000 --- a/ignifyr-server/src/test/resources/sql/sql-source-drop.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE person; -DROP TABLE death; \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/sql/sql-source-populate.sql b/ignifyr-server/src/test/resources/sql/sql-source-populate.sql deleted file mode 100644 index e1aa6678..00000000 --- a/ignifyr-server/src/test/resources/sql/sql-source-populate.sql +++ /dev/null @@ -1,122 +0,0 @@ -CREATE TABLE person -( - person_id integer NOT NULL PRIMARY KEY, - name character varying(50), - surname character varying(50), - date_of_birth date NOT NULL -); - -CREATE TABLE death -( - person_id integer NOT NULL PRIMARY KEY, - death_date date NOT NULL, - death_datetime timestamp without time zone, - death_type_concept_id integer, - cause_concept_id integer, - cause_source_value character varying(50), - cause_source_concept_id integer -); - -INSERT INTO person (person_id, name, surname, date_of_birth) -VALUES (8216, 'John', 'Doe', '1950-06-15'), - (3569, 'Jane', 'Smith', '1962-09-23'), - (8490, 'Robert', 'Johnson', '1945-04-11'), - (9855, 'Emily', 'Davis', '1978-03-05'), - (4004, 'Michael', 'Williams', '1965-12-22'), - (7897, 'Sarah', 'Brown', '1973-05-14'), - (8653, 'David', 'Jones', '1957-10-30'), - (341, 'Laura', 'Garcia', '1981-02-07'), - (480, 'James', 'Martinez', '1949-08-19'), - (7032, 'Patricia', 'Rodriguez', '1969-11-25'), - (2042, 'Charles', 'Hernandez', '1955-06-03'), - (2139, 'Jessica', 'Lopez', '1977-09-13'), - (5951, 'William', 'Gonzalez', '1963-07-29'), - (196, 'Nancy', 'Wilson', '1985-03-21'), - (4269, 'Daniel', 'Anderson', '1947-05-16'), - (409, 'Elizabeth', 'Thomas', '1960-01-08'), - (2738, 'Christopher', 'Taylor', '1958-09-29'), - (1812, 'Matthew', 'Moore', '1972-12-15'), - (78, 'Olivia', 'Jackson', '1967-04-02'), - (1017, 'Benjamin', 'White', '1956-11-09'), - (2807, 'Sophia', 'Harris', '1983-07-31'), - (1683, 'Alexander', 'Clark', '1943-10-18'), - (7574, 'Ava', 'Lewis', '1964-03-22'), - (2801, 'Ethan', 'Walker', '1952-05-05'), - (7931, 'Isabella', 'Hall', '1948-07-07'), - (2478, 'Mason', 'Allen', '1971-08-30'), - (5867, 'Mia', 'Young', '1962-02-28'), - (4477, 'Harper', 'King', '1975-01-19'), - (4074, 'Elijah', 'Wright', '1959-06-08'), - (3633, 'Abigail', 'Scott', '1946-03-17'), - (8012, 'Henry', 'Green', '1970-10-10'), - (7517, 'Lucas', 'Adams', '1953-12-05'), - (8380, 'Charlotte', 'Baker', '1980-09-03'), - (6352, 'Daniel', 'Nelson', '1961-07-27'), - (9232, 'Scarlett', 'Carter', '1979-05-15'), - (469, 'Ella', 'Mitchell', '1950-06-23'), - (1266, 'Jack', 'Perez', '1968-08-21'), - (9527, 'Zoe', 'Roberts', '1957-01-14'), - (7443, 'Gabriel', 'Phillips', '1982-09-29'), - (1462, 'Nathan', 'Evans', '1949-12-09'), - (5548, 'Lily', 'Campbell', '1976-02-11'), - (9978, 'Samuel', 'Parker', '1954-05-26'), - (3674, 'Chloe', 'Edwards', '1963-10-17'), - (9365, 'Sebastian', 'Collins', '1984-07-30'), - (1499, 'Eleanor', 'Stewart', '1955-04-04'), - (9848, 'Owen', 'Sanchez', '1974-11-18'), - (3288, 'Grace', 'Morris', '1959-03-07'), - (9959, 'Ryan', 'Rogers', '1965-06-14'), - (7604, 'Violet', 'Reed', '1977-01-09'), - (3874, 'Caleb', 'Cook', '1951-08-12'); - -INSERT INTO death (person_id, death_date, death_datetime) -VALUES (8216, '2020-03-01', '2020-06-01 23:00:00'), - (3569, '2022-04-06', '2020-10-26 00:00:00'), - (8490, '2020-02-25', '2020-03-24 00:00:00'), - (9855, '2020-07-20', '2019-07-21 12:41:00'), - (4004, '2020-08-15', '2019-08-12 23:00:00'), - (7897, '2021-06-02', '2021-09-30 08:58:00'), - (8653, '2022-05-08', '2022-05-06 22:59:00'), - (341, '2022-04-11', '2021-04-16 23:00:00'), - (480, '2020-11-27', '2022-07-16 23:00:00'), - (7032, '2022-02-06', '2021-12-12 00:00:00'), - (2042, '2021-04-16', '2022-01-22 23:03:00'), - (2139, '2019-11-06', '2019-08-17 00:37:00'), - (5951, '2021-09-07', '2021-07-30 23:00:00'), - (196, '2020-12-13', '2022-04-27 23:00:00'), - (4269, '2021-06-22', '2020-04-11 21:19:00'), - (409, '2021-01-13', '2022-07-22 23:00:00'), - (2738, '2020-07-05', '2020-02-15 00:00:00'), - (1812, '2020-05-27', '2019-11-07 14:05:00'), - (78, '2022-01-11', '2021-07-22 08:30:00'), - (1017, '2021-03-25', '2020-11-01 11:01:00'), - (2807, '2021-11-10', '2020-03-18 00:00:00'), - (1683, '2020-10-31', '2021-06-05 23:00:00'), - (7574, '2022-08-15', '2021-03-16 00:00:00'), - (2801, '2021-09-26', '2019-10-12 23:00:00'), - (7931, '2019-09-07', '2019-07-22 23:00:00'), - (2478, '2020-08-10', '2020-05-31 23:00:00'), - (5867, '2019-11-27', '2020-12-06 00:00:00'), - (4477, '2019-08-27', '2021-10-18 23:00:00'), - (4074, '2021-08-31', '2019-05-03 23:00:00'), - (3633, '2020-09-12', '2022-03-07 00:00:00'), - (8012, '2019-11-01', '2020-07-11 23:40:00'), - (7517, '2021-09-02', '2019-12-25 00:00:00'), - (8380, '2021-01-16', '2020-01-19 00:00:00'), - (6352, '2020-12-11', '2022-02-17 00:00:00'), - (9232, '2022-03-26', '2020-02-14 00:00:00'), - (469, '2021-07-06', '2020-04-17 23:00:00'), - (1266, '2021-05-14', '2020-04-12 23:00:00'), - (9527, '2022-04-27', '2020-09-30 23:00:00'), - (7443, '2021-02-06', '2022-08-17 23:00:00'), - (1462, '2020-07-04', '2020-05-31 17:05:00'), - (5548, '2020-12-31', '2021-12-03 00:00:00'), - (9978, '2020-03-24', '2021-07-31 00:20:00'), - (3674, '2022-03-28', '2020-04-27 23:00:00'), - (9365, '2021-05-19', '2021-06-29 23:00:00'), - (1499, '2021-01-14', '2021-03-02 00:00:00'), - (9848, '2020-01-06', '2019-10-02 23:00:00'), - (3288, '2022-06-12', '2021-06-12 05:00:00'), - (9959, '2021-03-16', '2022-06-02 23:00:00'), - (7604, '2021-03-11', '2020-04-25 23:00:00'), - (3874, '2019-06-10', '2021-10-12 23:00:00'); diff --git a/ignifyr-server/src/test/resources/streaming-parent-folder/patients/patients.csv b/ignifyr-server/src/test/resources/streaming-parent-folder/patients/patients.csv deleted file mode 100644 index 0c02f6a3..00000000 --- a/ignifyr-server/src/test/resources/streaming-parent-folder/patients/patients.csv +++ /dev/null @@ -1,11 +0,0 @@ -pid,gender,birthDate,deceasedDateTime,homePostalCode -p1,male,2000-05-10,, -p2,male,1985-05-08,2017-03-10,G02547 -p3,male,1997-02,, -p4,male,1999-06-05,,H10564 -p5,male,1965-10-01,2019-04-21,G02547 -p6,female,1991-03,, -p7,female,1972-10-25,,V13135 -p8,female,2010-01-10,,Z54564 -p9,female,1999-05-12,, -p10,female,2003-11,, diff --git a/ignifyr-server/src/test/resources/test-data/other-observations.csv b/ignifyr-server/src/test/resources/test-data/other-observations.csv deleted file mode 100644 index 80e36541..00000000 --- a/ignifyr-server/src/test/resources/test-data/other-observations.csv +++ /dev/null @@ -1,15 +0,0 @@ -pid,time,encounterId,code,value -p1,2007-10-12 10:00:00,e1,9110-8,450 -p2,2007-11-12 10:00:00,e2,1335-9,420 -p3,2007-10-13 10:00:00,e3,1299-7,30.5 -p1,2007-10-13 10:00:00,e1,1035-5,45 -p3,2007-10-13 10:00:00,e3,1298-9,43.2 -p4,2007-10-13 10:00:00,e4,38214-3,10 -p5,2007-10-13 10:00:00,e5,9187-6,4 -p6,2007-10-13 10:00:00,e6,445619006,3 -p7,2007-10-13 10:00:00,e7,445597002,5 -p8,2007-10-13 10:00:00,e8,9269-2,5 -p4,2007-10-13 10:00:00,e4,313002,25 -p1,2007-10-13 10:00:00,e1,35629,10 -p2,2007-10-13 10:00:00,e2,33834,35 -p4,2007-10-13 10:00:00,e4,9269-2,4-3-5-4 diff --git a/ignifyr-server/src/test/resources/test-data/patient-gender-simple.csv b/ignifyr-server/src/test/resources/test-data/patient-gender-simple.csv deleted file mode 100644 index 5e454c4d..00000000 --- a/ignifyr-server/src/test/resources/test-data/patient-gender-simple.csv +++ /dev/null @@ -1,2 +0,0 @@ -pid,gender -test-patient,male \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/test-data/patient-simple.csv b/ignifyr-server/src/test/resources/test-data/patient-simple.csv deleted file mode 100644 index 3ea24eec..00000000 --- a/ignifyr-server/src/test/resources/test-data/patient-simple.csv +++ /dev/null @@ -1,2 +0,0 @@ -pid -test-patient \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/test-data/patients.csv b/ignifyr-server/src/test/resources/test-data/patients.csv deleted file mode 100644 index 0c02f6a3..00000000 --- a/ignifyr-server/src/test/resources/test-data/patients.csv +++ /dev/null @@ -1,11 +0,0 @@ -pid,gender,birthDate,deceasedDateTime,homePostalCode -p1,male,2000-05-10,, -p2,male,1985-05-08,2017-03-10,G02547 -p3,male,1997-02,, -p4,male,1999-06-05,,H10564 -p5,male,1965-10-01,2019-04-21,G02547 -p6,female,1991-03,, -p7,female,1972-10-25,,V13135 -p8,female,2010-01-10,,Z54564 -p9,female,1999-05-12,, -p10,female,2003-11,, diff --git a/ignifyr-server/src/test/resources/test-mapping-context/sample-unit-conversion.csv b/ignifyr-server/src/test/resources/test-mapping-context/sample-unit-conversion.csv deleted file mode 100644 index 8c204768..00000000 --- a/ignifyr-server/src/test/resources/test-mapping-context/sample-unit-conversion.csv +++ /dev/null @@ -1,5 +0,0 @@ -source_code,source_unit,target_unit,conversion_function -10839-9,"ng/ml","ng/mL","$this" -14627-4,"mmol/L","mmol/L","$this" -1742-6,"UI/L","U/L","$this" -1920-8,"UI/L","U/L","$this" diff --git a/ignifyr-server/src/test/resources/test-mapping-job/patient-mapping-job-with-two-sources.json b/ignifyr-server/src/test/resources/test-mapping-job/patient-mapping-job-with-two-sources.json deleted file mode 100644 index d9b62656..00000000 --- a/ignifyr-server/src/test/resources/test-mapping-job/patient-mapping-job-with-two-sources.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "name": "patient-job-test", - "sourceSettings": { - "patientSource": { - "jsonClass": "FileSystemSourceSettings", - "name": "patient-test-data", - "sourceUri": "http://test-data", - "dataFolderPath": "./test-data", - "asStream": false - }, - "genderSource": { - "jsonClass": "FileSystemSourceSettings", - "name": "patient-gender-test-data", - "sourceUri": "http://test-data", - "dataFolderPath": "./test-data", - "asStream": false - } - }, - "sinkSettings": { - "jsonClass": "FhirRepositorySinkSettings", - "fhirRepoUrl": "http://localhost:8081/fhir", - "returnMinimal": true - }, - "mappings": [ - { - "name": "patient-mapping-with-two-sources", - "mappingRef": "http://patient-mapping-with-two-sources", - "sourceBinding": { - "patient": { - "jsonClass": "FileSystemSource", - "path": "patient-simple.csv", - "contentType": "csv", - "options": {}, - "sourceRef": "patientSource" - }, - "patientGender": { - "jsonClass": "FileSystemSource", - "path": "patient-gender-simple.csv", - "contentType": "csv", - "options": {}, - "sourceRef": "genderSource" - } - } - } - ], - "dataProcessingSettings": { - "saveErroneousRecords": true, - "archiveMode": "off" - }, - "useFhirSinkAsIdentityService": false -} \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/test-mappings/other-observation-concept-map.csv b/ignifyr-server/src/test/resources/test-mappings/other-observation-concept-map.csv deleted file mode 100644 index 177aeb0a..00000000 --- a/ignifyr-server/src/test/resources/test-mappings/other-observation-concept-map.csv +++ /dev/null @@ -1,14 +0,0 @@ -source_code,source_system,source_display,unit,profile -9110-8,http://loinc.org,Bleeding (cumulative),mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation -1335-9,http://loinc.org,Blood bags used (cumulative),mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation -1299-7,http://loinc.org,Trom (cumulative) given,mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation -1035-5,http://loinc.org,Octoplas (cumulative) given,mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation -1298-9,http://loinc.org,RBC given,mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation -38214-3,http://loinc.org,VASmax (0-10),{score},https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation -9187-6,http://loinc.org,Urine Output,cm3,https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation -445619006,http://snomed.info/sct,NEWS score,{score},https://aiccelerate.eu/fhir/StructureDefinition/AIC-NEWSScore -445597002,http://snomed.info/sct,PEWS score,{score},https://aiccelerate.eu/fhir/StructureDefinition/AIC-PEWSScore -9269-2,http://loinc.org,Galscow Coma Scale (GCS),{score},https://aiccelerate.eu/fhir/StructureDefinition/AIC-GlascowComaScaleObservation -313002,http://www.nlm.nih.gov/research/umls/rxnorm,NaCl0.9% (cumulative) given,mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-MedicationAdministration -35629,http://www.nlm.nih.gov/research/umls/rxnorm,Ringer (cumulative) given,mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-MedicationAdministration -33834,http://www.nlm.nih.gov/research/umls/rxnorm,Plasmalyte (cumulative) given,mL,https://aiccelerate.eu/fhir/StructureDefinition/AIC-MedicationAdministration diff --git a/ignifyr-server/src/test/resources/test-mappings/other-observation-mapping.json b/ignifyr-server/src/test/resources/test-mappings/other-observation-mapping.json deleted file mode 100644 index dc90e02e..00000000 --- a/ignifyr-server/src/test/resources/test-mappings/other-observation-mapping.json +++ /dev/null @@ -1,213 +0,0 @@ -{ - "id": "other-observation-mapping", - "url": "https://aiccelerate.eu/fhir/mappings/other-observation-mapping", - "name": "other-observation-mapping", - "title": "Mapping of schema Other Observation to AIC-IntraOperativeObservation, AIC-MedicationAdministration, AIC-NEWSScore, AIC-PEWSScore and AIC-GlascowScale FHIR profiles", - "source": [{ - "alias": "source", - "url": "https://aiccelerate.eu/fhir/StructureDefinition/Ext-other-observation" - }], - "context": { - "obsConceptMap": { - "category": "concept-map", - "url": "$CONTEXT_REPO/deadbeef-dead-dead-dead-deaddeafbeef/other-observation-concept-map.csv" - } - }, - "variable": [ - { - "name": "hashedObsId", - "language": "text/fhirpath", - "expression": "mpp:getHashedId('Observation', pid & code & time.toString())" - }, - { - "name": "obsUnit", - "language": "text/fhirpath", - "expression": "mpp:getConcept(%obsConceptMap, code, 'unit')" - } - ], - "mapping": [ - { - "precondition": { - "name": "isSimpleObservation", - "language": "text/fhirpath", - "expression": "code != '9269-2' and code != '313002' and code!= '35629' and code != '33834'" - }, - "expression": { - "name": "result", - "language": "application/fhir-template+json", - "value": { - "resourceType": "Observation", - "id": "{{%hashedObsId}}", - "meta": { - "profile": ["{{mpp:getConcept(%obsConceptMap, code, 'profile')}}"], - "source": "{{%sourceSystem.sourceUri}}" - }, - "status": "final", - "code": { - "coding": [ - { - "system": "{{mpp:getConcept(%obsConceptMap, code, 'source_system')}}", - "code": "{{code}}", - "display": "{{mpp:getConcept(%obsConceptMap, code, 'source_display')}}" - } - ] - }, - "subject": "{{mpp:createFhirReferenceWithHashedId('Patient', pid)}}", - "encounter": "{{? mpp:createFhirReferenceWithHashedId('Encounter', encounterId)}}", - "effectiveDateTime": "{{time.utl:toFhirDateTime('yyyy-MM-dd HH:mm:ss')}}", - "valueQuantity": { - "value": "{{value.toDecimal()}}", - "unit": "{{%obsUnit}}", - "system": "http://unitsofmeasure.org", - "code": "{{%obsUnit}}" - } - } - } - }, - { - "precondition": { - "name": "isGcs", - "language": "text/fhirpath", - "expression": "code = '9269-2'" - }, - "expression": { - "name": "result", - "language": "application/fhir-template+json", - "value": { - "resourceType": "Observation", - "id": "{{%hashedObsId}}", - "meta": { - "profile": ["{{mpp:getConcept(%obsConceptMap, code, 'profile')}}"], - "source": "{{%sourceSystem.sourceUri}}" - }, - "status": "final", - "code": { - "coding": [ - { - "system": "{{mpp:getConcept(%obsConceptMap, code, 'source_system')}}", - "code": "{{code}}", - "display": "{{mpp:getConcept(%obsConceptMap, code, 'source_display')}}" - } - ] - }, - "subject": "{{mpp:createFhirReferenceWithHashedId('Patient', pid)}}", - "encounter": "{{? mpp:createFhirReferenceWithHashedId('Encounter', encounterId)}}", - "effectiveDateTime": "{{time.utl:toFhirDateTime('yyyy-MM-dd HH:mm:ss')}}", - "valueQuantity": { - "value": "{{value.split('-')[0].toDecimal()}}", - "unit": "{{%obsUnit}}", - "system": "http://unitsofmeasure.org", - "code": "{{%obsUnit}}" - }, - "component": { - "{{#comp}}": "{{value.split('-').tail().select($this.toDecimal())}}", - "{{?}}": [ - { - "code": { - "coding": [ - { - "system": "http://loinc.org", - "code": "9267-6", - "display": "GCS eye opening" - } - ] - }, - "valueQuantity": { - "value": "{{%comp[0]}}", - "unit": "{score}", - "system": "http://unitsofmeasure.org", - "code": "{score}" - } - }, - { - "code": { - "coding": [ - { - "system": "http://loinc.org", - "code": "9270-0", - "display": "GCS verbal response" - } - ] - }, - "valueQuantity": { - "value": "{{%comp[1]}}", - "unit": "{score}", - "system": "http://unitsofmeasure.org", - "code": "{score}" - } - }, - { - "code": { - "coding": [ - { - "system": "http://loinc.org", - "code": "9268-4", - "display": "GCS motor response" - } - ] - }, - "valueQuantity": { - "value": "{{%comp[2]}}", - "unit": "{score}", - "system": "http://unitsofmeasure.org", - "code": "{score}" - } - } - ] - } - } - } - }, - { - "precondition": { - "name": "isMedicationAdministration", - "language": "text/fhirpath", - "expression": "code = '313002' or code = '35629' or code = '33834'" - }, - "expression": { - "name": "result", - "language": "application/fhir-template+json", - "value": { - "resourceType": "MedicationAdministration", - "id": "{{mpp:getHashedId('MedicationAdministration', pid & code & time.toString())}}", - "meta": { - "profile": ["https://aiccelerate.eu/fhir/StructureDefinition/AIC-MedicationAdministration"], - "source": "{{%sourceSystem.sourceUri}}" - }, - "status": "completed", - "category": [{ - "coding": [ - { - "system": "http://terminology.hl7.org/CodeSystem/medication-admin-category", - "code": "inpatient", - "display": "Inpatient" - } - ] - }], - "medication": { - "concept": { - "coding": [ - { - "system": "http://www.nlm.nih.gov/research/umls/rxnorm", - "code": "{{code}}", - "display": "{{mpp:getConcept(%obsConceptMap, code, 'source_display')}}" - } - ] - } - }, - "subject": "{{mpp:createFhirReferenceWithHashedId('Patient', pid)}}", - "encounter": "{{? mpp:createFhirReferenceWithHashedId('Encounter', encounterId)}}", - "occurenceDateTime": "{{time.utl:toFhirDateTime('yyyy-MM-dd HH:mm:ss')}}", - "dosage": { - "dose": { - "value": "{{value.toDecimal()}}", - "unit": "mL", - "system": "http://unitsofmeasure.org", - "code": "mL" - } - } - } - } - } - ] -} diff --git a/ignifyr-server/src/test/resources/test-mappings/other-observation-mapping2.json b/ignifyr-server/src/test/resources/test-mappings/other-observation-mapping2.json deleted file mode 100644 index 5dc865cd..00000000 --- a/ignifyr-server/src/test/resources/test-mappings/other-observation-mapping2.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "id": "other-observation-mapping2", - "url": "https://aiccelerate.eu/fhir/mappings/other-observation-mapping2", - "name": "other-observation-mapping2", - "title": "Mapping of schema Other Observation to AIC-IntraOperativeObservation, AIC-MedicationAdministration, AIC-NEWSScore, AIC-PEWSScore and AIC-GlascowScale FHIR profiles", - "source": [{ - "alias": "source", - "url": "https://aiccelerate.eu/fhir/StructureDefinition/Ext-other-observation" - }], - "context": { - "obsConceptMap": { - "category": "concept-map", - "url": "$CONTEXT_REPO/deadbeef-dead-dead-dead-deaddeafbeef/other-observation-concept-map.csv" - } - }, - "variable": [ - { - "name": "hashedObsId", - "language": "text/fhirpath", - "expression": "mpp:getHashedId('Observation', pid & code & time.toString())" - }, - { - "name": "obsUnit", - "language": "text/fhirpath", - "expression": "mpp:getConcept(%obsConceptMap, code, 'unit')" - } - ], - "mapping": [ - { - "expression": { - "name": "result", - "language": "application/fhir-template+json", - "value": { - "resourceType": "Observation", - "id": "{{%hashedObsId}}", - "meta": { - "profile": ["{{mpp:getConcept(%obsConceptMap, code, 'profile')}}"], - "source": "{{%sourceSystem.sourceUri}}" - }, - "status": "test", - "code": { - "coding": [ - { - "system": "{{mpp:getConcept(%obsConceptMap, code, 'source_system')}}", - "code": "{{code}}", - "display": "{{mpp:getConcept(%obsConceptMap, code, 'source_display')}}" - } - ] - }, - "subject": "{{mpp:createFhirReferenceWithHashedId('Patient', pid)}}", - "encounter": "{{? mpp:createFhirReferenceWithHashedId('Encounter', encounterId)}}", - "effectiveDateTime": "{{time.utl:toFhirDateTime('yyyy-MM-dd HH:mm:ss')}}", - "valueQuantity": { - "value": "{{value.toDecimal()}}", - "unit": "{{%obsUnit}}", - "system": "http://unitsofmeasure.org", - "code": "{{%obsUnit}}" - } - } - } - } - ] -} diff --git a/ignifyr-server/src/test/resources/test-mappings/patient-flat-mapping.json b/ignifyr-server/src/test/resources/test-mappings/patient-flat-mapping.json deleted file mode 100644 index acb82185..00000000 --- a/ignifyr-server/src/test/resources/test-mappings/patient-flat-mapping.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "id": "patient-flat-mapping", - "url": "http://patient-flat-mapping", - "name": "patient-flat-mapping", - "title": "Patient Flat Mapping", - "source": [ - { - "alias": "patient", - "url": "http://hl7.org/fhir/StructureDefinition/Patient-R4", - "joinOn": [] - } - ], - "context": {}, - "variable": [], - "mapping": [ - { - "expression": { - "name": "flatPatient", - "language": "application/fhir-template+json", - "value": { - "id": "{{id}}", - "active": "{{active}}", - "officialName": "{{name.where(use='official').first().given[0]}} {{name.where(use='official').first().family}}", - "phone": "{{telecom.where(system='phone').first().value}}", - "gender": "{{gender}}", - "birthDate": "{{birthDate}}", - "homeCountry": "{{address.where(use='home').first().country}}", - "resourceType": null, - "meta": { - "profile": [ - "http://patient-flat-schema" - ] - } - } - } - } - ] -} \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/test-mappings/patient-mapping-with-two-sources.json b/ignifyr-server/src/test/resources/test-mappings/patient-mapping-with-two-sources.json deleted file mode 100644 index 4c66a0fe..00000000 --- a/ignifyr-server/src/test/resources/test-mappings/patient-mapping-with-two-sources.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "id": "patient-mapping-two-sources", - "url": "http://patient-mapping-with-two-sources", - "name": "patient-mapping", - "title": "Patient Mapping", - "source": [ - { - "alias": "patient", - "url": "http://patient-schema", - "joinOn": [ - "pid" - ] - }, - { - "alias": "patientGender", - "url": "http://patient-gender", - "joinOn": [ - "pid" - ] - } - ], - "context": {}, - "variable": [], - "mapping": [ - { - "expression": { - "name": "patient", - "language": "application/fhir-template+json", - "value": { - "gender": "{{%patientGender.gender + 1}}", - "id": "{{pid}}", - "meta": { - "profile": [ - "http://hl7.org/fhir/StructureDefinition/Patient" - ] - }, - "resourceType": "Patient" - } - } - } - ] -} \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/test-mappings/patient-mapping.json b/ignifyr-server/src/test/resources/test-mappings/patient-mapping.json deleted file mode 100644 index 0c419e96..00000000 --- a/ignifyr-server/src/test/resources/test-mappings/patient-mapping.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "id": "patient-mapping", - "url": "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping", - "name": "patient-mapping", - "title": "Mapping of patient schema for pilots to AIC-Patient FHIR profile", - "source": [{ - "alias": "source", - "url": "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-patient" - }], - "mapping": [ - { - "expression": { - "name": "result", - "language": "application/fhir-template+json", - "value": { - "resourceType": "Patient", - "id": "{{mpp:getHashedId('Patient',pid)}}", - "meta": { - "profile": ["https://aiccelerate.eu/fhir/StructureDefinition/AIC-Patient"], - "source": "{{%sourceSystem.sourceUri}}" - }, - "active": true, - "identifier": [ - { - "use": "official", - "system": "{{%sourceSystem.sourceUri}}", - "value": "{{pid}}" - } - ], - "gender": "{{gender}}", - "birthDate": "{{birthDate}}", - "deceasedDateTime": "{{? deceasedDateTime}}", - "address": { - "{{#pc}}": "{{homePostalCode}}", - "{{?}}": [ - { - "use": "home", - "type": "both", - "postalCode": "{{%pc}}" - } - ] - } - } - } - } - ] -} diff --git a/ignifyr-server/src/test/resources/test-mappings/patient-mapping2.json b/ignifyr-server/src/test/resources/test-mappings/patient-mapping2.json deleted file mode 100644 index 2eca0299..00000000 --- a/ignifyr-server/src/test/resources/test-mappings/patient-mapping2.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "id": "patient-mapping2", - "url": "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping2", - "name": "patient-mapping2", - "title": "Mapping of patient schema for pilots to AIC-Patient FHIR profile", - "source": [{ - "alias": "source", - "url": "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-patient" - }], - "mapping": [ - { - "expression": { - "name": "result", - "language": "application/fhir-template+json", - "value": { - "resourceType": "Patient", - "id": "{{mpp:getHashedId('Patient',pid)}}", - "meta": { - "profile": ["https://aiccelerate.eu/fhir/StructureDefinition/AIC-Patient"], - "source": "{{%sourceSystem.sourceUri}}" - }, - "active": false, - "identifier": [ - { - "use": "official", - "system": "{{%sourceSystem.sourceUri}}", - "value": "{{pid}}" - } - ], - "gender": "{{gender}}", - "birthDate": "{{birthDate}}", - "deceasedDateTime": "{{? deceasedDateTime}}", - "address": { - "{{#pc}}": "{{homePostalCode}}", - "{{?}}": [ - { - "use": "home", - "type": "both", - "postalCode": "{{%pc}}" - } - ] - } - } - } - } - ] -} diff --git a/ignifyr-server/src/test/resources/test-schemas/blood-pressure-schema.json b/ignifyr-server/src/test/resources/test-schemas/blood-pressure-schema.json deleted file mode 100644 index 7cb16920..00000000 --- a/ignifyr-server/src/test/resources/test-schemas/blood-pressure-schema.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "id": "blood-pressure-schema", - "resourceType": "StructureDefinition", - "url": "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-hsjd-blood-pressure", - "name": "blood-pressure-schema", - "status": "draft", - "fhirVersion": "4.0.1", - "kind": "logical", - "abstract": false, - "type": "Ext-plt1-hsjd-blood-pressure", - "baseDefinition": "http://hl7.org/fhir/StructureDefinition/Element", - "derivation": "specialization", - "differential": { - "element": [ - { - "id": "Ext-plt1-hsjd-blood-pressure", - "path": "Ext-plt1-hsjd-blood-pressure", - "min": 0, - "max": "*", - "type": [ - { - "code": "Element" - } - ] - }, - { - "id": "Ext-plt1-hsjd-blood-pressure.pid", - "path": "Ext-plt1-hsjd-blood-pressure.pid", - "min": 1, - "max": "1", - "type": [ - { - "code": "id", - "profile": [ - "http://hl7.org/fhir/StructureDefinition/id" - ] - } - ] - }, - { - "id": "Ext-plt1-hsjd-blood-pressure.time", - "path": "Ext-plt1-hsjd-blood-pressure.time", - "min": 1, - "max": "1", - "type": [ - { - "code": "dateTime", - "profile": [ - "http://hl7.org/fhir/StructureDefinition/dateTime" - ] - } - ] - }, - { - "id": "Ext-plt1-hsjd-blood-pressure.unit", - "path": "Ext-plt1-hsjd-blood-pressure.unit", - "min": 0, - "max": "1", - "type": [ - { - "code": "code", - "profile": [ - "http://hl7.org/fhir/StructureDefinition/code" - ] - } - ] - }, - { - "id": "Ext-plt1-hsjd-blood-pressure.episodeId", - "path": "Ext-plt1-hsjd-blood-pressure.episodeId", - "min": 0, - "max": "1", - "type": [ - { - "code": "id", - "profile": [ - "http://hl7.org/fhir/StructureDefinition/id" - ] - } - ] - }, - { - "id": "Ext-plt1-hsjd-blood-pressure.systolic", - "path": "Ext-plt1-hsjd-blood-pressure.systolic", - "min": 0, - "max": "1", - "type": [ - { - "code": "string", - "profile": [ - "http://hl7.org/fhir/StructureDefinition/string" - ] - } - ] - }, - { - "id": "Ext-plt1-hsjd-blood-pressure.diastolic", - "path": "Ext-plt1-hsjd-blood-pressure.diastolic", - "min": 0, - "max": "1", - "type": [ - { - "code": "string", - "profile": [ - "http://hl7.org/fhir/StructureDefinition/string" - ] - } - ] - } - ] - } -} \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/test-schemas/other-observation-schema.json b/ignifyr-server/src/test/resources/test-schemas/other-observation-schema.json deleted file mode 100644 index e751f379..00000000 --- a/ignifyr-server/src/test/resources/test-schemas/other-observation-schema.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "id": "Ext-plt1-other-observation", - "url": "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-other-observation", - "version": "latest", - "type": "Ext-plt1-other-observation", - "name": "Ext-plt1-other-observation", - "rootDefinition": { - "id": "Ext-plt1-other-observation", - "path": "Ext-plt1-other-observation", - "dataTypes": [ - { - "dataType": "Element" - } - ], - "isPrimitive": false, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 0 - }, - "fieldDefinitions": [ - { - "id": "pid", - "path": "Ext-plt1-other-observation.pid", - "dataTypes": [ - { - "dataType": "id", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/id" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 1, - "maxCardinality": 1, - "short": "Patient identifier", - "definition": "Unique patient identifier (a pseudoynm)" - }, - { - "id": "time", - "path": "Ext-plt1-other-observation.time", - "dataTypes": [ - { - "dataType": "dateTime", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/dateTime" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 1, - "maxCardinality": 1, - "short": "Observation time", - "definition": "Time that this measurement is done" - }, - { - "id": "encounterId", - "path": "Ext-plt1-other-observation.encounterId", - "dataTypes": [ - { - "dataType": "id", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/id" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 0, - "maxCardinality": 1, - "short": "Encounter", - "definition": "Reference to the encounter that this observation is performed" - }, - { - "id": "code", - "path": "Ext-plt1-other-observation.code", - "dataTypes": [ - { - "dataType": "code", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/code" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 1, - "maxCardinality": 1, - "short": "Observation code", - "definition": "Code for the type of the measurement" - }, - { - "id": "value", - "path": "Ext-plt1-other-observation.value", - "dataTypes": [ - { - "dataType": "string", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/string" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 1, - "maxCardinality": 1, - "short": "Value of the measurement.", - "definition": "Value of the measurement(s). If there are multiple measurements list of them seperated by '-'" - } - ] -} diff --git a/ignifyr-server/src/test/resources/test-schemas/patient-schema.json b/ignifyr-server/src/test/resources/test-schemas/patient-schema.json deleted file mode 100644 index 48cc3932..00000000 --- a/ignifyr-server/src/test/resources/test-schemas/patient-schema.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "id": "Ext-plt1-patient", - "url": "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-patient", - "version": "latest", - "type": "Ext-plt1-patient", - "name": "Ext-plt1-patient", - "abstract": false, - "rootDefinition": { - "id": "Ext-plt1-patient", - "path": "Ext-plt1-patient", - "dataTypes": [ - { - "dataType": "Element" - } - ], - "isPrimitive": false, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 0 - }, - "fieldDefinitions": [ - { - "id": "pid", - "path": "Ext-plt1-patient.pid", - "dataTypes": [ - { - "dataType": "id", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/id" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 1, - "maxCardinality": 1, - "short": "Patient identifier", - "definition": "Unique patient identifier (a pseudoynm)" - }, - { - "id": "gender", - "path": "Ext-plt1-patient.gender", - "dataTypes": [ - { - "dataType": "code", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/code" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 1, - "maxCardinality": 1, - "short": "Gender of patient", - "definition": "Gender of patient" - }, - { - "id": "birthDate", - "path": "Ext-plt1-patient.birthDate", - "dataTypes": [ - { - "dataType": "date", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/date" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 1, - "maxCardinality": 1, - "short": "Birth date", - "definition": "The birth year of patient" - }, - { - "id": "deceasedDateTime", - "path": "Ext-plt1-patient.deceasedDateTime", - "dataTypes": [ - { - "dataType": "date", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/date" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 0, - "maxCardinality": 1, - "short": "Deceseased date", - "definition": "If patient is deceased, its date" - }, - { - "id": "homePostalCode", - "path": "Ext-plt1-patient.homePostalCode", - "dataTypes": [ - { - "dataType": "string", - "profiles": [ - "http://hl7.org/fhir/StructureDefinition/string" - ] - } - ], - "isPrimitive": true, - "isChoiceRoot": false, - "isArray": false, - "minCardinality": 0, - "maxCardinality": 1, - "short": "Home postal code", - "definition": "Postal code for patient home address" - } - ] -} diff --git a/ignifyr-server/src/test/resources/test-terminology-service/sample-code-system.csv b/ignifyr-server/src/test/resources/test-terminology-service/sample-code-system.csv deleted file mode 100644 index da6855e4..00000000 --- a/ignifyr-server/src/test/resources/test-terminology-service/sample-code-system.csv +++ /dev/null @@ -1,3 +0,0 @@ -code,display,fr,de -309068002,Specimen from skin,Spécimen de peau,Probe von der Haut -119323008,Pus specimen,Spécimen de pus,Eiterprobe \ No newline at end of file diff --git a/ignifyr-server/src/test/resources/test-terminology-service/sample-concept-map.csv b/ignifyr-server/src/test/resources/test-terminology-service/sample-concept-map.csv deleted file mode 100644 index 842718a4..00000000 --- a/ignifyr-server/src/test/resources/test-terminology-service/sample-concept-map.csv +++ /dev/null @@ -1,5 +0,0 @@ -source_system,source_code,target_system,target_code,target_display,equivalence -http://terminology.hl7.org/CodeSystem/v2-0487,ACNE,http://snomed.info/sct,309068002,Specimen from skin,equivalent -http://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323008,Pus specimen,equivalent -http://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323009,Pus specimen 2,equivalent -http://terminology.hl7.org/CodeSystem/v2-0487,BULLA,http://snomed.info/sct,258482009,Pus specimen 2,narrower \ No newline at end of file diff --git a/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/FhirDefinitionsEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/FhirDefinitionsEndpointTest.scala deleted file mode 100644 index b3870833..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/FhirDefinitionsEndpointTest.scala +++ /dev/null @@ -1,216 +0,0 @@ -package io.ignifyr.integrationtest - -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes} -import io.onfhir.definitions.resource.endpoint.FhirDefinitionsEndpoint -import io.onfhir.definitions.resource.endpoint.FhirDefinitionsEndpoint.{ - DefinitionsQuery, - QUERY_PARAM_PROFILE, - QUERY_PARAM_Q -} -import io.ignifyr.OnFhirTestContainer -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.onfhir.definitions.common.model.{SchemaDefinition, SimpleStructureDefinition} -import io.ignifyr.engine.util.MajorFhirVersion -import io.ignifyr.server.BaseEndpointTest -import org.json4s.JsonAST.{JString, JValue} -import org.json4s._ -import org.json4s.jackson.JsonMethods - -import scala.io.Source - -/** - * Test class for the FhirDefinitionsEndpoint. - */ -class FhirDefinitionsEndpointTest extends BaseEndpointTest with OnFhirTestContainer { - - // Bundle with three patients: 1 is valid, 1 has an invalid birthdate, and 1 has an invalid active field - val patientBundleJson: String = - Source.fromInputStream(getClass.getResourceAsStream("/fhir-resources/patient-bundle.json")).mkString - // Valid condition resource - val conditionResourceJson: String = - Source.fromInputStream(getClass.getResourceAsStream("/fhir-resources/condition-resource.json")).mkString - // Condition resource with missing subject - val invalidConditionResourceJson: String = - Source.fromInputStream(getClass.getResourceAsStream("/fhir-resources/invalid-condition-resource.json")).mkString - - /** - * Creates a project to be used in the tests. - * */ - override def beforeAll(): Unit = { - super.beforeAll() - this.createProject() - } - - "Fhir definitions endpoint" should { - - /** - * Test case for validating FHIR resources. - */ - "validate FHIR resources" in { - - // Validate the resource without providing a FHIR validation URL - Post( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_VALIDATE}", - HttpEntity(ContentTypes.`application/json`, conditionResourceJson) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - } - - // Validate the resource with an invalid FHIR validation URL - Post( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_VALIDATE}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIRVALIDATIONURL}=test-url", - HttpEntity(ContentTypes.`application/json`, conditionResourceJson) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - } - - // Validate the resource with a valid FHIR validation URL - Post( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_VALIDATE}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIRVALIDATIONURL}=${this.onFhirClient.getBaseUrl()}/Condition/$$validate", - HttpEntity(ContentTypes.`application/json`, conditionResourceJson) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - } - - // Validate the resource with a valid FHIR validation URL - Post( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_VALIDATE}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIRVALIDATIONURL}=${this.onFhirClient.getBaseUrl()}/Condition/$$validate", - HttpEntity(ContentTypes.`application/json`, invalidConditionResourceJson) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - - // Convert the JSON response to a JValue - val jsonResponse: JValue = JsonMethods.parse(responseAs[String]) - - // Check the outcome issues - val errorIssue = (jsonResponse \ "issue").asInstanceOf[JArray].arr.lift(1).get - (errorIssue \ "severity") should be(JString("error")) - (errorIssue \ "code") should be(JString("invalid")) - (errorIssue \ "diagnostics") should be( - JString( - "[Validating against 'http://hl7.org/fhir/StructureDefinition/Condition'] => Element 'subject' with data type(s) 'Reference' is required , but does not exist!" - ) - ) - (errorIssue \ "expression").asInstanceOf[JArray].arr.head should be(JString("subject")) - } - - // Validate a bundle of resources - Post( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_VALIDATE}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIRVALIDATIONURL}=${this.onFhirClient.getBaseUrl()}", - HttpEntity(ContentTypes.`application/json`, patientBundleJson) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - - // Convert the JSON response to a JValue - val jsonResponse: JValue = JsonMethods.parse(responseAs[String]) - - // Check the outcome issues for each entry - val entries = (jsonResponse \ "entry").asInstanceOf[JArray] - entries.arr.size should be(3) - - val firstEntryFirstIssue = (entries.arr.head \ "resource" \ "issue").asInstanceOf[JArray].arr.head - (firstEntryFirstIssue \ "severity") should be(JString("information")) - (firstEntryFirstIssue \ "code") should be(JString("informational")) - (firstEntryFirstIssue \ "diagnostics") should be(JString("All OK :)")) - (firstEntryFirstIssue \ "expression").asInstanceOf[JArray].arr.isEmpty should be(true) - - val secondEntryFirstIssue = (entries.arr.lift(1).get \ "resource" \ "issue").asInstanceOf[JArray].arr.head - (secondEntryFirstIssue \ "severity") should be(JString("error")) - (secondEntryFirstIssue \ "code") should be(JString("invalid")) - (secondEntryFirstIssue \ "diagnostics").asInstanceOf[JString].s.split(" => ").last should be( - "Invalid value '1' for FHIR primitive type 'boolean'!" - ) - (secondEntryFirstIssue \ "expression").asInstanceOf[JArray].arr.head should be(JString("active")) - - val thirdEntryFirstIssue = (entries.arr.last \ "resource" \ "issue").asInstanceOf[JArray].arr.head - (thirdEntryFirstIssue \ "severity") should be(JString("error")) - (thirdEntryFirstIssue \ "code") should be(JString("invalid")) - (thirdEntryFirstIssue \ "diagnostics").asInstanceOf[JString].s.split(" => ").last should be( - "Invalid value 'test' for FHIR primitive type 'date'!" - ) - (thirdEntryFirstIssue \ "expression").asInstanceOf[JArray].arr.head should be(JString("birthDate")) - } - } - - /** - * Test suite for retrieving base FHIR resources via the FHIR Definitions endpoint. - */ - "retrieve base FHIR resources" in { - - /** - * Test case to check if a BadRequest status is returned when the query parameter 'fhirVersion' is missing. - */ - Get(s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_BASE_PROFILES}") ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - responseAs[String] should include( - "Detail: base-profiles path cannot be invoked without the query parameter 'fhirVersion'." - ) - } - - /** - * Test case to check if the correct base FHIR resource types are returned for FHIR version R4. - */ - Get( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_BASE_PROFILES}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIR_VERSION}=${MajorFhirVersion.R4}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - JsonMethods.parse(responseAs[String]).extract[List[String]].length shouldEqual 147 - } - - /** - * Test case to check if a BadRequest status is returned when an invalid FHIR version is provided. - */ - Get( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_BASE_PROFILES}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIR_VERSION}=R6" - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - responseAs[String] should include( - "Detail: fhirVersion on base-profiles cannot take the value: R6. Possible values are: R4 and R5" - ) - } - - /** - * Test case to check if the correct schema definition is returned for a single profile URL with FHIR version R4. - */ - Get( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_BASE_PROFILES}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIR_VERSION}=${MajorFhirVersion.R4}&${FhirDefinitionsEndpoint.QUERY_PARAM_PROFILE}=http://hl7.org/fhir/StructureDefinition/Patient" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - val responseBody = JsonMethods.parse(responseAs[String]).extract[Seq[SchemaDefinition]] - responseBody.length shouldEqual 1 - responseBody.head.url shouldEqual "http://hl7.org/fhir/StructureDefinition/Patient" - responseBody.head.id shouldEqual "Patient" - responseBody.head.fieldDefinitions.get.length shouldEqual 22 - } - - /** - * Test case to check if the correct schema definitions are returned for multiple profile URLs with FHIR version R4. - */ - Get( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_BASE_PROFILES}?${FhirDefinitionsEndpoint.QUERY_PARAM_FHIR_VERSION}=${MajorFhirVersion.R4}&${FhirDefinitionsEndpoint.QUERY_PARAM_PROFILE}=http://hl7.org/fhir/StructureDefinition/Patient,http://hl7.org/fhir/StructureDefinition/Condition" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - val responseBody = JsonMethods.parse(responseAs[String]).extract[Seq[SchemaDefinition]] - responseBody.length shouldEqual 2 - responseBody.head.url shouldEqual "http://hl7.org/fhir/StructureDefinition/Condition" - responseBody.head.id shouldEqual "Condition" - responseBody.head.fieldDefinitions.get.length shouldEqual 23 - responseBody.last.url shouldEqual "http://hl7.org/fhir/StructureDefinition/Patient" - responseBody.last.id shouldEqual "Patient" - responseBody.last.fieldDefinitions.get.length shouldEqual 22 - } - } - - /** - * Test case to verify retrieval of simplified element definitions for a given FHIR profile. - * */ - "retrieve simplified element definitions of a FHIR profile" in { - Get( - s"/${webServerConfig.baseUri}/${FhirDefinitionsEndpoint.SEGMENT_FHIR_DEFINITIONS}?$QUERY_PARAM_Q=${DefinitionsQuery.ELEMENTS}&$QUERY_PARAM_PROFILE=http://hl7.org/fhir/StructureDefinition/Patient" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - JsonMethods.parse(responseAs[String]).extract[Seq[SimpleStructureDefinition]].length shouldEqual 22 - } - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/MappingExecutionEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/MappingExecutionEndpointTest.scala deleted file mode 100644 index 8db47c41..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/MappingExecutionEndpointTest.scala +++ /dev/null @@ -1,851 +0,0 @@ -package io.ignifyr.integrationtest - -import akka.actor.ActorSystem -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, Multipart, StatusCodes} -import akka.http.scaladsl.testkit.RouteTestTimeout -import akka.testkit.TestDuration -import io.onfhir.api.Resource -import io.onfhir.api.client.FhirBatchTransactionRequestBuilder -import io.onfhir.api.model.HttpStatus -import io.ignifyr.OnFhirTestContainer -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.endpoint.{ - JobEndpoint, - MappingContextEndpoint, - MappingEndpoint, - ProjectEndpoint, - SchemaDefinitionEndpoint -} -import io.ignifyr.server.model.{ResourceFilter, TestResourceCreationRequest} -import io.ignifyr.server.util.{FileOperations, TestUtil} -import org.apache.spark.sql.SparkSession -import org.json4s._ -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -import java.io.File -import java.nio.file.Paths -import java.util.UUID -import scala.concurrent.duration.DurationInt -import scala.io.Source -import scala.util.control.Breaks.{break, breakable} - -class MappingExecutionEndpointTest extends BaseEndpointTest with OnFhirTestContainer { - // default timeout is 1 seconds, which is not enough for some tests - implicit def default(implicit system: ActorSystem): RouteTestTimeout = RouteTestTimeout( - new DurationInt(60).second.dilated(system) - ) - - // first job to be created - val fsSinkFolderName: String = "fsSink" - val fsSinkFolder: File = new File(fsSinkFolderName) - val sparkSession: SparkSession = IgnifyrConfig.sparkSession - - val resourceFilter: ResourceFilter = ResourceFilter(numberOfRows = 3, order = "start") - - val job1Id: String = UUID.randomUUID().toString - val job2Id: String = UUID.randomUUID().toString - - val mappingJobSourceSettings: Map[String, MappingJobSourceSettings] = - Map( - "source" -> - FileSystemSourceSettings( - "test-source", - "https://aiccelerate.eu/data-integration-suite/test-data", - Paths.get(getClass.getResource("/test-data").toURI).normalize().toAbsolutePath.toString - ) - ) - var sinkSettings: SinkSettings = - FileSystemSinkSettings(path = s"./$fsSinkFolderName/job1_1", SinkContentTypes.NDJSON) - val patientMappingTask: FhirMappingTask = FhirMappingTask( - name = "patient-mapping", - mappingRef = "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping", - sourceBinding = Map("source" -> FileSystemSource(path = "patients.csv", contentType = SourceContentTypes.CSV)), - mapping = Some( - FileOperations.readJsonContentAsObject[FhirMapping]( - FileOperations.getFileIfExists(getClass.getResource("/test-mappings/patient-mapping.json").getPath) - ) - ) - ) - val batchJob: FhirMappingJob = FhirMappingJob( - id = job1Id, - name = Some("mappingJob"), - sourceSettings = mappingJobSourceSettings, - sinkSettings = sinkSettings, - mappings = Seq(patientMappingTask), - dataProcessingSettings = DataProcessingSettings(saveErroneousRecords = true, archiveMode = ArchiveModes.OFF) - ) - - // streaming job initialization - val parentStreamingFolderName = "streaming-parent-folder" - val parentStreamingFolder: File = new File(parentStreamingFolderName) - - val patientStreamingFolderName = "patients" - val patientStreamingFolder: File = new File(patientStreamingFolderName) - - var sinkSettingsForStreaming: SinkSettings = - FileSystemSinkSettings(path = s"./$fsSinkFolderName/job3", SinkContentTypes.NDJSON) - val streamingMappingJobSourceSettings: Map[String, MappingJobSourceSettings] = - Map( - "source" -> - FileSystemSourceSettings( - "streaming-test-source", - "https://some-url-for-data-source", - Paths.get(getClass.getResource(s"/$parentStreamingFolderName").toURI).normalize().toAbsolutePath.toString, - asStream = true - ) - ) - val patientStreamingMappingTask: FhirMappingTask = FhirMappingTask( - name = "some-url-for-streaming-patient-mapping", - mappingRef = "https://some-url-for-streaming-patient-mapping", - sourceBinding = - Map("source" -> FileSystemSource(path = s"$patientStreamingFolderName", contentType = SourceContentTypes.CSV)), - mapping = Some( - FileOperations.readJsonContentAsObject[FhirMapping]( - FileOperations.getFileIfExists(getClass.getResource("/test-mappings/patient-mapping.json").getPath) - ) - ) - ) - - val streamingJob: FhirMappingJob = FhirMappingJob( - id = UUID.randomUUID().toString, - name = Some("streamingMappingJob"), - sourceSettings = streamingMappingJobSourceSettings, - sinkSettings = sinkSettingsForStreaming, - mappings = Seq(patientStreamingMappingTask), - dataProcessingSettings = DataProcessingSettings(saveErroneousRecords = true, archiveMode = ArchiveModes.OFF) - ) - - // Resources to be created on onFhir server as the source data - val testPatientResource: Resource = JsonMethods - .parse(Source.fromInputStream(getClass.getResourceAsStream("/fhir-resources/patient-resource.json")).mkString) - .extract[Resource] - - "The service" should { - "run a job including a mapping" in { - // Create the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(batchJob)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that job metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check job folder is created - FileUtils - .getPath(ignifyrEngineConfig.jobRepositoryFolderPath, projectId, s"${batchJob.id}${FileExtensions.JSON}") - .toFile should exist - } - - // Run the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${batchJob.id}/${JobEndpoint.SEGMENT_RUN}", - HttpEntity(ContentTypes.`application/json`, "") - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - - // Mappings run asynchronously. Wait at most 30 seconds for mappings to complete. - val success = waitForCondition(90) { - fsSinkFolder.listFiles.exists(_.getName.contains("job1_1")) && { - // Check the resources created in the file system - val outputFolder: File = fsSinkFolder.listFiles.find(_.getName.contains("job1_1")).get - val results = sparkSession.read.text(outputFolder.getPath) - results.count() == 10 - } - } - if (!success) - fail( - "Failed to find expected number of results. Either the results are not available or the number of results does not match" - ) - } - } - - "rerun a job including a mapping" ignore { // TODO activate this test - - var firstId: Option[String] = Option.empty - - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${batchJob.id}/${JobEndpoint.SEGMENT_EXECUTIONS}?page=1" - ) ~> route ~> check { - // Get id of previous execution - val jValue = JsonMethods.parse(responseAs[String]) - firstId = (jValue.asInstanceOf[JArray].arr.head.asInstanceOf[JObject] \ "id").extractOpt[String] - } - - // Rerun the previous job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${batchJob.id}/${JobEndpoint.SEGMENT_EXECUTIONS}/${firstId.get}/${JobEndpoint.SEGMENT_RUN}", - HttpEntity(ContentTypes.`application/json`, "") - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - - // Mappings run asynchronously. Wait at most 30 seconds for mappings to complete. - val success = waitForCondition(90) { - fsSinkFolder.listFiles.exists(_.getName.contains("job1_1")) && { - // Check the resources created in the file system - val outputFolder: File = fsSinkFolder.listFiles.find(_.getName.contains("job1_1")).get - val results = sparkSession.read.text(outputFolder.getPath) - results.count() == 20 - } - } - if (!success) - fail( - "Failed to find expected number of results. Either the results are not available or the number of results does not match" - ) - } - } - - /** - * This test ensures that mapping resources (i.e. mapping definitions, context maps, etc.) becomes available to execute even if they are created - * after the server is up. - * - * Furthermore, the test also validates whether erroneous records are created appropriately - */ - "run a job with a mapping and context map that are created after the server is up" in { - // Create context map for the global project - createContextMapAndVerify("test-mappings/other-observation-concept-map.csv", "other-observation-concept-map.csv") - - // Create a new mapping - createMappingAndVerify("test-mappings/other-observation-mapping2.json", 1) - - // Update the job with the new mapping and new sink configuration - val observationsMappingTask: FhirMappingTask = FhirMappingTask( - name = "other-observation-mapping2", - mappingRef = "https://aiccelerate.eu/fhir/mappings/other-observation-mapping2", - sourceBinding = - Map("source" -> FileSystemSource(path = "other-observations.csv", contentType = SourceContentTypes.CSV)) - ) - sinkSettings = FileSystemSinkSettings(path = s"./$fsSinkFolderName/job1_2", SinkContentTypes.NDJSON) - val job1Updated = - batchJob.copy(mappings = Seq(observationsMappingTask), sinkSettings = sinkSettings, name = Some("updatedJob")) - - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/$job1Id", - HttpEntity(ContentTypes.`application/json`, writePretty(job1Updated)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the updated job - val job: FhirMappingJob = JsonMethods.parse(responseAs[String]).extract[FhirMappingJob] - job.name shouldEqual Some("updatedJob") - } - - // Run the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${batchJob.id}/${JobEndpoint.SEGMENT_RUN}", - HttpEntity(ContentTypes.`application/json`, "") - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - - // Mappings run asynchronously. Wait at most 30 seconds for mappings to complete. - var success = waitForCondition(90) { - fsSinkFolder.listFiles.exists(_.getName.contains("job1_2")) && { - // Check the resources created in the file system - val parquetFolder = fsSinkFolder.listFiles.find(_.getName.contains("job1_2")).get - val results = sparkSession.read.text(parquetFolder.getPath) - results.count() == 13 - } - } - if (!success) - fail( - "Failed to find expected number of results. Either the results are not available or the number of results does not match" - ) - - // test if erroneous records are written to error folder - success = waitForCondition(90) { - val erroneousRecordsFolder = - Paths.get(ignifyrEngineConfig.erroneousRecordsFolder, FhirMappingErrorCodes.MAPPING_ERROR) - val jobFolder = Paths.get(erroneousRecordsFolder.toString, s"job-${batchJob.id}").toFile - jobFolder.exists() && { - val csvFile = jobFolder - .listFiles() - .headOption - .flatMap( // execution folder - _.listFiles().headOption.flatMap( // mapping task folder - _.listFiles().headOption.flatMap( // source folder i.e. main source, secondary source etc. - _.listFiles().headOption - ) - ) - ) // csv file - // Spark initially writes data to files in the "_temporary" directory. After all tasks complete successfully, - // the files are moved from "_temporary" to the parent output directory, and "_temporary" is deleted. This - // intermediate step can be observed during testing, which is why we check if the file is a CSV. - csvFile.isDefined && csvFile.get.exists() && csvFile.get.getName.endsWith(".csv") && { - val csvFileContent = sparkSession.read.option("header", "true").csv(csvFile.get.getPath) - csvFileContent.count() == 1 - } - } - } - if (!success) - fail( - "Failed to find expected number of erroneous records. Either the erroneous record file is not available or the number of records does not match" - ) - } - } - - "save erroneous records for job with multiple sources" in { - // Create a new mapping - createMappingAndVerify("test-mappings/patient-mapping-with-two-sources.json", 2) - - // Update the job with the new mapping and new sink configuration - val patientMappingTask: FhirMappingTask = FhirMappingTask( - name = "patient-mapping-two-sources", - mappingRef = "http://patient-mapping-with-two-sources", - sourceBinding = Map( - "patient" -> FileSystemSource(path = "patient-simple.csv", contentType = SourceContentTypes.CSV), - "patientGender" -> FileSystemSource(path = "patient-gender-simple.csv", contentType = SourceContentTypes.CSV) - ) - ) - sinkSettings = FhirRepositorySinkSettings(fhirRepoUrl = onFhirClient.getBaseUrl()) - val job = batchJob.copy( - id = UUID.randomUUID().toString, - mappings = Seq(patientMappingTask), - sinkSettings = sinkSettings, - name = Some("twoSourceJob") - ) - - // Create the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(job)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - } - - // Run the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${job.id}/${JobEndpoint.SEGMENT_RUN}", - HttpEntity(ContentTypes.`application/json`, "") - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - - // test if erroneous records are written to error folder - val success = waitForCondition(120) { - val erroneousRecordsFolder = - Paths.get(ignifyrEngineConfig.erroneousRecordsFolder, FhirMappingErrorCodes.MAPPING_ERROR) - val jobFolder = Paths.get(erroneousRecordsFolder.toString, s"job-${job.id}").toFile - jobFolder.exists() && { - val sourceFolders = jobFolder - .listFiles() - .headOption - .flatMap( // execution folder - _.listFiles().headOption.map( // mapping task folder - _.listFiles() - ) - ) // source folder i.e. main source, secondary source etc. - sourceFolders.isDefined && sourceFolders.get.length == 2 && { - val mainSource = sourceFolders.get match { - case folders if folders.head.getName.contentEquals("mainSource") => folders.head - case folders => folders.last - } - val csvFile = mainSource.listFiles().head - // Spark initially writes data to files in the "_temporary" directory. After all tasks complete successfully, - // the files are moved from "_temporary" to the parent output directory, and "_temporary" is deleted. This - // intermediate step can be observed during testing, which is why we check if the file is a CSV. - csvFile.exists() && csvFile.getName.endsWith(".csv") && { - val csvFileContent = sparkSession.read.option("header", "true").csv(csvFile.getPath) - csvFileContent.count() == 1 - } - } && { - val secondarySource = sourceFolders.get match { - case folders if folders.last.getName.contentEquals("patientGender") => folders.last - case folders => folders.head - } - val csvFile = secondarySource.listFiles().head - // Spark initially writes data to files in the "_temporary" directory. After all tasks complete successfully, - // the files are moved from "_temporary" to the parent output directory, and "_temporary" is deleted. This - // intermediate step can be observed during testing, which is why we check if the file is a CSV. - csvFile.exists() && csvFile.getName.endsWith(".csv") && { - val csvFileContent = sparkSession.read.option("header", "true").csv(csvFile.getPath) - csvFileContent.count() == 1 - } - } - } - } - if (!success) - fail( - "Failed to find expected number of erroneous records. Either the erroneous record file is not available or the number of records does not match" - ) - } - } - - sinkSettings = FileSystemSinkSettings(path = s"./$fsSinkFolderName/job2", SinkContentTypes.NDJSON) - val job2: FhirMappingJob = FhirMappingJob( - name = Some("mappingJob2"), - sourceSettings = mappingJobSourceSettings, - sinkSettings = sinkSettings, - mappings = Seq(patientMappingTask), - dataProcessingSettings = DataProcessingSettings() - ) - - "execute a mapping that is included in the mapping task" in { - // create the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(job2)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that job metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 3 - // check job folder is created - FileUtils - .getPath(ignifyrEngineConfig.jobRepositoryFolderPath, projectId, s"${job2.id}${FileExtensions.JSON}") - .toFile should exist - } - - // Create the schema - createSchemaAndVerify("test-schemas/patient-schema.json", 1) - - // Run mapping and verify results - initializeTestMappingQuery( - job2.id, - "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping", - Map("source" -> FileSystemSource(path = "patients.csv", contentType = SourceContentTypes.CSV)), - Some( - FileOperations.readJsonContentAsObject[FhirMapping]( - FileOperations.getFileIfExists(getClass.getResource("/test-mappings/patient-mapping.json").getPath) - ) - ) - ) ~> check { - - status shouldEqual StatusCodes.OK - val results: Seq[FhirMappingResultsForInput] = - JsonMethods.parse(responseAs[String]).extract[Seq[FhirMappingResultsForInput]] - results.length shouldEqual 3 - results.head.mappingTaskName shouldEqual "patient-mapping" - results.head.mappedFhirResources.head.mappedResource.get shouldEqual "{\"resourceType\":\"Patient\"," + - "\"id\":\"34dc88d5972fd5472a942fc80f69f35c\"," + - "\"meta\":{\"profile\":[\"https://aiccelerate.eu/fhir/StructureDefinition/AIC-Patient\"]," + - "\"source\":\"https://aiccelerate.eu/data-integration-suite/test-data\"}," + - "\"active\":true,\"identifier\":[{\"use\":\"official\",\"system\":\"https://aiccelerate.eu/data-integration-suite/test-data\",\"value\":\"p1\"}]," + - "\"gender\":\"male\",\"birthDate\":\"2000-05-10\"}" - } - } - - "execute a mapping within a job without passing the mapping in the mapping task" in { - // create the mapping that will be tested - createMappingAndVerify("test-mappings/patient-mapping2.json", 3) - - initializeTestMappingQuery( - job2.id, - "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping2", - Map("source" -> FileSystemSource(path = "patients.csv", contentType = SourceContentTypes.CSV)) - ) ~> check { - status shouldEqual StatusCodes.OK - val results: Seq[FhirMappingResultsForInput] = - JsonMethods.parse(responseAs[String]).extract[Seq[FhirMappingResultsForInput]] - results.length shouldEqual 3 - results.head.mappingTaskName shouldEqual "patient-mapping2" - results.head.mappedFhirResources.head.mappedResource.get shouldEqual "{\"resourceType\":\"Patient\"," + - "\"id\":\"34dc88d5972fd5472a942fc80f69f35c\"," + - "\"meta\":{\"profile\":[\"https://aiccelerate.eu/fhir/StructureDefinition/AIC-Patient\"]," + - "\"source\":\"https://aiccelerate.eu/data-integration-suite/test-data\"}," + - "\"active\":false,\"identifier\":[{\"use\":\"official\",\"system\":\"https://aiccelerate.eu/data-integration-suite/test-data\",\"value\":\"p1\"}]," + - "\"gender\":\"male\",\"birthDate\":\"2000-05-10\"}" - } - } - - /** - * This test aims to test a mapping, containing a reference to a concept map, can be executed when a mapping is created after the server is up. - * The following activities are performed in the test: - * 1) Source schema is created - * 2) The mapping is created - * 3) Mapping is run via the test endpoint - */ - "execute a mapping with a context within a job" in { - createSchemaAndVerify("test-schemas/other-observation-schema.json", 2) - createMappingAndVerify("test-mappings/other-observation-mapping.json", 4) - - // test a mapping - initializeTestMappingQuery( - job2.id, - "https://aiccelerate.eu/fhir/mappings/other-observation-mapping", - Map("source" -> FileSystemSource(path = "other-observations.csv", contentType = SourceContentTypes.CSV)) - ) ~> check { - status shouldEqual StatusCodes.OK - val results: Seq[FhirMappingResultsForInput] = - JsonMethods.parse(responseAs[String]).extract[Seq[FhirMappingResultsForInput]] - results.length shouldEqual 3 - results.head.mappingTaskName shouldEqual "other-observation-mapping" - - val result: JObject = - JsonMethods.parse(results.head.mappedFhirResources.head.mappedResource.get).asInstanceOf[JObject] - (result \ "meta" \ "profile") - .asInstanceOf[JArray] - .arr - .head - .extract[String] shouldEqual "https://aiccelerate.eu/fhir/StructureDefinition/AIC-IntraOperativeObservation" - (result \ "effectiveDateTime").extract[String] startsWith "2007-10-12T10:00:00" - (result \ "valueQuantity" \ "value").extract[Int] shouldEqual 450 - } - } - - /** - * Run streaming job with streaming data source folder, which contains some initial data - * Then add data to the streaming data source folder and check if the streaming job is triggered - */ - "run a streaming job including a mapping" in { - // Create the streaming job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(streamingJob)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that job metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 4 - // check job folder is created - FileUtils - .getPath(ignifyrEngineConfig.jobRepositoryFolderPath, projectId, s"${streamingJob.id}${FileExtensions.JSON}") - .toFile should exist - } - - // Run the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${streamingJob.id}/${JobEndpoint.SEGMENT_RUN}", - HttpEntity(ContentTypes.`application/json`, "") - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - - // Mappings run asynchronously. Wait at most 30 seconds for the job to finish - val success = waitForCondition(90) { - fsSinkFolder.listFiles.exists(_.getName.contains("job3")) && { - // Check the resources created in the file system - val outputFolder: File = fsSinkFolder.listFiles.find(_.getName.contains("job3")).get - val results = sparkSession.read.text(outputFolder.getPath) - results.count() == 10 - } - } - if (!success) - fail( - "Failed to find expected number of results. Either the results are not available or the number of results does not match." - ) - - // put more data by copying same csv file with different name - val csvFile = Paths - .get(getClass.getResource(s"/$parentStreamingFolderName/$patientStreamingFolder/patients.csv").toURI) - .toFile - // clone the csv file with different name - val csvFile2 = new File(csvFile.getParentFile, "patients2.csv") - org.apache.commons.io.FileUtils.copyFile(csvFile, csvFile2) - // mappings run asynchronously, wait at most 30 seconds for the job to finish - // check if the mapping is executed with cloned/new csv file automatically - val success2 = waitForCondition(90) { - fsSinkFolder.listFiles.exists(_.getName.contains("job3")) && { - // Check the resources created in the file system - val outputFolder: File = fsSinkFolder.listFiles.find(_.getName.contains("job3")).get - val results = sparkSession.read.text(outputFolder.getPath) - results.count() == 20 - } - } - if (!success2) fail("Failed to find empty sink file.") - } - } - - /** - * Test case to run a batch mapping job which converts FHIR Resources into a flat schema and writes them to a CSV file. - * - * Steps: - * 1. Create a mapping. - * 2. Create a batch mapping job with the generated mapping, source settings pointing to an FHIR server, and sink settings - * to write the results to a CSV file. - * 3. Post the job creation request to the server and check for the Created status. - * 4. Run the created job and check for the OK status. - * 5. Wait up to 30 seconds for the job to complete and verify that the CSV file is created with the expected results. - * 6. Check that the CSV file contains the correct data with specific fields and values. - */ - "run a batch mapping job which converts FHIR Resources into a flat schema and write them to a CSV file" in { - // create the mapping - createMappingAndVerify("test-mappings/patient-flat-mapping.json", 5) - // create the job - val jobId = UUID.randomUUID().toString - val job: FhirMappingJob = FhirMappingJob( - id = jobId, - name = Some("patient-flat-job"), - sourceSettings = Map( - "source" -> FhirServerSourceSettings( - name = "fhir-server", - sourceUri = "http://fhir-server", - serverUrl = onFhirClient.getBaseUrl() - ) - ), - sinkSettings = FileSystemSinkSettings( - path = s"$fsSinkFolder/results.csv", - contentType = SinkContentTypes.CSV, - options = Map("header" -> "true") - ), - mappings = Seq( - FhirMappingTask( - name = "patient-flat-mapping", - mappingRef = "http://patient-flat-mapping", - sourceBinding = Map("patient" -> FhirServerSource(resourceType = "Patient")), - mapping = Some( - FileOperations.readJsonContentAsObject[FhirMapping]( - FileOperations.getFileIfExists(getClass.getResource("/test-mappings/patient-flat-mapping.json").getPath) - ) - ) - ) - ) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(job)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - } - // run the job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/$jobId/${JobEndpoint.SEGMENT_RUN}", - HttpEntity(ContentTypes.`application/json`, "") - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // Mappings run asynchronously. Wait at most 45 seconds for mappings to complete. - val success = waitForCondition(90) { - fsSinkFolder.listFiles.exists(_.getName.contains("results.csv")) && { - // catch any exception that might throw by spark during reading the results file - // because the file might not be ready yet (e.g. file is created but still being written) - try { - val outputFolder = fsSinkFolder.listFiles.find(_.getName.contains("results.csv")).get - val results = sparkSession.read.option("header", "true").csv(outputFolder.getPath) - results.count() == 1 && { - val row = results.head() - val isValid = row.getAs[String]("active") == "true" && - row.getAs[String]("birthDate") == "1974-12-25" && - row.getAs[String]("gender") == "male" && - row.getAs[String]("homeCountry") == "Australia" && - row.getAs[String]("id") == "example" && - row.getAs[String]("officialName") == "Peter Chalmers" && - row.getAs[String]("phone") == "(03) 5555 6473" - isValid - } - } catch { - case e: Exception => false - } - } - } - - if (!success) - fail( - "Failed to find expected number of results. Either the results are not available or the number of results does not match" - ) - } - } - } - - /** - * Deserializes a [[SchemaDefinition]] from the specified file. Submits the definition into the global project. Then verifies the results. - * - * @param schemaDefinitionResourceFile Name of the file to read the schema definition content - * @param expectedSchemaCount Expected number of schemas within global project - */ - private def createSchemaAndVerify(schemaDefinitionResourceFile: String, expectedSchemaCount: Int): Unit = { - val otherObservationSchemaFile: File = - FileOperations.getFileIfExists(getClass.getResource(s"/$schemaDefinitionResourceFile").getPath) - val otherObservationSourceSchema: SchemaDefinition = - FileOperations.readJsonContentAsObject[SchemaDefinition](otherObservationSchemaFile) - - // create the schema that the mapping uses - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(otherObservationSourceSchema)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that schema metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "schemas") - .asInstanceOf[JArray] - .arr - .length shouldEqual expectedSchemaCount - // check schema folder is created - FileUtils - .getPath( - ignifyrEngineConfig.schemaRepositoryFolderPath, - projectId, - s"${otherObservationSourceSchema.id}${FileExtensions.JSON}" - ) - .toFile should exist - } - } - - /** - * First creates mapping context for the global project. Afterwards, the content of the concept map is read from the specified file and associated with the mapping context. - * - * @param conceptMapResourceFile File to read the content of the test concept map - * @param mappingContextId Name of the mapping context. This id is also used as the name of the file keeping the concept map as maintained by the repository. - */ - def createContextMapAndVerify(conceptMapResourceFile: String, mappingContextId: String): Unit = { - // upload the concept map inside the project - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}", - HttpEntity(ContentTypes.`text/plain(UTF-8)`, mappingContextId) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that mapping context is updated in projects.json file - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingContexts") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check mapping context file is created - FileUtils - .getPath(ignifyrEngineConfig.mappingContextRepositoryFolderPath, projectId, mappingContextId) - .toFile should exist - } - // get file from resources - val file: File = FileOperations.getFileIfExists(getClass.getResource(s"/$conceptMapResourceFile").getPath) - val fileData = Multipart.FormData.BodyPart.fromPath("attachment", ContentTypes.`text/plain(UTF-8)`, file.toPath) - val formData = Multipart.FormData(fileData) - // save a csv file to mapping context - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}/$mappingContextId/file", - formData.toEntity() - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - responseAs[String] shouldEqual "OK" - } - } - - /** - * Reads the mapping file from the file system into a [[FhirMapping]] instance. Posts the mapping into the global project and verifies that the mapping is inside the project folder. - * - * @param mappingResourceFile Name of the file including the mapping definition - * @param expectedMappingCount Number of mappings to be included in the global project - */ - private def createMappingAndVerify(mappingResourceFile: String, expectedMappingCount: Int): Unit = { - val mappingFile: File = FileOperations.getFileIfExists(getClass.getResource(s"/$mappingResourceFile").getPath) - val mapping: FhirMapping = FileOperations.readJsonContentAsObject[FhirMapping](mappingFile) - - // create the mapping that will be tested - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that mapping metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappings") - .asInstanceOf[JArray] - .arr - .length shouldEqual expectedMappingCount - // check mapping folder is created - FileUtils - .getPath(ignifyrEngineConfig.mappingRepositoryFolderPath, projectId, s"${mapping.id}${FileExtensions.JSON}") - .toFile should exist - } - } - - /** - * Creates a [[TestResourceCreationRequest]] with the given input and returns a [[RouteTestResult]] to be checked. - * - * @param jobId Identifier of the job for which the test will be run - * @param mappingRef Url of the mapping to be tested - * @param sourceBinding A map of defining the sources from which the test data will be read e.g. Map("source" -> FileSystemSource(path = "patients.csv")) - * @param mapping Mapping itself to be tested. If the mapping itself is not provided, it is resolved via the mappingRef - * @return - */ - private def initializeTestMappingQuery( - jobId: String, - mappingRef: String, - sourceBinding: Map[String, MappingSourceBinding], - mapping: Option[FhirMapping] = None - ): RouteTestResult = { - val otherObservationsMappingTask: FhirMappingTask = FhirMappingTask( - name = mappingRef.split("/").last, - mappingRef = mappingRef, - sourceBinding = sourceBinding, - mapping - ) - val createTestResourcesRequest: TestResourceCreationRequest = TestResourceCreationRequest( - fhirMappingTask = otherObservationsMappingTask, - resourceFilter = resourceFilter - ) - - // test a mapping - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/$jobId/${JobEndpoint.SEGMENT_TEST}", - HttpEntity(ContentTypes.`application/json`, writePretty(createTestResourcesRequest)) - ) ~> route - } - - /** - * Define a function to wait for a condition with a timeout - * - * @param timeoutSeconds timeout in seconds to wait for the condition - * @param condition condition to be checked - * @return - */ - private def waitForCondition(timeoutSeconds: Int)(condition: => Boolean): Boolean = { - var success = false - breakable { - for (_ <- 1 to timeoutSeconds) { // Sleep 1000ms up to `timeoutSeconds` times - Thread.sleep(1000) - if (condition) { - success = true - break() - } - } - } - success - } - - /** - * Creates a project to be used in the tests - * */ - override def beforeAll(): Unit = { - super.beforeAll() - - // Warm Spark up front so no timed request pays SparkSession startup. - IgnifyrConfig.sparkSession.sql("SELECT 1").collect() - org.apache.commons.io.FileUtils.deleteDirectory(new File(fsSinkFolderName)) - org.apache.commons.io.FileUtils.deleteDirectory(Paths.get(ignifyrEngineConfig.erroneousRecordsFolder).toFile) - fsSinkFolder.mkdirs() - this.createProject(Some("deadbeef-dead-dead-dead-deaddeafbeef")) - // create the test resources on onFHIR test container - onFhirClient - .batch() - .entry(_.update(testPatientResource)) - .returnMinimal() - .asInstanceOf[FhirBatchTransactionRequestBuilder] - .execute() map { res => - res.httpStatus shouldBe HttpStatus.OK - } - } - - override def afterAll(): Unit = { - super.afterAll() - org.apache.commons.io.FileUtils.deleteDirectory(fsSinkFolder) - // delete checkpoint folder - org.apache.commons.io.FileUtils.deleteDirectory(Paths.get(IgnifyrConfig.sparkCheckpointDirectory).toFile) - // delete erroneous folder - org.apache.commons.io.FileUtils.deleteDirectory(Paths.get(ignifyrEngineConfig.erroneousRecordsFolder).toFile) - // remove cloned csv file used for streaming job - org.apache.commons.io.FileUtils.delete( - Paths.get(getClass.getResource(s"/$parentStreamingFolderName/$patientStreamingFolder/patients2.csv").toURI).toFile - ) - // remove the test resources from onFHIR test container - onFhirClient - .batch() - .entry(_.delete("Patient")) - .returnMinimal() - .asInstanceOf[FhirBatchTransactionRequestBuilder] - .execute() map { res => - res.httpStatus shouldBe HttpStatus.OK - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/SchemaEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/SchemaEndpointTest.scala deleted file mode 100644 index b4e2571f..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/integrationtest/SchemaEndpointTest.scala +++ /dev/null @@ -1,1215 +0,0 @@ -package io.ignifyr.integrationtest - -import akka.actor.ActorSystem -import akka.http.scaladsl.model.{ContentType, ContentTypes, HttpEntity, MediaTypes, Multipart, StatusCodes} -import akka.http.scaladsl.testkit.RouteTestTimeout -import io.onfhir.api.client.FhirBatchTransactionRequestBuilder -import io.onfhir.api.model.HttpStatus -import io.onfhir.api.{FHIR_DATA_TYPES, FHIR_FOUNDATION_RESOURCES, Resource} -import io.ignifyr.OnFhirTestContainer -import io.onfhir.definitions.common.model.{DataTypeWithProfiles, SchemaDefinition, SimpleStructureDefinition} -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.redcap.RedCapSchemaImportEndpoint -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.endpoint.{MappingEndpoint, ProjectEndpoint, SchemaDefinitionEndpoint, SchemaFormats} -import io.ignifyr.server.endpoint.SchemaDefinitionEndpoint.{QUERY_PARAM_TYPE, QUERY_PARAM_URL} -import io.ignifyr.server.model.{ImportSchemaSettings, InferTask} -import io.ignifyr.server.util.{FileOperations, TestUtil} -import org.json4s.JArray -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -import java.io.File -import java.nio.file.{Files, Paths} -import java.sql.{Connection, DriverManager, Statement} -import scala.concurrent.duration._ -import scala.io.{BufferedSource, Source} -import scala.util.{Failure, Success, Using} - -class SchemaEndpointTest extends BaseEndpointTest with OnFhirTestContainer { - // database url for infer schema test - val DATABASE_URL = "jdbc:h2:mem:inputDb;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE" - - // generous timeout: preprocessSql inference falls back to a Spark read inside the request - implicit def default(implicit system: ActorSystem): RouteTestTimeout = RouteTestTimeout(60.seconds) - - // inferTask object for infer schema test - val inferTaskWithTableName: InferTask = InferTask( - name = "test", - mappingJobSourceSettings = Map( - "source" -> - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = DATABASE_URL, - username = "", - password = "" - ) - ), - sourceBinding = SqlSource(tableName = Some("death")) - ) - - val q = - """SELECT - | p.person_id, - | p.name, - | p.surname, - | p.date_of_birth, - | d.death_date, - | d.death_datetime - |FROM person p, death d WHERE p.person_id = d.person_id""".stripMargin - - val inferTaskWithSqlQuery: InferTask = InferTask( - name = "test", - mappingJobSourceSettings = Map( - "source" -> - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = DATABASE_URL, - username = "", - password = "" - ) - ), - sourceBinding = SqlSource(query = Some(q)) - ) - - val inferTaskWithPreprocessSql: InferTask = InferTask( - name = "test", - mappingJobSourceSettings = Map( - "source" -> - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = DATABASE_URL, - username = "", - password = "" - ) - ), - sourceBinding = SqlSource( - query = Some("select * from death"), - preprocessSql = Some("select person_id, death_date, death_datetime, cause_source_value from test") - ) - ) - - // first schema to be created - val schema1: SchemaDefinition = SchemaDefinition( - url = "https://example.com/fhir/StructureDefinition/schema", - version = SchemaDefinition.VERSION_LATEST, - `type` = "Ty", - name = "name", - description = Some("description"), - rootDefinition = None, - fieldDefinitions = None - ) - // second schema to be created - val schema2: SchemaDefinition = SchemaDefinition( - url = "https://example.com/fhir/StructureDefinition/schema2", - version = SchemaDefinition.VERSION_LATEST, - `type` = "Ty2", - name = "name2", - description = Some("description2"), - rootDefinition = None, - fieldDefinitions = None - ) - val schema2_2: SchemaDefinition = SchemaDefinition( - url = "https://example.com/fhir/StructureDefinition/schema2", - version = "0.9.12", - `type` = "Ty2", - name = "name2", - description = Some("old-version-description2"), - rootDefinition = None, - fieldDefinitions = None - ) - // third schema to be created - // it includes two elements: - // - element-with-definition => An element having a definition - // - element-with-no-definition => An element having no definition - val schema3: SchemaDefinition = SchemaDefinition( - url = "https://example.com/fhir/StructureDefinition/schema3", - version = SchemaDefinition.VERSION_LATEST, - `type` = "Ty3", - name = "name3", - description = Some("description3"), - rootDefinition = None, - fieldDefinitions = Some( - Seq( - SimpleStructureDefinition( - id = "element-with-definition", - path = "Ty3.element-with-definition", - dataTypes = Some( - Seq( - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CANONICAL, - profiles = Some(Seq("http://hl7.org/fhir/StructureDefinition/canonical")) - ) - ) - ), - isPrimitive = true, - isChoiceRoot = false, - isArray = false, - minCardinality = 0, - maxCardinality = None, - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = Some("element-with-definition"), - definition = Some("element definition"), - comment = None, - elements = None - ), - SimpleStructureDefinition( - id = "element-with-no-definition", - path = "Ty3.element-with-no-definition", - dataTypes = Some( - Seq( - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CANONICAL, - profiles = Some(Seq("http://hl7.org/fhir/StructureDefinition/canonical")) - ) - ) - ), - isPrimitive = true, - isChoiceRoot = false, - isArray = false, - minCardinality = 0, - maxCardinality = None, - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = Some("element-with-no-definition"), - definition = None, - comment = None, - elements = None - ) - ) - ) - ) - // fourth schema to be created - // it includes two elements: - // - element-with-short => An element having a short - // - element-with-no-short => An element having no short - val schema4: SchemaDefinition = SchemaDefinition( - url = "https://example.com/fhir/StructureDefinition/schema4", - version = SchemaDefinition.VERSION_LATEST, - `type` = "Ty4", - name = "name4", - description = Some("description4"), - rootDefinition = None, - fieldDefinitions = Some( - Seq( - SimpleStructureDefinition( - id = "element-with-short", - path = "Ty4.element-with-short", - dataTypes = Some( - Seq( - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CANONICAL, - profiles = Some(Seq("http://hl7.org/fhir/StructureDefinition/canonical")) - ) - ) - ), - isPrimitive = true, - isChoiceRoot = false, - isArray = false, - minCardinality = 0, - maxCardinality = None, - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = Some("element-with-short"), - definition = None, - comment = None, - elements = None - ), - SimpleStructureDefinition( - id = "element-with-no-short", - path = "Ty4.element-with-no-short", - dataTypes = Some( - Seq( - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CANONICAL, - profiles = Some(Seq("http://hl7.org/fhir/StructureDefinition/canonical")) - ) - ) - ), - isPrimitive = true, - isChoiceRoot = false, - isArray = false, - minCardinality = 0, - maxCardinality = None, - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = None, - definition = None, - comment = None, - elements = None - ) - ) - ) - ) - // fifth schema with the same url and version as the second schema (but different id's) - val schema5: SchemaDefinition = SchemaDefinition( - id = "schema5", - url = "https://example.com/fhir/StructureDefinition/schema2", - version = SchemaDefinition.VERSION_LATEST, - `type` = "Ty5", - name = "name5", - description = Some("description5"), - rootDefinition = None, - fieldDefinitions = None - ) - - // mapping using schema2 - val mapping: FhirMapping = FhirMapping( - id = "mapping", - url = "http://example.com/mapping", - name = "mapping", - source = Seq(FhirMappingSource(alias = "test", url = "https://example.com/fhir/StructureDefinition/schema2")), - context = Map.empty, - mapping = Seq.empty - ) - - "The service" should { - - "create a schema within project" in { - // create the first schema - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that schema metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "schemas") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check schema folder is created - FileUtils - .getPath(ignifyrEngineConfig.schemaRepositoryFolderPath, projectId, s"${schema1.id}${FileExtensions.JSON}") - .toFile should exist - } - // create the second schema - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema2)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that schema metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "schemas") - .asInstanceOf[JArray] - .arr - .length shouldEqual 2 - FileUtils - .getPath(ignifyrEngineConfig.schemaRepositoryFolderPath, projectId, s"${schema2.id}${FileExtensions.JSON}") - .toFile should exist - } - } - - "get all schemas in a project" in { - // get all schemas within a project - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns two schemas - val schemas: Seq[SchemaDefinition] = JsonMethods.parse(responseAs[String]).extract[Seq[SchemaDefinition]] - schemas.length shouldEqual 2 - } - } - - "get a schema in a project" in { - // get a schema - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${schema1.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual schema1.url - schema.`type` shouldEqual schema1.`type` - } - // get a schema with invalid id - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/123123" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "get a schema by url in a project" in { - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?url=${schema1.url}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual schema1.url - schema.name shouldEqual schema1.name - } - } - - "update a schema in a project" in { - // update a schema - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${schema1.id}", - HttpEntity( - ContentTypes.`application/json`, - writePretty(schema1.copy(url = "https://example.com/fhir/StructureDefinition/schema3", name = schema3.name)) - ) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned schema includes the update - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual "https://example.com/fhir/StructureDefinition/schema3" - // validate that schema metadata is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - ( - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "schemas") - .asInstanceOf[JArray] - .arr - .find(m => (m \ "id").extract[String].contentEquals(schema1.id)) - .get \ "url" - ) - .extract[String] shouldEqual "https://example.com/fhir/StructureDefinition/schema3" - } - // update a schema with invalid id - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/123123", - HttpEntity(ContentTypes.`application/json`, writePretty(schema2)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - } - } - - "delete a schema from a project" in { - // delete a schema - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${schema1.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.NoContent - // validate that schema metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "schemas") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check schema folder is deleted - FileUtils - .getPath(ignifyrEngineConfig.schemaRepositoryFolderPath, projectId, s"${schema1.id}${FileExtensions.JSON}") - .toFile shouldNot exist - } - // delete a schema with invalid id - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/123123" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "cannot delete a schema from a project if it is referenced by some mappings" in { - // create the mapping which makes use of schema2 - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that mapping metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappings") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check mapping folder is created - FileUtils - .getPath(ignifyrEngineConfig.mappingRepositoryFolderPath, projectId, s"${mapping.id}${FileExtensions.JSON}") - .toFile should exist - } - - // delete schema2 - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${schema2.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - } - } - - "create a schema having some elements with/without description" in { - // create the third schema - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema3)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - } - // retrieve the third schema - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?url=${schema3.url}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual schema3.url - schema.name shouldEqual schema3.name - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 2 - fieldDefinitions.head.definition.get shouldEqual "element definition" - fieldDefinitions.last.definition.nonEmpty shouldEqual false - } - } - - "create a schema having some elements with/without short" in { - // create the fourth schema - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema4)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - } - // retrieve the fourth schema - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?url=${schema4.url}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual schema4.url - schema.name shouldEqual schema4.name - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 2 - fieldDefinitions.head.short.get shouldEqual "element-with-short" - fieldDefinitions.last.short.nonEmpty shouldEqual false - } - } - - "infer the schema on a given table name and retrieve column types" in { - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(inferTaskWithTableName)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 7 - fieldDefinitions.head.dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.INTEGER - fieldDefinitions(1).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.DATE - fieldDefinitions(1).minCardinality shouldEqual 1 - fieldDefinitions(2).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.DATETIME - fieldDefinitions(2).minCardinality shouldEqual 0 - fieldDefinitions(5).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.STRING - } - } - - "infer the schema on a given SQL query and retrieve column types" in { - // infer the schema - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(inferTaskWithSqlQuery)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 6 - fieldDefinitions.head.dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.INTEGER - fieldDefinitions(1).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.STRING - fieldDefinitions(1).minCardinality shouldEqual 0 - fieldDefinitions(3).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.DATE - fieldDefinitions(3).minCardinality shouldEqual 1 - fieldDefinitions(5).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.DATETIME - } - } - - "infer the schema on a given SQL query with a preprocess SQL and retrieve column types" in { - // infer the schema - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(inferTaskWithPreprocessSql)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate data types of schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 4 - fieldDefinitions.head.dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.INTEGER - fieldDefinitions(1).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.DATE - fieldDefinitions(2).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.DATETIME - fieldDefinitions(3).dataTypes.get.head.dataType shouldEqual FHIR_DATA_TYPES.STRING - } - } - - "create schemas having the same URL but different versions" in { - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity( - ContentTypes.`application/json`, - writePretty(schema5.copy(version = "some-version-other-than-latest")) - ) - ) ~> route ~> check { - // Expect a successful write - status shouldEqual StatusCodes.Created - // validate that schema metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "schemas") - .asInstanceOf[JArray] - .arr - .length shouldEqual 4 - FileUtils - .getPath(ignifyrEngineConfig.schemaRepositoryFolderPath, projectId, s"${schema5.id}${FileExtensions.JSON}") - .toFile should exist - } - } - - "cannot create a schema having the same id, url or name as another schema" in { - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema5)) - ) ~> route ~> check { - // Expect a conflict status because schema5 has the same url and version as the schema2 - status shouldEqual StatusCodes.Conflict - } - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema5.copy(id = schema2.id))) - ) ~> route ~> check { - status shouldEqual StatusCodes.Conflict - } - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema5.copy(name = schema2.name))) - ) ~> route ~> check { - status shouldEqual StatusCodes.Conflict - } - } - - "import REDCap data dictionary file" in { - // get file from resources - val file: File = FileOperations.getFileIfExists(getClass.getResource("/redcap/instrument.csv").getPath) - val fileData = Multipart.FormData.BodyPart.fromPath("attachment", ContentTypes.`text/csv(UTF-8)`, file.toPath) - val formData = Multipart.FormData(fileData) - // the root URL of the schemas - val definitionRootUrl = "http://test-schema" - // the form field which keeps the unique identifier of REDCap records - val recordIdField = "record_id" - // the identifiers of schemas to be created. They are the form names given in the instrument.csv - val firstSchemaId = "Test" - val secondSchemaId = "Additional_information" - - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${RedCapSchemaImportEndpoint.SEGMENT_REDCAP}?rootUrl=$definitionRootUrl&recordIdField=$recordIdField", - formData.toEntity() - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the schemas - val schemas: Seq[SchemaDefinition] = JsonMethods.parse(responseAs[String]).extract[Seq[SchemaDefinition]] - schemas.size shouldEqual 2 - // validate the first one - val firstSchema = schemas.head - firstSchema.url shouldEqual s"$definitionRootUrl/${FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION}/$firstSchemaId" - var fieldDefinitions = firstSchema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 5 - // every field of a REDCap schema should be primitive - fieldDefinitions.count(_.isPrimitive) shouldEqual 5 - - // validate the second one - val secondSchema = schemas.last - secondSchema.url shouldEqual s"$definitionRootUrl/${FHIR_FOUNDATION_RESOURCES.FHIR_STRUCTURE_DEFINITION}/$secondSchemaId" - fieldDefinitions = secondSchema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 4 - // 'record_id' field should be added automatically even if it does not exist in the form definition - fieldDefinitions.count(definition => definition.id.contentEquals(recordIdField)) shouldEqual 1 - } - } - - "create an HTTP response with bad request for file data sources with wrong file extension" in { - val erroneousInferTask = inferTaskWithSqlQuery.copy( - mappingJobSourceSettings = inferTaskWithSqlQuery.mappingJobSourceSettings.updated( - "source", - FileSystemSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - dataFolderPath = "data-integration-suite/test-data" - ) - ), - sourceBinding = FileSystemSource("WRONG.wrong", contentType = SourceContentTypes.CSV) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(erroneousInferTask)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - } - } - - "create an HTTP response with bad request for data data sources with wrong file path" in { - val erroneousInferTask = inferTaskWithSqlQuery.copy( - mappingJobSourceSettings = inferTaskWithSqlQuery.mappingJobSourceSettings.updated( - "source", - FileSystemSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - dataFolderPath = "data-integration-suite/test-data" - ) - ), - sourceBinding = FileSystemSource("lab-results.csv", contentType = SourceContentTypes.CSV) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(erroneousInferTask)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - } - } - - "create an HTTP response with bad request for wrong preprocess SQL string" in { - val erroneousInferTask = inferTaskWithSqlQuery.copy( - mappingJobSourceSettings = inferTaskWithSqlQuery.mappingJobSourceSettings.updated( - "source", - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = DATABASE_URL, - username = "", - password = "" - ) - ), - sourceBinding = SqlSource(query = Some("Select * from death"), preprocessSql = Some("Wrong query string")) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(erroneousInferTask)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - } - } - - "create an HTTP response with bad request for wrong user credentials to access the DB" in { - val erroneousInferTask = inferTaskWithSqlQuery.copy(mappingJobSourceSettings = - inferTaskWithSqlQuery.mappingJobSourceSettings.updated( - "source", - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = DATABASE_URL, - username = "Wrong username", - password = "" - ) - ) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(erroneousInferTask)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - } - } - - "create an HTTP response with bad request for wrong database URL" in { - val erroneousInferTask = inferTaskWithSqlQuery.copy(mappingJobSourceSettings = - inferTaskWithSqlQuery.mappingJobSourceSettings.updated( - "source", - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = "WRONG DATABASE URL", - username = "", - password = "" - ) - ) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(erroneousInferTask)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - } - } - - "create an HTTP response with bad request for erroneous SQL query" in { - val erroneousInferTask = inferTaskWithSqlQuery.copy( - mappingJobSourceSettings = inferTaskWithSqlQuery.mappingJobSourceSettings.updated( - "source", - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = DATABASE_URL, - username = "", - password = "" - ) - ), - sourceBinding = SqlSource(query = Some("WRONG")) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(erroneousInferTask)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - } - } - - "create an HTTP response with bad request for wrong column name in the SQL query" in { - val erroneousInferTask = inferTaskWithSqlQuery.copy( - mappingJobSourceSettings = inferTaskWithSqlQuery.mappingJobSourceSettings.updated( - "source", - SqlSourceSettings( - name = "test-db-source", - sourceUri = "https://aiccelerate.eu/data-integration-suite/test-data", - databaseUrl = DATABASE_URL, - username = "", - password = "" - ) - ), - sourceBinding = SqlSource(query = Some("select WRONG from death")) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_INFER}", - HttpEntity(ContentTypes.`application/json`, writePretty(erroneousInferTask)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - } - } - - "get structure definition of a schema to export" in { - // get a schema structure definition by defining format parameter - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${schema3.id}?format=${SchemaFormats.STRUCTURE_DEFINITION}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schemaResource: Resource = JsonMethods.parse(responseAs[String]).extract[Resource] - // Create a map from resource json - var schemaResourceMap: Map[String, Any] = Map.empty - schemaResource.values.foreach(tuple => schemaResourceMap += tuple._1 -> tuple._2) - // Validate some fields of the schema - schemaResourceMap should contain allOf ( - "id" -> schema3.id, - "url" -> schema3.url, - "name" -> schema3.name, - "resourceType" -> SchemaFormats.STRUCTURE_DEFINITION, - "type" -> schema3.`type`, - "fhirVersion" -> "4.0.1", - "kind" -> "logical", - "baseDefinition" -> "http://hl7.org/fhir/StructureDefinition/Element", - "derivation" -> "specialization", - "status" -> "draft", - "abstract" -> false - ) - - // Extract the elements of the StructureDefinition to a List - val differential: Map[String, List[Any]] = schemaResourceMap.get("differential") match { - case Some(value) => value.asInstanceOf[Map[String, List[Any]]] - case None => Map.empty - } - // Validate if fieldDefinitions length of the SchemaDefinition and element length of the StructureDefinition are the same - differential.get("element") match { - case Some(elementList) => elementList should have length 3 - case None => fail("Field definitions are missing in the structure definition of the schema.") - } - } - - // get a schema with invalid id by a url including format parameter - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/123123?format=${SchemaFormats.STRUCTURE_DEFINITION}" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "import schema as structure definition" in { - // blood pressure schema information for the test - val bloodPressureSchemaUrl: String = - "https://aiccelerate.eu/fhir/StructureDefinition/Ext-plt1-hsjd-blood-pressure" - val bloodPressureSchemaName: String = "blood-pressure-schema" - - // read the StructureDefinition of the schema from file - val schemaResource: Some[Resource] = Some( - FileOperations.readJsonContentAsObject[Resource]( - FileOperations.getFileIfExists(getClass.getResource("/test-schemas/blood-pressure-schema.json").getPath) - ) - ) - - // create a schema by using StructureDefinition - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?format=${SchemaFormats.STRUCTURE_DEFINITION}", - HttpEntity(ContentTypes.`application/json`, writePretty(schemaResource)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - } - - // validate if the schema is imported correctly - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?url=$bloodPressureSchemaUrl" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual bloodPressureSchemaUrl - schema.name shouldEqual bloodPressureSchemaName - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 6 - fieldDefinitions.head.path shouldEqual "Ext-plt1-hsjd-blood-pressure.pid" - fieldDefinitions.last.path shouldEqual "Ext-plt1-hsjd-blood-pressure.diastolic" - } - } - - "import a non-existent StructureDefinition from a FHIR server" in { - // settings to import a StructureDefinition named 'NonExistentStructureDefinition' from the onFHIR - val schemaID = "NonExistentStructureDefinition" - val importSchemaSettings = ImportSchemaSettings(onFhirClient.getBaseUrl(), schemaID, None) - // import the StructureDefinition from onFHIR - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_IMPORT}", - HttpEntity(ContentTypes.`application/json`, writePretty(importSchemaSettings)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include(s"Detail: Structure Definition with id '$schemaID' does not exist.") - } - } - - "import StructureDefinition from a FHIR server" in { - // settings to import a StructureDefinition named 'CustomPatient' from the onFHIR - val schemaID = "CustomPatient" - val importSchemaSettings = ImportSchemaSettings(onFhirClient.getBaseUrl(), schemaID, None) - // import the StructureDefinition from onFHIR - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_IMPORT}", - HttpEntity(ContentTypes.`application/json`, writePretty(importSchemaSettings)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - } - // url of the generated schema - val schemaURL = "http://example.org/fhir/StructureDefinition/CustomPatient" - // validate if the schema is imported correctly - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?url=$schemaURL" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual schemaURL - schema.id shouldEqual schemaID - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 23 - val birthDateField = fieldDefinitions.head - birthDateField.path shouldEqual "Patient.birthDate" - birthDateField.minCardinality shouldEqual 1 - val extension = fieldDefinitions.lift(1).get - extension.path shouldEqual "Patient.extension" - extension.elements.get.size shouldEqual 2 - val extensionLastElement = extension.elements.get.last - extensionLastElement.path shouldEqual "Patient.extension.nationality" - extensionLastElement.sliceName.get shouldEqual "nationality" - extensionLastElement.elements.get.size shouldEqual 3 - extensionLastElement.elements.get.head.path shouldEqual "Patient.extension.nationality.url" - } - } - - "import StructureDefinitions from a ZIP of FHIR Profiles" in { - // create the FormData including the ZIP file - val zipFilePath = Paths.get(getClass.getResource("/fhir-resources/custom-profiles.zip").toURI) - val zipFileBytes = Files.readAllBytes(zipFilePath) - val formData = Multipart.FormData( - Multipart.FormData.BodyPart.Strict( - "file", - HttpEntity(ContentType(MediaTypes.`application/zip`), zipFileBytes), - Map("filename" -> "custom-profiles.zip") - ) - ) - // import the StructureDefinitions from the ZIP - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_IMPORT_ZIP}", - formData - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - } - // url of the generated Condition schema - val conditionSchemaURL = "http://example.org/fhir/StructureDefinition/CustomCondition" - // validate if the schema is imported correctly - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?url=$conditionSchemaURL" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual conditionSchemaURL - schema.id shouldEqual "CustomCondition" - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 25 - val birthDateField = fieldDefinitions.head - birthDateField.path shouldEqual "Condition.onsetDateTime" - birthDateField.minCardinality shouldEqual 1 - val extension = fieldDefinitions.lift(1).get - extension.path shouldEqual "Condition.extension" - extension.elements.get.size shouldEqual 2 - val extensionLastElement = extension.elements.get.last - extensionLastElement.path shouldEqual "Condition.extension.severity" - extensionLastElement.sliceName.get shouldEqual "severity" - extensionLastElement.elements.get.size shouldEqual 3 - extensionLastElement.elements.get.head.path shouldEqual "Condition.extension.severity.url" - } - // url of the generated Observation schema - val observationSchemaURL = "http://example.org/fhir/StructureDefinition/CustomObservation" - // validate if the schema is imported correctly - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?url=$observationSchemaURL" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved schema - val schema: SchemaDefinition = JsonMethods.parse(responseAs[String]).extract[SchemaDefinition] - schema.url shouldEqual observationSchemaURL - schema.id shouldEqual "CustomObservation" - val fieldDefinitions = schema.fieldDefinitions.get - fieldDefinitions.size shouldEqual 32 - val birthDateField = fieldDefinitions.head - birthDateField.path shouldEqual "Observation.valueQuantity" - birthDateField.minCardinality shouldEqual 1 - val extension = fieldDefinitions.lift(1).get - extension.path shouldEqual "Observation.extension" - extension.elements.get.size shouldEqual 2 - val extensionLastElement = extension.elements.get.last - extensionLastElement.path shouldEqual "Observation.extension.observationMethod" - extensionLastElement.sliceName.get shouldEqual "observationMethod" - extensionLastElement.elements.get.size shouldEqual 3 - extensionLastElement.elements.get.head.path shouldEqual "Observation.extension.observationMethod.url" - } - } - - "return an error when importing a FHIR profiles ZIP with missing referenced schemas" in { - // create the FormData including the ZIP file - val zipFilePath = - Paths.get(getClass.getResource("/fhir-resources/fhir-profiles-missing-referenced-schemas.zip").toURI) - val zipFileBytes = Files.readAllBytes(zipFilePath) - val formData = Multipart.FormData( - Multipart.FormData.BodyPart.Strict( - "file", - HttpEntity(ContentType(MediaTypes.`application/zip`), zipFileBytes), - Map("filename" -> "fhir-profiles-missing-referenced-schemas.zip") - ) - ) - // import the StructureDefinitions from the ZIP - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}/${SchemaDefinitionEndpoint.SEGMENT_IMPORT_ZIP}", - formData - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include( - "Detail: The schema with URL 'https://aiccelerate.eu/fhir/StructureDefinition/AIC-Condition' references a non-existent schema: 'https://aiccelerate.eu/fhir/StructureDefinition/AIC-Patient'. Ensure all referenced schemas exist." - ) - } - } - - /** - * Test case to verify retrieval of simplified element definitions for a given schema. - * - * This test performs the following steps: - * 1. Creates a schema with two elements and posts it to the Schema Definition endpoint. - * 2. Sends a GET request to the Schema Definition endpoint with the schema URL and SimpleStructureDefinition format - * and expects a sequence of SimpleStructureDefinition objects to be returned in the response. - */ - "retrieve simplified element definitions of a schema" in { - // create a schema with two elements - val schemaUrl: String = "https://example.com/fhir/StructureDefinition/schema" - val schema: SchemaDefinition = SchemaDefinition( - url = schemaUrl, - version = "1.4.2", - `type` = "Ty", - name = "name", - description = Some("description"), - rootDefinition = None, - fieldDefinitions = Some( - Seq( - SimpleStructureDefinition( - id = "element-with-definition", - path = "Ty.element-with-definition", - dataTypes = Some( - Seq( - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CANONICAL, - profiles = Some(Seq("http://hl7.org/fhir/StructureDefinition/canonical")) - ) - ) - ), - isPrimitive = true, - isChoiceRoot = false, - isArray = false, - minCardinality = 0, - maxCardinality = None, - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = Some("element-with-definition"), - definition = Some("element definition"), - comment = None, - elements = None - ), - SimpleStructureDefinition( - id = "element-with-no-definition", - path = "Ty.element-with-no-definition", - dataTypes = Some( - Seq( - DataTypeWithProfiles( - dataType = FHIR_DATA_TYPES.CANONICAL, - profiles = Some(Seq("http://hl7.org/fhir/StructureDefinition/canonical")) - ) - ) - ), - isPrimitive = true, - isChoiceRoot = false, - isArray = false, - minCardinality = 0, - maxCardinality = None, - boundToValueSet = None, - isValueSetBindingRequired = None, - referencableProfiles = None, - constraintDefinitions = None, - sliceDefinition = None, - sliceName = None, - fixedValue = None, - patternValue = None, - referringTo = None, - short = Some("element-with-no-definition"), - definition = None, - comment = None, - elements = None - ) - ) - ) - ) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - HttpEntity(ContentTypes.`application/json`, writePretty(schema)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - } - // retrieve the simplified element definitions of this schema - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}?$QUERY_PARAM_TYPE=${SchemaFormats.SIMPLE_STRUCTURE_DEFINITION}&$QUERY_PARAM_URL=$schemaUrl" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - JsonMethods.parse(responseAs[String]).extract[Seq[SimpleStructureDefinition]].length shouldEqual 2 - } - } - } - - /** - * Creates a project to be used in the tests. Moreover, it creates a SQL table to be used in the testing of infer schema functionality. - * */ - override def beforeAll(): Unit = { - super.beforeAll() - - // Warm Spark up front so no timed request pays SparkSession startup. - IgnifyrConfig.sparkSession.sql("SELECT 1").collect() - val sql = readFileContent("/sql/sql-source-populate.sql") - runSQL(sql) - this.createProject() - // create a test patient profile on the onFHIR server. - // this profile will be used later to generate a schema. - val testPatientProfile: Resource = JsonMethods - .parse( - Source.fromInputStream(getClass.getResourceAsStream("/fhir-resources/custom-patient-profile.json")).mkString - ) - .extract[Resource] - onFhirClient - .batch() - .entry(_.update(testPatientProfile)) - .returnMinimal() - .asInstanceOf[FhirBatchTransactionRequestBuilder] - .execute() map { res => - res.httpStatus shouldBe HttpStatus.OK - } - } - - /** - * Drop tables after schema inferring. - * */ - override def afterAll(): Unit = { - val sql = readFileContent("/sql/sql-source-drop.sql") - runSQL(sql) - super.afterAll() - // delete test resources on onFHIR - onFhirClient - .batch() - .entry(_.delete("StructureDefinition")) - .returnMinimal() - .asInstanceOf[FhirBatchTransactionRequestBuilder] - .execute() map { res => - res.httpStatus shouldBe HttpStatus.OK - } - } - - /** - * Read sql file from file system - * */ - private def readFileContent(fileName: String): String = { - val source: BufferedSource = Source.fromInputStream(getClass.getResourceAsStream(fileName)) - try source.mkString - finally source.close() - } - - /** - * Run SQL queries for setting up database - * */ - private def runSQL(sql: String): Unit = { - Using.Manager { use => - val con: Connection = use(DriverManager.getConnection(DATABASE_URL)) - val stm: Statement = use(con.createStatement) - stm.execute(sql) - con.close() - } match { - case Success(value) => value - case Failure(e) => throw e - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/BaseEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/BaseEndpointTest.scala deleted file mode 100644 index 8b89561d..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/BaseEndpointTest.scala +++ /dev/null @@ -1,106 +0,0 @@ -package io.ignifyr.server - -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes} -import akka.http.scaladsl.server.Route -import akka.http.scaladsl.testkit.ScalatestRouteTest -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.common.config.WebServerConfig -import io.ignifyr.server.common.spi.{IgnifyrServerExtension, IgnifyrServerExtensions} -import io.ignifyr.server.endpoint.{ProjectEndpoint, IgnifyrServerEndpoint} -import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig -import io.ignifyr.server.model.Project -import io.ignifyr.server.repository.project.ProjectFolderRepository -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty -import org.scalatest.BeforeAndAfterAll -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -import java.io.File -import java.util.UUID - -trait BaseEndpointTest extends AnyWordSpec with Matchers with ScalatestRouteTest with BeforeAndAfterAll { - // Ignifyr engine config - val ignifyrEngineConfig: IgnifyrEngineConfig = new IgnifyrEngineConfig(system.settings.config.getConfig("ignifyr")) - val webServerConfig = new WebServerConfig(system.settings.config.getConfig("webserver")) - val fhirDefinitionsConfig = new FhirDefinitionsConfig(system.settings.config.getConfig("fhir")) - // Server extension modules on the test classpath (e.g. REDCap), configured from the test config - val serverExtensions: Seq[IgnifyrServerExtension] = IgnifyrServerExtensions.load(system.settings.config) - // route endpoint - var route: Route = _ - - /** - * Identifier of test project which can be used in endpoint tests. - * Endpoint tests, which require a test project, should call [[createProject]] method to create it - * */ - var projectId: String = _ - - /** - * Creates a test project whose identifier is stored in [[projectId]]. - * */ - def createProject(id: Option[String] = None): Unit = { - val project1: Project = - Project(id = id.getOrElse(UUID.randomUUID().toString), name = "example", description = Some("example project")) - // create a project - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}", - HttpEntity(ContentTypes.`application/json`, writePretty(project1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - val project: Project = JsonMethods.parse(responseAs[String]).extract[Project] - // set the created project - projectId = project.id - } - } - - /** - * Create the folders and initialize the endpoint and route - */ - override def beforeAll(): Unit = { - // Deleting folders to start with a clean environment - cleanFolders() - - // onfhir needs schema folder to be created in advance, - // terminology, job folders are created automatically - FileUtils.getPath(ignifyrEngineConfig.schemaRepositoryFolderPath).toFile.mkdirs() - // Folder for the mapping repository is also created manually, as the engine's mapping repository requires it during the initialization - FileUtils.getPath(ignifyrEngineConfig.mappingRepositoryFolderPath).toFile.mkdirs() - FileUtils.getPath(fhirDefinitionsConfig.profilesPath.get).toFile.mkdirs() - FileUtils.getPath(fhirDefinitionsConfig.codesystemsPath.get).toFile.mkdirs() - FileUtils.getPath(fhirDefinitionsConfig.valuesetsPath.get).toFile.mkdirs() - // initialize endpoint and route - val endpoint = - new IgnifyrServerEndpoint(ignifyrEngineConfig, webServerConfig, fhirDefinitionsConfig, serverExtensions) - route = endpoint.ignifyrRoute - } - - /** - * Deletes the repository folders after all test cases are completed. - * */ - override def afterAll(): Unit = { - cleanFolders() - } - - private def cleanFolders(): Unit = { - // delete projects metadata file if exists - val projectsJson: File = FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile - if (projectsJson.exists()) { - org.apache.commons.io.FileUtils.delete(projectsJson) - } - org.apache.commons.io.FileUtils - .deleteDirectory(FileUtils.getPath(ignifyrEngineConfig.terminologySystemFolderPath).toFile) - org.apache.commons.io.FileUtils - .deleteDirectory(FileUtils.getPath(ignifyrEngineConfig.schemaRepositoryFolderPath).toFile) - org.apache.commons.io.FileUtils - .deleteDirectory(FileUtils.getPath(ignifyrEngineConfig.jobRepositoryFolderPath).toFile) - org.apache.commons.io.FileUtils - .deleteDirectory(FileUtils.getPath(ignifyrEngineConfig.mappingRepositoryFolderPath).toFile) - org.apache.commons.io.FileUtils - .deleteDirectory(FileUtils.getPath(ignifyrEngineConfig.mappingContextRepositoryFolderPath).toFile) - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath(fhirDefinitionsConfig.profilesPath.get).toFile) - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath(fhirDefinitionsConfig.codesystemsPath.get).toFile) - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath(fhirDefinitionsConfig.valuesetsPath.get).toFile) - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpointTest.scala deleted file mode 100644 index 3cf5a824..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/FileSystemTreeStructureEndpointTest.scala +++ /dev/null @@ -1,91 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.StatusCodes -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.endpoint.FileSystemTreeStructureEndpoint -import io.ignifyr.server.model.FilePathNode -import io.onfhir.definitions.common.model.Json4sSupport.formats -import org.json4s.jackson.JsonMethods - -import java.nio.file.{Files, Path} - -class FileSystemTreeStructureEndpointTest extends BaseEndpointTest { - - /** - * A hermetic fixture tree the suite creates and destroys itself, rather than asserting against the - * shared context root (`base_path=.`). The endpoint rejects any base path outside the context root, - * so the fixture is created *inside* it — but with a unique name and a known structure, so the - * asserted counts stay deterministic regardless of the transient folders (repository dirs, logs, - * checkpoints, …) other suites leave in the context root during a full reactor run. - * - * Layout: {{{ - * / - * alpha/ (folder, with a file inside → still leaf when files are excluded) - * data.csv - * beta/ (folder, empty) - * gamma/ (folder, empty) - * projects.json (file) - * }}} - */ - private var fixtureRoot: Path = _ - private var fixtureBasePath: String = _ - - override def beforeAll(): Unit = { - super.beforeAll() - val contextRoot = FileUtils.getPath("").toAbsolutePath - Files.createDirectories(contextRoot) - fixtureRoot = Files.createTempDirectory(contextRoot, "fs-tree-test-") - // base_path is resolved relative to the context root, so hand the endpoint the relative name. - fixtureBasePath = contextRoot.relativize(fixtureRoot).toString.replaceAll("\\\\", "/") - - val alpha = Files.createDirectory(fixtureRoot.resolve("alpha")) - Files.createDirectory(fixtureRoot.resolve("beta")) - Files.createDirectory(fixtureRoot.resolve("gamma")) - Files.createFile(fixtureRoot.resolve("projects.json")) - Files.createFile(alpha.resolve("data.csv")) - } - - override def afterAll(): Unit = { - if (fixtureRoot != null) { - org.apache.commons.io.FileUtils.deleteDirectory(fixtureRoot.toFile) - } - super.afterAll() - } - - "File system tree structure endpoint" should { - - "retrieve all folders (only directories) under the given base path" in { - Get( - s"/${webServerConfig.baseUri}/${FileSystemTreeStructureEndpoint.SEGMENT_FILE_SYSTEM_PATH}?${FileSystemTreeStructureEndpoint.QUERY_PARAM_BASE_PATH}=$fixtureBasePath&${FileSystemTreeStructureEndpoint.QUERY_PARAM_INCLUDE_FILES}=false" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - val fileNode = JsonMethods.parse(responseAs[String]).extract[FilePathNode] - fileNode.isFolder shouldEqual true - // Only the three folders; projects.json and alpha/data.csv are excluded when include_files=false. - fileNode.children.map(_.label).sorted shouldBe List("alpha", "beta", "gamma") - fileNode.children.foreach { child => - child.isFolder shouldBe true - child.children shouldBe empty // files inside a folder are excluded, so every leaf is empty - } - } - } - - "retrieve all folders including all files under the given base path" in { - Get( - s"/${webServerConfig.baseUri}/${FileSystemTreeStructureEndpoint.SEGMENT_FILE_SYSTEM_PATH}?${FileSystemTreeStructureEndpoint.QUERY_PARAM_BASE_PATH}=$fixtureBasePath&${FileSystemTreeStructureEndpoint.QUERY_PARAM_INCLUDE_FILES}=true" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - val fileNode = JsonMethods.parse(responseAs[String]).extract[FilePathNode] - fileNode.isFolder shouldEqual true - fileNode.children.length shouldBe 4 // three folders + projects.json - fileNode.children.count(_.label == "projects.json") shouldBe 1 - fileNode.children.count(_.isFolder) shouldBe 3 - // Files are now included during recursion into a folder. - fileNode.children.find(_.label == "alpha").flatMap(_.children.headOption).map(_.label) shouldBe Some("data.csv") - } - } - - } - -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrErrorHandlerTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrErrorHandlerTest.scala deleted file mode 100644 index bcbb8f9a..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrErrorHandlerTest.scala +++ /dev/null @@ -1,87 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{HttpEntity, HttpMethods, StatusCodes, Uri} -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import akka.http.scaladsl.testkit.ScalatestRouteTest -import io.onfhir.definitions.resource.model -import io.ignifyr.server.common.interceptor.IErrorHandler -import io.ignifyr.server.common.model.{IgnifyrRestCall, RequestTimeout, ResourceNotFound} -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -/** - * The exception half of the server's error handling. `IgnifyrRejectionHandlerTest` covers *rejections* — - * a different Akka mechanism, reached when no route matches or a directive refuses the request. What is - * covered here is what happens when a route **throws**: every endpoint is wrapped in this handler, so it - * is the last thing standing between an unexpected failure and an empty 500 with a leaked stack trace. - * - * The suite drives a minimal route rather than the real API because no endpoint can be made to throw an - * arbitrary exception on demand. - */ -class IgnifyrErrorHandlerTest extends AnyWordSpec with Matchers with ScalatestRouteTest with IErrorHandler { - - private val restCall = - new IgnifyrRestCall(HttpMethods.GET, Uri("/ignifyr/boom"), "test-request", HttpEntity.Empty) - - /** A route whose only job is to throw whatever the test hands it. */ - private def routeThrowing(exception: Exception): Route = - handleExceptions(exceptionHandler(restCall)) { - path("boom") { - get { - complete { - throw exception - } - } - } - } - - "The error handler" should { - - "answer an unexpected exception with 500 and name the exception type" in { - Get("/boom") ~> routeThrowing(new IllegalStateException("spark session is gone")) ~> check { - status shouldEqual StatusCodes.InternalServerError - val body = responseAs[String] - body should include("Type: https://ignifyr.io/errors/InternalError") - body should include("java.lang.IllegalStateException") - body should include("spark session is gone") - } - } - - // An IgnifyrError already carries its own status; the handler must pass it through untouched rather - // than flattening everything to 500. - "pass an IgnifyrError through with its own status" in { - Get("/boom") ~> routeThrowing(ResourceNotFound("Job not found", "No job with id x")) ~> check { - status shouldEqual StatusCodes.NotFound - val body = responseAs[String] - body should include("Type: https://ignifyr.io/errors/ResourceNotFound") - body should include("No job with id x") - } - } - - "pass a request timeout through as 408" in { - Get("/boom") ~> routeThrowing(RequestTimeout("Timed out", "The FHIR server did not answer")) ~> check { - status shouldEqual StatusCodes.RequestTimeout - responseAs[String] should include("Type: https://ignifyr.io/errors/RequestTimeout") - } - } - - // onFHIR's definitions layer raises its own error type; a bad request from it must stay a 400 rather - // than being reported to the user as an internal failure. - "translate an onFHIR bad request into a 400" in { - Get("/boom") ~> routeThrowing(model.BadRequest("Invalid profile", "Profile url is malformed", None)) ~> check { - status shouldEqual StatusCodes.BadRequest - val body = responseAs[String] - body should include("Type: https://ignifyr.io/errors/BadRequest") - body should include("Profile url is malformed") - } - } - - "answer with 500 when an exception carries no message" in { - Get("/boom") ~> routeThrowing(new RuntimeException()) ~> check { - status shouldEqual StatusCodes.InternalServerError - responseAs[String] should include("Type: https://ignifyr.io/errors/InternalError") - } - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrRejectionHandlerTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrRejectionHandlerTest.scala deleted file mode 100644 index 74627d61..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/IgnifyrRejectionHandlerTest.scala +++ /dev/null @@ -1,66 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{ContentTypes, StatusCodes} -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.model.Project -import org.json4s.jackson.Serialization.writePretty - -class IgnifyrRejectionHandlerTest extends BaseEndpointTest { - val project1: Project = Project(name = "example", description = Some("example project")) - - "IgnifyrRejectionHandler" should { - - "create an HTTP response with unsupported media type status for invalid content type" in { - // Send a POST request without changing the contentType to application/json - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}", - akka.http.scaladsl.model.HttpEntity.apply(writePretty(project1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.UnsupportedMediaType - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/UnsupportedMediaType") - } - } - - "create an HTTP response with bad request status for content not complying with the data model" in { - // Send a POST request with a project without name - val projectWithoutUrl = - Map("url" -> "http://example3.com/example-project", "description" -> Some("example project")) - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(projectWithoutUrl)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/BadRequest") - response should include("Detail: No usable value for name") - } - } - - "create an HTTP response with method not allowed status for requests initiated with an unsupported method" in { - // Send a PUT request to projects which is not applicable - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(project1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.MethodNotAllowed - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/MethodForbidden") - } - } - - "create an HTTP response with not found status for wrong URL" in { - // Send a POST request to an invalid URL - Post( - s"/${webServerConfig.baseUri}/invalidURL", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(project1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - val response = responseAs[String] - response should include("Type: https://ignifyr.io/errors/ResourceNotFound") - response should include(s"/${webServerConfig.baseUri}/invalidURL does not exist") - } - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobEndpointTest.scala deleted file mode 100644 index b24ca9bf..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobEndpointTest.scala +++ /dev/null @@ -1,331 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.actor.ActorSystem -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes} -import akka.http.scaladsl.testkit.RouteTestTimeout -import akka.testkit.TestDuration -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.util.TestUtil -import org.json4s.JArray -import org.json4s.JsonAST.{JBool, JString, JValue} -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -import scala.concurrent.duration.DurationInt - -class JobEndpointTest extends BaseEndpointTest { - // default timeout is 1 seconds, which is not enough for some tests - implicit def default(implicit system: ActorSystem): RouteTestTimeout = RouteTestTimeout( - new DurationInt(60).second.dilated(system) - ) - - // first job to be created - val sinkSettings: SinkSettings = - FileSystemSinkSettings(path = "http://example.com/fhir", contentType = SinkContentTypes.CSV) - val job1: FhirMappingJob = FhirMappingJob( - name = Some("mappingJob1"), - sourceSettings = Map.empty, - sinkSettings = sinkSettings, - mappings = Seq.empty, - dataProcessingSettings = DataProcessingSettings() - ) - // second job using kafka as a data source to be created - val kafkaSourceSettings: KafkaSourceSettings = KafkaSourceSettings( - name = "kafka-source", - sourceUri = "http://example.com/kafka", - bootstrapServers = "http://some-kafka-server:9092" - ) - val mappingJobSourceSettings: Map[String, MappingJobSourceSettings] = - Map("source1" -> kafkaSourceSettings) - val kafkaMappingTask: Seq[FhirMappingTask] = Seq( - FhirMappingTask( - "mappingTest1", - "mappingRef1", - Map( - "sourceBinding1" -> KafkaSource( - topicName = "topic", - sourceRef = Some("source1"), - options = Map("startingOffsets" -> "latest") - ) - ) - ) - ) - val kafkaSourceJob: FhirMappingJob = FhirMappingJob( - name = Some("mappingJob2"), - sourceSettings = mappingJobSourceSettings, - sinkSettings = sinkSettings, - mappings = kafkaMappingTask, - dataProcessingSettings = DataProcessingSettings() - ) - // a malformed job with a source reference to a missing data source in the mapping tasks, to be rejected - val malformedMappingTasks: Seq[FhirMappingTask] = Seq( - FhirMappingTask( - "mappingTest2", - "mappingRef1", - Map("sourceBinding1" -> SqlSource(tableName = Some("table"), sourceRef = Some("source2"))) - ) - ) - val mappingTaskMalformedJob: FhirMappingJob = FhirMappingJob( - name = Some("malformedJob1"), - sourceSettings = mappingJobSourceSettings, - sinkSettings = sinkSettings, - mappings = malformedMappingTasks, - dataProcessingSettings = DataProcessingSettings() - ) - // a malformed job which is a scheduling job and has a stream file system data source, to be rejected - val streamFileSystemSourceSettings: FileSystemSourceSettings = FileSystemSourceSettings( - name = "file-system-source", - sourceUri = "http://example.co/filesystem", - dataFolderPath = "test/data", - asStream = true - ) - val streamAndSchedulingMalformedJob: FhirMappingJob = FhirMappingJob( - name = Some("malformedJob2"), - schedulingSettings = Some(SchedulingSettings(cronExpression = "* * * * *")), - sourceSettings = Map( - "source1" -> streamFileSystemSourceSettings.copy(asStream = false), - "source2" -> streamFileSystemSourceSettings - ), - sinkSettings = sinkSettings, - mappings = Seq.empty, - dataProcessingSettings = DataProcessingSettings() - ) - - "The service" should { - - "create a job within project" in { - // create the first job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(job1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that job metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check job folder is created - FileUtils - .getPath(ignifyrEngineConfig.jobRepositoryFolderPath, projectId, s"${job1.id}${FileExtensions.JSON}") - .toFile should exist - } - - // create the second job - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(kafkaSourceJob)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that job metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 2 - // check job file is created - FileUtils - .getPath( - ignifyrEngineConfig.jobRepositoryFolderPath, - projectId, - s"${kafkaSourceJob.id}${FileExtensions.JSON}" - ) - .toFile should exist - } - } - - "get all jobs in a project" in { - // get all jobs - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved jobs - val jobs: Seq[FhirMappingJob] = JsonMethods.parse(responseAs[String]).extract[Seq[FhirMappingJob]] - jobs.length shouldEqual 2 - } - } - - "get a job in a project" in { - // get a job - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${job1.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved job - val job: FhirMappingJob = JsonMethods.parse(responseAs[String]).extract[FhirMappingJob] - job.name shouldEqual job1.name - } - // get a job with invalid id - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/123123" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "update a job in a project" in { - // update a job - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${job1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(job1.copy(name = Some("updatedJob")))) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the updated job - val job: FhirMappingJob = JsonMethods.parse(responseAs[String]).extract[FhirMappingJob] - job.name shouldEqual Some("updatedJob") - } - // update a job with invalid id - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/123123", - HttpEntity(ContentTypes.`application/json`, writePretty(job1.copy(id = "123123"))) - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "delete a job in a project" in { - // delete a job - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${job1.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.NoContent - // validate that job metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check job folder is deleted - FileUtils - .getPath(ignifyrEngineConfig.jobRepositoryFolderPath, projectId, s"${job1.id}${FileExtensions.JSON}") - .toFile shouldNot exist - } - // delete a job with invalid id - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/123123" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "get a job with kafka source type in a project" in { - // get a job - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${kafkaSourceJob.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // convert the JSON response to a JValue - val jsonResponse: JValue = JsonMethods.parse(responseAs[String]) - // check if the fields exist - (jsonResponse \ "name") should be(JString("mappingJob2")) - (jsonResponse \ "sourceSettings" \ "source1" \ "asStream") should be(JBool(true)) - } - } - - "prevent creation of jobs with malformed content" in { - // try to create a job with malformed mapping content - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(mappingTaskMalformedJob)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - // validate error message - val response = responseAs[String] - response should include( - "Detail: The data source referenced by source name 'source2' in the mapping task 'mappingTest2' is not found in the source settings of the job." - ) - // validate that job metadata file is still the same - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check job folder is not created - FileUtils - .getPath( - ignifyrEngineConfig.jobRepositoryFolderPath, - projectId, - s"${mappingTaskMalformedJob.id}${FileExtensions.JSON}" - ) - .toFile shouldNot exist - } - - // try to crate a job which is scheduling and has a stream data source - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(streamAndSchedulingMalformedJob)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - // validate error message - val response = responseAs[String] - response should include("Detail: Streaming jobs cannot be scheduled.") - // validate that job metadata file is still the same - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check job folder is not created - FileUtils - .getPath( - ignifyrEngineConfig.jobRepositoryFolderPath, - projectId, - s"${mappingTaskMalformedJob.id}${FileExtensions.JSON}" - ) - .toFile shouldNot exist - } - } - } - - "prevent creation of a job with two mappingTask share same name" in { - - val mappingTask: FhirMappingTask = FhirMappingTask("testname1", "mappingref1", sourceBinding = Map.empty) - val mappingTask2: FhirMappingTask = FhirMappingTask("testname1", "mappingref1", sourceBinding = Map.empty) - val jobWithSameMappingTaskName = job1.copy( - id = "jobWithSameMappingTaskName", - name = Option("jobWithSameMappingTaskName"), - mappings = Seq(mappingTask, mappingTask2) - ) - // try to create a job with two mappingTask share same name - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(jobWithSameMappingTaskName)) - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - // validate error message - val response = responseAs[String] - response should include( - "Detail: Duplicate MappingTask name(s) found: testname1. Each MappingTask must have a unique name." - ) - // validate that job metadata file is still the same - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check job folder is not created - FileUtils - .getPath( - ignifyrEngineConfig.jobRepositoryFolderPath, - projectId, - s"${jobWithSameMappingTaskName.id}${FileExtensions.JSON}" - ) - .toFile shouldNot exist - } - } - - /** - * Creates a project to be used in the tests - * */ - override def beforeAll(): Unit = { - super.beforeAll() - this.createProject() - } - -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobExecutionControlEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobExecutionControlEndpointTest.scala deleted file mode 100644 index e30e02e2..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/JobExecutionControlEndpointTest.scala +++ /dev/null @@ -1,110 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes} -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.server.BaseEndpointTest -import org.json4s.JArray -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -/** - * The execution-control half of the job API — status, list executions, stop, deschedule. Starting an - * execution needs a real source and a FHIR server (that is the long-tier `MappingExecutionEndpointTest`), - * but the answers for a job that is *not* running are pure registry lookups, and they are what the web UI - * polls on every job page. They also have to distinguish "no such job" (404) from "job exists, nothing - * running" (200 with an empty list) — collapsing those two is the mistake this suite guards against. - */ -class JobExecutionControlEndpointTest extends BaseEndpointTest { - - private val job: FhirMappingJob = FhirMappingJob( - name = Some("execution-control-job"), - sourceSettings = Map.empty, - sinkSettings = FileSystemSinkSettings(path = "./out", contentType = SinkContentTypes.CSV), - mappings = Seq.empty, - dataProcessingSettings = DataProcessingSettings() - ) - - private def jobUri(jobId: String = job.id): String = - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/$jobId" - - "The job execution control endpoints" should { - - // Note the quotes: the route stringifies the boolean before marshalling, so the body is the JSON - // string "false", not the JSON literal false. That is the contract the web UI parses. - "report a job that was never started as not running" in { - Get(s"${jobUri()}/${JobEndpoint.SEGMENT_STATUS}") ~> route ~> check { - status shouldEqual StatusCodes.OK - responseAs[String] shouldEqual "\"false\"" - } - } - - "report an unknown job as not running rather than failing" in { - // The status route consults only the running-job registry, so it does not resolve the job at all. - Get(s"${jobUri("no-such-job")}/${JobEndpoint.SEGMENT_STATUS}") ~> route ~> check { - status shouldEqual StatusCodes.OK - responseAs[String] shouldEqual "\"false\"" - } - } - - "return an empty execution list for a job that has never run" in { - Get(s"${jobUri()}/${JobEndpoint.SEGMENT_EXECUTIONS}") ~> route ~> check { - status shouldEqual StatusCodes.OK - JsonMethods.parse(responseAs[String]).asInstanceOf[JArray].arr shouldBe empty - } - } - - "return 404 when listing the executions of a job that does not exist" in { - Get(s"${jobUri("no-such-job")}/${JobEndpoint.SEGMENT_EXECUTIONS}") ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - // Stopping every execution of a job is idempotent on purpose: the UI offers it whenever a job page is - // open, and it must not fail just because nothing happens to be running. - "accept a request to stop all executions of a job that is not running" in { - Delete(s"${jobUri()}/${JobEndpoint.SEGMENT_EXECUTIONS}") ~> route ~> check { - status.isSuccess() shouldBe true - } - } - - "return 404 when stopping an execution that is not running" in { - Delete( - s"${jobUri()}/${JobEndpoint.SEGMENT_EXECUTIONS}/no-such-execution/${JobEndpoint.SEGMENT_STOP}" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "return 404 when stopping a mappingTask execution that is not running" in { - Delete( - s"${jobUri()}/${JobEndpoint.SEGMENT_EXECUTIONS}/no-such-execution/" + - s"${JobEndpoint.SEGMENT_MAPPINGS}/no-such-mapping/${JobEndpoint.SEGMENT_STOP}" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "return 404 when descheduling an execution that is not scheduled" in { - Delete( - s"${jobUri()}/${JobEndpoint.SEGMENT_EXECUTIONS}/no-such-execution/${JobEndpoint.SEGMENT_DESCHEDULE}" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - } - - /** - * Creates the project and the job the execution-control routes are exercised against. - * */ - override def beforeAll(): Unit = { - super.beforeAll() - this.createProject() - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(job)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingContextEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingContextEndpointTest.scala deleted file mode 100644 index 33c3bcb0..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingContextEndpointTest.scala +++ /dev/null @@ -1,132 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, Multipart, StatusCodes} -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.endpoint.{MappingContextEndpoint, ProjectEndpoint} -import io.ignifyr.server.util.{FileOperations, TestUtil} -import org.json4s.JArray -import org.json4s.jackson.JsonMethods - -import java.io.File - -class MappingContextEndpointTest extends BaseEndpointTest { - // first mapping context to be created - val mappingContext1: String = "mappingContext1" - // second mapping context to be created - val mappingContext2: String = "mappingContext2" - - "The service" should { - - "create a mapping context within project" in { - // create the first mapping context - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}", - HttpEntity(ContentTypes.`text/plain(UTF-8)`, mappingContext1) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that mapping context is updated in projects.json file - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingContexts") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check mapping context file is created - FileUtils - .getPath(ignifyrEngineConfig.mappingContextRepositoryFolderPath, projectId, mappingContext1) - .toFile should exist - } - // create the second mapping context - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}", - HttpEntity(ContentTypes.`text/plain(UTF-8)`, mappingContext2) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that mapping context is updated in projects.json file - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingContexts") - .asInstanceOf[JArray] - .arr - .length shouldEqual 2 - FileUtils - .getPath(ignifyrEngineConfig.mappingContextRepositoryFolderPath, projectId, mappingContext2) - .toFile should exist - } - } - - "get all mapping contexts within a project" in { - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - val mappingContexts = JsonMethods.parse(responseAs[String]).extract[List[String]] - mappingContexts.length shouldEqual 2 - mappingContexts.contains(mappingContext1) shouldEqual true - mappingContexts.contains(mappingContext2) shouldEqual true - } - } - - "delete a mapping context within a project" in { - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}/$mappingContext2" - ) ~> route ~> check { - status shouldEqual StatusCodes.NoContent - // validate that mapping context is updated in projects.json file - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingContexts") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check mapping context file is deleted - FileUtils - .getPath(ignifyrEngineConfig.mappingContextRepositoryFolderPath, projectId, mappingContext2) - .toFile shouldNot exist - } - } - - "upload a csv content to a mapping context" in { - // get file from resources - val file: File = - FileOperations.getFileIfExists(getClass.getResource("/test-mapping-context/sample-unit-conversion.csv").getPath) - val fileData = Multipart.FormData.BodyPart.fromPath("attachment", ContentTypes.`text/plain(UTF-8)`, file.toPath) - val formData = Multipart.FormData(fileData) - // save a csv file to mapping context - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}/$mappingContext1/${MappingContextEndpoint.SEGMENT_FILE}", - formData.toEntity() - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - responseAs[String] shouldEqual "OK" - Thread.sleep(5000) - } - } - - "download csv content of a mapping context" in { - // download csv content of the mapping context - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingContextEndpoint.SEGMENT_CONTEXTS}/$mappingContext1/${MappingContextEndpoint.SEGMENT_FILE}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns the csv content - val csvContent: String = responseAs[String] - // remove new lines to compare without system specific line endings - csvContent.replaceAll("\\r", "").replaceAll("\\n", "") shouldEqual - "source_code,source_unit,target_unit,conversion_function" + - "10839-9,\"ng/ml\",\"ng/mL\",\"$this\"" + - "14627-4,\"mmol/L\",\"mmol/L\",\"$this\"" + - "1742-6,\"UI/L\",\"U/L\",\"$this\"" + - "1920-8,\"UI/L\",\"U/L\",\"$this\"" - } - } - } - - /** - * Creates a project to be used in the tests - * */ - override def beforeAll(): Unit = { - super.beforeAll() - this.createProject() - } - -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingEndpointTest.scala deleted file mode 100644 index 66c0c586..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MappingEndpointTest.scala +++ /dev/null @@ -1,243 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes} -import io.ignifyr.engine.model.SinkContentTypes -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.server.util.FileOperations -import io.ignifyr.engine.model.{ - FhirMapping, - FhirMappingContextDefinition, - FhirMappingJob, - FhirMappingTask, - SinkSettings, - FileSystemSinkSettings -} -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.util.TestUtil -import org.json4s.JArray -import org.json4s.jackson.JsonMethods -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.server.endpoint.{JobEndpoint, MappingEndpoint, ProjectEndpoint} -import org.json4s.jackson.Serialization.writePretty - -class MappingEndpointTest extends BaseEndpointTest { - - // first mapping to be created - val mapping1: FhirMapping = FhirMapping( - id = "mapping1", - url = "http://example.com/mapping1", - name = "mapping1", - source = Seq.empty, - context = Map( - "practitionerConceptMap" -> FhirMappingContextDefinition( - category = "concept-map", - url = Some("$CONTEXT_REPO/pilot1/practitioner-concept-map.csv"), - value = None - ) - ), - mapping = Seq.empty - ) - // second mapping to be created - val mapping2: FhirMapping = FhirMapping( - id = "mapping2", - url = "http://example.com/mapping2", - name = "mapping2", - source = Seq.empty, - context = Map.empty, - mapping = Seq.empty - ) - // job using mapping2 - val sinkSettings: SinkSettings = - FileSystemSinkSettings(path = "http://example.com/fhir", contentType = SinkContentTypes.CSV) - val job: FhirMappingJob = FhirMappingJob( - name = Some("mappingJob"), - sourceSettings = Map.empty, - sinkSettings = sinkSettings, - mappings = Seq(FhirMappingTask(name = mapping2.id, mappingRef = mapping2.url, sourceBinding = Map.empty)) - ) - - "The service" should { - - "create a mapping within project" in { - // create the first mapping - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that mapping metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappings") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check mapping folder is created - FileUtils - .getPath(ignifyrEngineConfig.mappingRepositoryFolderPath, projectId, s"${mapping1.id}${FileExtensions.JSON}") - .toFile should exist - } - // create the second mapping - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping2)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that mapping metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappings") - .asInstanceOf[JArray] - .arr - .length shouldEqual 2 - FileUtils - .getPath(ignifyrEngineConfig.mappingRepositoryFolderPath, projectId, s"${mapping2.id}${FileExtensions.JSON}") - .toFile should exist - } - } - - "get all mappings in a project" in { - // get all mappings within a project - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns two mappings - val mappings: Seq[FhirMapping] = JsonMethods.parse(responseAs[String]).extract[Seq[FhirMapping]] - mappings.length shouldEqual 2 - } - } - - "get a mapping in a project" in { - // get a mapping - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/${mapping1.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved mapping - val mapping: FhirMapping = JsonMethods.parse(responseAs[String]).extract[FhirMapping] - mapping.url shouldEqual mapping1.url - mapping.name shouldEqual mapping1.name - // validate that the url of concept map is preserved - mapping.context.size shouldEqual 1 - mapping - .context("practitionerConceptMap") - .url - .get shouldEqual "$CONTEXT_REPO/pilot1/practitioner-concept-map.csv" - } - // get a mapping with invalid id - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/123123" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "update a mapping in a project" in { - // update a mapping - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/${mapping1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping1.copy(url = "http://example.com/mapping3"))) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned mapping includes the update - val mapping: FhirMapping = JsonMethods.parse(responseAs[String]).extract[FhirMapping] - mapping.url shouldEqual "http://example.com/mapping3" - // validate that mapping metadata is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - ( - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappings") - .asInstanceOf[JArray] - .arr - .find(m => (m \ "id").extract[String].contentEquals(mapping1.id)) - .get \ "url" - ) - .extract[String] shouldEqual "http://example.com/mapping3" - } - // update a mapping with invalid id - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/123123", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping1.copy(id = "123123"))) - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - // update a mapping with existing url - Put( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/${mapping1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping1.copy(url = mapping2.url))) - ) ~> route ~> check { - status shouldEqual StatusCodes.Conflict - } - } - - "delete a mapping from a project" in { - // delete a mapping - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/${mapping1.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.NoContent - // validate that mapping metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappings") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check mapping folder is deleted - FileUtils - .getPath(ignifyrEngineConfig.mappingRepositoryFolderPath, projectId, s"${mapping1.id}${FileExtensions.JSON}") - .toFile shouldNot exist - } - // delete a mapping with invalid id - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/123123" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "cannot delete a mapping from a project if it is referenced by some jobs" in { - // create a job using mapping1 - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(job)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // check job folder is created - FileUtils - .getPath(ignifyrEngineConfig.jobRepositoryFolderPath, projectId, s"${job.id}${FileExtensions.JSON}") - .toFile should exist - } - // delete a mapping - Delete( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}/${mapping2.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.BadRequest - } - } - - "cannot create a mapping with the existing ID or URL" in { - // create the mapping with existing id - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping2)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Conflict - } - // create the mapping with existing url - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${MappingEndpoint.SEGMENT_MAPPINGS}", - HttpEntity(ContentTypes.`application/json`, writePretty(mapping2.copy(id = "id-updated"))) - ) ~> route ~> check { - status shouldEqual StatusCodes.Conflict - } - } - - } - - /** - * Creates a project to be used in the tests - * */ - override def beforeAll(): Unit = { - super.beforeAll() - this.createProject() - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MetadataEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MetadataEndpointTest.scala deleted file mode 100644 index df6b97e1..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/MetadataEndpointTest.scala +++ /dev/null @@ -1,38 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.StatusCodes -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.endpoint.MetadataEndpoint -import io.ignifyr.server.model.Metadata -import org.json4s.jackson.JsonMethods -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.engine.config.IgnifyrConfig - -import java.io.File - -class MetadataEndpointTest extends BaseEndpointTest { - - "Metadata endpoint" should { - - "return metadata information" in { - Get(s"/${webServerConfig.baseUri}/${MetadataEndpoint.SEGMENT_METADATA}") ~> route ~> check { - status shouldEqual StatusCodes.OK - val metadata = JsonMethods.parse(responseAs[String]).extract[Metadata] - - metadata.name shouldEqual "Ignifyr" - metadata.description shouldEqual "Ignifyr is a tool for mapping data from various sources to FHIR resources." - metadata.fhirDefinitionsVersion shouldEqual "R4" - metadata.repositoryNames.mappings shouldEqual "mappings" - metadata.repositoryNames.schemas shouldEqual "schemas" - metadata.repositoryNames.contexts shouldEqual "mapping-contexts" - metadata.repositoryNames.jobs shouldEqual "mapping-jobs" - metadata.repositoryNames.terminologySystems shouldEqual "terminology-systems" - metadata.archiving.erroneousRecordsFolder shouldEqual s"${IgnifyrConfig.engineConfig.contextPath}${File.separator}erroneous-records-folder" - metadata.archiving.archiveFolder shouldEqual s"${IgnifyrConfig.engineConfig.contextPath}${File.separator}archive-folder" - metadata.archiving.streamArchivingFrequency shouldEqual 5000 - } - } - - } - -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ProjectEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ProjectEndpointTest.scala deleted file mode 100644 index 42b441f3..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ProjectEndpointTest.scala +++ /dev/null @@ -1,149 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model.{ContentTypes, StatusCodes} -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.model.{Project, ProjectEditableFields} -import io.ignifyr.server.util.TestUtil -import org.json4s.JArray -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -class ProjectEndpointTest extends BaseEndpointTest { - // first project to be created - val project1: Project = Project( - name = "example", - description = Some("example project"), - schemaUrlPrefix = Some("https://example.com/StructureDefinition/"), - mappingUrlPrefix = Some("https://example.com/mappings/") - ) - // second project to be created - val project2: Project = Project(name = "second example", description = Some("second example project")) - // patch to be applied to the existing project - val projectPatch: (String, String) = ProjectEditableFields.DESCRIPTION -> "updated description" - // schema definition - val schemaDefinition: SchemaDefinition = SchemaDefinition( - id = "id", - version = SchemaDefinition.VERSION_LATEST, - url = "https://example.com/fhir/StructureDefinition/schema", - `type` = "Ty", - name = "name", - description = Some("description"), - rootDefinition = None, - fieldDefinitions = None - ) - - "The service" should { - - "create a project" in { - // create the first project - // note that in the initialization of database, a dummy project is already created due to the schemas defined in test resources - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(project1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that projects metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - projects.arr.length shouldEqual 1 - } - // create the second project - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(project2)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that projects metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - projects.arr.length shouldEqual 2 - } - } - - "get all projects" in { - // retrieve all projects - Get(s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}") ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns two projects - val projects: Seq[Project] = JsonMethods.parse(responseAs[String]).extract[Seq[Project]] - projects.length shouldEqual 2 - } - } - - "get a project" in { - // get a project - Get(s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/${project1.id}") ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved project - val project: Project = JsonMethods.parse(responseAs[String]).extract[Project] - project.id shouldEqual project1.id - project.name shouldEqual project1.name - project.schemaUrlPrefix shouldEqual project1.schemaUrlPrefix - project.mappingUrlPrefix shouldEqual project1.mappingUrlPrefix - } - // get a project with invalid id - Get(s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/123123") ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "patch a project" in { - // patch a project - Patch( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/${project1.id}", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(projectPatch)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned project includes the update - val project: Project = JsonMethods.parse(responseAs[String]).extract[Project] - project.description.get should not equal project1.description.get - // validate that projects metadata is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == project1.id).get \ "description") - .extract[String] shouldEqual "updated description" - } - // patch a project with invalid id - Patch( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/123123", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(projectPatch)) - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "delete a project" in { - // first create a schema to trigger creation of the project folder under the schemas folder - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/${project1.id}/${SchemaDefinitionEndpoint.SEGMENT_SCHEMAS}", - akka.http.scaladsl.model.HttpEntity.apply(ContentTypes.`application/json`, writePretty(schemaDefinition)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that projects metadata file is updated for the first project - val firstProject = TestUtil - .getProjectJsonFile(ignifyrEngineConfig) - .arr - .find(p => (p \ "id").extract[String].contentEquals(project1.id)) - .get // find first project - (firstProject \ "schemas").asInstanceOf[JArray].arr.length shouldEqual 1 - // validate the project folder has been created within the schemas - FileUtils.getPath(ignifyrEngineConfig.schemaRepositoryFolderPath, project1.id).toFile should exist - } - - // delete a project - Delete(s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/${project1.id}") ~> route ~> check { - status shouldEqual StatusCodes.NoContent - // validate that projects metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - projects.arr.length shouldEqual 1 - - // validate the project file has been deleted under the schemas folder - FileUtils.getPath(ignifyrEngineConfig.schemaRepositoryFolderPath, project1.id).toFile shouldNot exist - } - // delete a non-existent project - Delete(s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/${project1.id}") ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ReloadingEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ReloadingEndpointTest.scala deleted file mode 100644 index 68e29428..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/ReloadingEndpointTest.scala +++ /dev/null @@ -1,58 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.actor.ActorSystem -import akka.http.scaladsl.model.StatusCodes -import akka.http.scaladsl.testkit.RouteTestTimeout -import akka.testkit.TestDuration -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.model.Project -import io.onfhir.definitions.common.model.Json4sSupport.formats -import io.ignifyr.server.repository.project.ProjectFolderRepository -import io.ignifyr.server.util.FileOperations -import org.json4s.{JArray, JObject, JString} -import org.json4s.jackson.{JsonMethods, Serialization} - -import java.io.FileWriter -import scala.concurrent.duration.DurationInt - -class ReloadingEndpointTest extends BaseEndpointTest { - - // default timeout is 1 seconds, which is not enough for this test - implicit def default(implicit system: ActorSystem): RouteTestTimeout = RouteTestTimeout( - new DurationInt(10).second.dilated(system) - ) - - "The service" should { - - "should reload projects successfully after updating project file" in { - // Read projects and update description fields from projects.json - val projectsFile = FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile - val parsedProjects = - FileOperations.readFileIntoJson(projectsFile).asInstanceOf[JArray].arr.map(p => p.asInstanceOf[JObject]) - val reloadedProjects = parsedProjects.map(project => { - project.mapField { - case ("description", _) => ("description", JString("reloaded")) - case otherwise => otherwise - } - }) - val fw = new FileWriter(projectsFile) - try fw.write(Serialization.writePretty(reloadedProjects)) - finally fw.close() - - // Trigger reload endpoint - Get(s"/${webServerConfig.baseUri}/${ReloadEndpoint.SEGMENT_RELOAD}") ~> route ~> check { - status shouldEqual StatusCodes.NoContent - - // Check project endpoint for whether project descriptions are updated - Get(s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}") ~> route ~> check { - status shouldEqual StatusCodes.OK - val projects: Seq[Project] = JsonMethods.parse(responseAs[String]).extract[Seq[Project]] - projects.foreach(project => { - project.description.get shouldEqual "reloaded" - }) - } - } - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpointTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpointTest.scala deleted file mode 100644 index 4529b835..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/endpoint/TerminologyServiceManagerEndpointTest.scala +++ /dev/null @@ -1,535 +0,0 @@ -package io.ignifyr.server.endpoint - -import akka.http.scaladsl.model._ -import akka.http.scaladsl.model.headers.RawHeader -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FhirMappingJobFormatter.formats -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.engine.util.FileUtils.FileExtensions -import io.ignifyr.server.BaseEndpointTest -import io.ignifyr.server.model.TerminologySystem -import io.ignifyr.server.model.TerminologySystem.{TerminologyCodeSystem, TerminologyConceptMap} -import io.ignifyr.server.repository.terminology.TerminologySystemFolderRepository.getTerminologySystemsJsonPath -import io.ignifyr.server.util.{FileOperations, TestUtil} -import org.json4s.JArray -import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.writePretty - -import java.io.File - -class TerminologyServiceManagerEndpointTest extends BaseEndpointTest { - var conceptMap1: TerminologyConceptMap = - ConceptMapFile(name = "testCM", conceptMapUrl = "", sourceValueSetUrl = "", targetValueSetUrl = "") - var conceptMap2: TerminologyConceptMap = - ConceptMapFile(name = "testCM2", conceptMapUrl = "", sourceValueSetUrl = "", targetValueSetUrl = "") - - var codeSystem1: TerminologyCodeSystem = CodeSystemFile(name = "testCS", codeSystem = "") - var codeSystem2: TerminologyCodeSystem = CodeSystemFile(name = "testCS2", codeSystem = "") - - var terminologySystem1: TerminologySystem = TerminologySystem( - name = "testTerminology1", - description = "example terminology 1", - codeSystems = Seq.empty, - conceptMaps = Seq.empty - ) - - val terminologySystem2: TerminologySystem = TerminologySystem( - name = "testTerminology2", - description = "example terminology 2", - codeSystems = Seq.empty, - conceptMaps = Seq.empty - ) - - // Create job for test update on job terminology - val sinkSettings: SinkSettings = - FileSystemSinkSettings(path = "http://example.com/fhir", contentType = SinkContentTypes.CSV) - val terminologyServiceSettings: TerminologyServiceSettings = LocalFhirTerminologyServiceSettings( - s"/${ignifyrEngineConfig.terminologySystemFolderPath}/${terminologySystem1.id}", - codeSystemFiles = Seq.empty, - conceptMapFiles = Seq.empty - ) - val jobTest: FhirMappingJob = FhirMappingJob( - name = Some("mappingJobTest"), - sourceSettings = Map.empty, - sinkSettings = sinkSettings, - terminologyServiceSettings = Some(terminologyServiceSettings), - mappings = Seq.empty, - dataProcessingSettings = DataProcessingSettings() - ) - - "The terminology service" should { - "create a terminology system" in { - // create the first terminology - Post( - s"/${webServerConfig.baseUri}/terminologies", - HttpEntity(ContentTypes.`application/json`, writePretty(terminologySystem1)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - val terminologySystem: TerminologySystem = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - // set the created terminology - terminologySystem1 = terminologySystem - // validate that a folder is created for the terminology - FileUtils.getPath(ignifyrEngineConfig.terminologySystemFolderPath, terminologySystem.id).toFile should exist - // validate that terminologies metadata file is updated - val terminologySystems = FileOperations.readJsonContent[TerminologySystem]( - FileUtils.getPath(getTerminologySystemsJsonPath(ignifyrEngineConfig.terminologySystemFolderPath)).toFile - ) - terminologySystems.length shouldEqual 1 - } - // create the second terminology - Post( - s"/${webServerConfig.baseUri}/terminologies", - HttpEntity(ContentTypes.`application/json`, writePretty(terminologySystem2)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that terminologies metadata file is updated - val terminologySystems = FileOperations.readJsonContent[TerminologySystem]( - FileUtils.getPath(getTerminologySystemsJsonPath(ignifyrEngineConfig.terminologySystemFolderPath)).toFile - ) - terminologySystems.length shouldEqual 2 - } - } - - "get all terminology systems" in { - // retrieve all terminologies - Get(s"/${webServerConfig.baseUri}/terminologies") ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns two terminologies - val terminologySystems: Seq[TerminologySystem] = - JsonMethods.parse(responseAs[String]).extract[Seq[TerminologySystem]] - terminologySystems.length shouldEqual 2 - } - } - - "get a terminology system" in { - // get a terminology system - Get(s"/${webServerConfig.baseUri}/terminologies/${terminologySystem1.id}") ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved terminology system - val terminologySystem: TerminologySystem = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - terminologySystem.id shouldEqual terminologySystem1.id - terminologySystem.name shouldEqual terminologySystem1.name - } - - // try to get a non-existent terminology system - Get(s"/${webServerConfig.baseUri}/terminologies/Non-existent") ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "put a terminology system" in { - // create a copy of the first terminology and update its name and check if it is updated - Put( - s"/${webServerConfig.baseUri}/terminologies/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(terminologySystem1.copy(name = "nameUpdated1"))) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned terminology system includes the update - val terminologySystem: TerminologySystem = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - terminologySystem.name shouldEqual "nameUpdated1" - // validate that terminology systems is updated - val terminologySystems = FileOperations.readJsonContent[TerminologySystem]( - FileUtils.getPath(getTerminologySystemsJsonPath(ignifyrEngineConfig.terminologySystemFolderPath)).toFile - ) - terminologySystems - .find(_.id == terminologySystem.id) - .get - .name shouldEqual "nameUpdated1" - terminologySystem1 = terminologySystem - } - } - - "delete a terminology system" in { - // delete a terminology system - Delete(s"/${webServerConfig.baseUri}/terminologies/${terminologySystem2.id}") ~> route ~> check { - status shouldEqual StatusCodes.NoContent - // validate that terminology folder is deleted - FileUtils.getPath(ignifyrEngineConfig.terminologySystemFolderPath, terminologySystem2.id).toFile shouldNot exist - // validate that terminology metadata file is updated - val terminologySystems = FileOperations.readJsonContent[TerminologySystem]( - FileUtils.getPath(getTerminologySystemsJsonPath(ignifyrEngineConfig.terminologySystemFolderPath)).toFile - ) - terminologySystems.length shouldEqual 1 - } - // delete a non-existent terminology system - Delete(s"/${webServerConfig.baseUri}/terminologies/${terminologySystem2.id}") ~> route ~> check { - status should not equal StatusCodes.OK - } - } - } - - "The terminology concept map service" should { - "create a job and reference to a terminology system" in { - Post( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}", - HttpEntity(ContentTypes.`application/json`, writePretty(jobTest)) - ) ~> route ~> check { - status shouldEqual StatusCodes.Created - // validate that job metadata file is updated - val projects: JArray = TestUtil.getProjectJsonFile(ignifyrEngineConfig) - (projects.arr.find(p => (p \ "id").extract[String] == projectId).get \ "mappingJobs") - .asInstanceOf[JArray] - .arr - .length shouldEqual 1 - // check job folder is created - FileUtils - .getPath(ignifyrEngineConfig.jobRepositoryFolderPath, projectId, s"${jobTest.id}${FileExtensions.JSON}") - .toFile should exist - } - } - - // add a concept map to the terminology system object - var updatedTerminology: TerminologySystem = - terminologySystem1.copy(conceptMaps = terminologySystem1.conceptMaps :+ conceptMap1) - - "create a concept map within a terminology system" in { - // create a concept map - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned concept map included in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.conceptMaps.length shouldEqual 1 - updatedTerminology.conceptMaps.last.name shouldEqual "testCM" - // validate that concept map file is created - FileUtils - .getPath( - ignifyrEngineConfig.terminologySystemFolderPath, - terminologySystem1.id, - updatedTerminology.conceptMaps.last.id - ) - .toFile should exist - } - // add a concept map to the terminology system object - updatedTerminology = updatedTerminology.copy(conceptMaps = updatedTerminology.conceptMaps :+ conceptMap2) - // create a second concept map within same terminology - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned concept map included in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.conceptMaps.length shouldEqual 2 - updatedTerminology.conceptMaps.head.name shouldEqual "testCM" - updatedTerminology.conceptMaps.last.name shouldEqual "testCM2" - // validate that concept map file is created - FileUtils - .getPath( - ignifyrEngineConfig.terminologySystemFolderPath, - terminologySystem1.id, - updatedTerminology.conceptMaps.last.id - ) - .toFile should exist - } - } - - "get concept maps within a terminology system" in { - // get a concept map list within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${ConceptMapEndpoint.SEGMENT_CONCEPT_MAPS}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns one concept maps - val conceptMaps: Seq[TerminologyConceptMap] = - JsonMethods.parse(responseAs[String]).extract[Seq[TerminologyConceptMap]] - conceptMaps.length shouldEqual 2 - } - } - - "get a concept map within a terminology system" in { - // get a concept map within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${ConceptMapEndpoint.SEGMENT_CONCEPT_MAPS}/${conceptMap1.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns the concept map - val conceptMap: TerminologyConceptMap = JsonMethods.parse(responseAs[String]).extract[TerminologyConceptMap] - conceptMap.name shouldEqual conceptMap1.name - } - - // try to get a non-existent concept map within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${ConceptMapEndpoint.SEGMENT_CONCEPT_MAPS}/Non-existent" - ) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "get the job in a project to verify terminology system is do not updated with new conceptMaps" in { - // get the job - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${jobTest.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved job includes new conceptMaps - val job: FhirMappingJob = JsonMethods.parse(responseAs[String]).extract[FhirMappingJob] - job.id shouldEqual jobTest.id - job.terminologyServiceSettings.get - .asInstanceOf[LocalFhirTerminologyServiceSettings] - .conceptMapFiles - .length shouldEqual 0 - } - } - - "put a concept map within a terminology system" in { - // update concept maps in the terminology system object - updatedTerminology = updatedTerminology.copy(conceptMaps = - updatedTerminology.conceptMaps.map(conceptMap => conceptMap.copy(name = conceptMap.name + "Updated")) - ) - // update concept maps within same terminology - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the updated conceptMaps included in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.conceptMaps.length shouldEqual 2 - updatedTerminology.conceptMaps.head.name shouldEqual "testCMUpdated" - updatedTerminology.conceptMaps.last.name shouldEqual "testCM2Updated" - conceptMap1 = updatedTerminology.conceptMaps.head - } - } - - "delete a concept map within a terminology system" in { - // delete a concept map in the terminology system object - updatedTerminology = updatedTerminology.copy(conceptMaps = Seq(updatedTerminology.conceptMaps.head)) - // delete the concept map within same terminology - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned only first concept map in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.conceptMaps.length shouldEqual 1 - updatedTerminology.conceptMaps.head.name shouldEqual "testCMUpdated" - // validate that concept map file is deleted - FileUtils - .getPath(ignifyrEngineConfig.terminologySystemFolderPath, terminologySystem1.id, conceptMap2.id) - .toFile shouldNot exist - } - } - - "upload a csv content as a concept map within a terminology system" in { - // get file from resources - val file: File = - FileOperations.getFileIfExists(getClass.getResource("/test-terminology-service/sample-concept-map.csv").getPath) - val fileData = Multipart.FormData.BodyPart.fromPath("attachment", ContentTypes.`text/plain(UTF-8)`, file.toPath) - val formData = Multipart.FormData(fileData) - // save a csv file as a concept map within a terminology system - Post( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${ConceptMapEndpoint.SEGMENT_CONCEPT_MAPS}/${conceptMap1.id}/${TerminologyServiceManagerEndpoint.SEGMENT_FILE}", - formData.toEntity() - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - responseAs[String] shouldEqual "OK" - Thread.sleep(5000) - } - } - - "download csv content of a concept map within a terminology system" in { - // download csv content of a concept map within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${ConceptMapEndpoint.SEGMENT_CONCEPT_MAPS}/${conceptMap1.id}/${TerminologyServiceManagerEndpoint.SEGMENT_FILE}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns the csv content - val csvContent: String = responseAs[String] - // remove new lines to compare without system specific line endings - csvContent - .replaceAll("\\r", "") - .replaceAll( - "\\n", - "" - ) shouldEqual "source_system,source_code,target_system,target_code,target_display,equivalence" + - "http://terminology.hl7.org/CodeSystem/v2-0487,ACNE,http://snomed.info/sct,309068002,Specimen from skin,equivalent" + - "http://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323008,Pus specimen,equivalent" + - "http://terminology.hl7.org/CodeSystem/v2-0487,ACNFLD,http://snomed.info/sct,119323009,Pus specimen 2,equivalent" + - "http://terminology.hl7.org/CodeSystem/v2-0487,BULLA,http://snomed.info/sct,258482009,Pus specimen 2,narrower" - } - } - } - - "The terminology code system service" should { - - // add a code system to the terminology system object - var updatedTerminology: TerminologySystem = - terminologySystem1.copy(codeSystems = terminologySystem1.codeSystems :+ codeSystem1) - - "create a code system within a terminology system" in { - // create a code system - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned code system included in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.codeSystems.length shouldEqual 1 - updatedTerminology.codeSystems.last.name shouldEqual "testCS" - // validate that code system file is created - FileUtils - .getPath( - ignifyrEngineConfig.terminologySystemFolderPath, - terminologySystem1.id, - updatedTerminology.codeSystems.last.id - ) - .toFile should exist - } - // add a code system to the terminology system object - updatedTerminology = updatedTerminology.copy(codeSystems = updatedTerminology.codeSystems :+ codeSystem2) - // create a second code system within same terminology - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned code system included in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.codeSystems.length shouldEqual 2 - updatedTerminology.codeSystems.head.name shouldEqual "testCS" - updatedTerminology.codeSystems.last.name shouldEqual "testCS2" - // validate that code system file is created - FileUtils - .getPath( - ignifyrEngineConfig.terminologySystemFolderPath, - terminologySystem1.id, - updatedTerminology.codeSystems.last.id - ) - .toFile should exist - } - } - - "get code systems within a terminology system" in { - // get a code system list within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${CodeSystemEndpoint.SEGMENT_CODE_SYSTEMS}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns one code systems - val codeSystems: Seq[TerminologyCodeSystem] = - JsonMethods.parse(responseAs[String]).extract[Seq[TerminologyCodeSystem]] - codeSystems.length shouldEqual 2 - } - } - - "get a code system within a terminology system" in { - // get a code system within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${CodeSystemEndpoint.SEGMENT_CODE_SYSTEMS}/${codeSystem1.id}" - ).addHeader(RawHeader("Content-Type", "application/json")) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns the code system - val codeSystem: TerminologyCodeSystem = JsonMethods.parse(responseAs[String]).extract[TerminologyCodeSystem] - codeSystem.name shouldEqual codeSystem1.name - } - - // try to get a non-existent code system within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${CodeSystemEndpoint.SEGMENT_CODE_SYSTEMS}/Non-existent" - ).addHeader(RawHeader("Content-Type", "application/json")) ~> route ~> check { - status shouldEqual StatusCodes.NotFound - } - } - - "get the job in a project to verify terminology system is do not updated with new codeSystems" in { - // get the job - Get( - s"/${webServerConfig.baseUri}/${ProjectEndpoint.SEGMENT_PROJECTS}/$projectId/${JobEndpoint.SEGMENT_JOB}/${jobTest.id}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate the retrieved job includes new codeSystems - val job: FhirMappingJob = JsonMethods.parse(responseAs[String]).extract[FhirMappingJob] - job.id shouldEqual jobTest.id - job.terminologyServiceSettings.get - .asInstanceOf[LocalFhirTerminologyServiceSettings] - .codeSystemFiles - .length shouldEqual 0 - } - } - - "put a code system within a terminology system" in { - // update code systems in the terminology system object - updatedTerminology = updatedTerminology.copy(codeSystems = - updatedTerminology.codeSystems.map(codeSystem => codeSystem.copy(name = codeSystem.name + "Updated")) - ) - // update code systems within same terminology - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the updated codeSystems included in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.codeSystems.length shouldEqual 2 - updatedTerminology.codeSystems.head.name shouldEqual "testCSUpdated" - updatedTerminology.codeSystems.last.name shouldEqual "testCS2Updated" - codeSystem1 = updatedTerminology.codeSystems.head - } - } - - "delete a code system within a terminology system" in { - // delete a code system in the terminology system object - updatedTerminology = updatedTerminology.copy(codeSystems = Seq(updatedTerminology.codeSystems.head)) - // delete the code system within same terminology - Put( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}", - HttpEntity(ContentTypes.`application/json`, writePretty(updatedTerminology)) - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that the returned only first code system in the terminologySystem - updatedTerminology = JsonMethods.parse(responseAs[String]).extract[TerminologySystem] - updatedTerminology.codeSystems.length shouldEqual 1 - updatedTerminology.codeSystems.head.name shouldEqual "testCSUpdated" - // validate that code system file is deleted - FileUtils - .getPath(ignifyrEngineConfig.terminologySystemFolderPath, terminologySystem1.id, codeSystem2.id) - .toFile shouldNot exist - } - } - - "upload a csv content as a code system within a terminology system" in { - // get file from resources - val file: File = - FileOperations.getFileIfExists(getClass.getResource("/test-terminology-service/sample-code-system.csv").getPath) - val fileData = Multipart.FormData.BodyPart.fromPath("attachment", ContentTypes.`text/plain(UTF-8)`, file.toPath) - val formData = Multipart.FormData(fileData) - // save a csv file as a concept map within a terminology system - Post( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${CodeSystemEndpoint.SEGMENT_CODE_SYSTEMS}/${codeSystem1.id}/${TerminologyServiceManagerEndpoint.SEGMENT_FILE}", - formData.toEntity() - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - responseAs[String] shouldEqual "OK" - Thread.sleep(5000) - } - } - - "download csv content of a code system within a terminology system" in { - // download csv content of a code system within a terminology system - Get( - s"/${webServerConfig.baseUri}/${TerminologyServiceManagerEndpoint.SEGMENT_TERMINOLOGY}/${terminologySystem1.id}/${CodeSystemEndpoint.SEGMENT_CODE_SYSTEMS}/${codeSystem1.id}/${TerminologyServiceManagerEndpoint.SEGMENT_FILE}" - ) ~> route ~> check { - status shouldEqual StatusCodes.OK - // validate that it returns the csv content - val csvContent: String = responseAs[String] - // remove new lines to compare without system specific line endings - csvContent.replaceAll("\\r", "").replaceAll("\\n", "") shouldEqual "code,display,fr,de" + - "309068002,Specimen from skin,Spécimen de peau,Probe von der Haut" + - "119323008,Pus specimen,Spécimen de pus,Eiterprobe" - } - } - } - - /** - * Creates a project to be used in the tests - * */ - override def beforeAll(): Unit = { - super.beforeAll() - this.createProject() - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/repository/FolderDBInitializerTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/repository/FolderDBInitializerTest.scala deleted file mode 100644 index faa9263e..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/repository/FolderDBInitializerTest.scala +++ /dev/null @@ -1,266 +0,0 @@ -package io.ignifyr.server.repository - -import io.onfhir.definitions.common.model.SchemaDefinition -import io.ignifyr.engine.model.{FhirMapping, FhirMappingJob, FhirMappingSource, FhirRepositorySinkSettings} -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.model.Project -import io.ignifyr.server.repository.job.JobFolderRepository -import io.ignifyr.server.repository.mapping.ProjectMappingFolderRepository -import io.ignifyr.server.repository.mappingContext.MappingContextFolderRepository -import io.ignifyr.server.repository.project.ProjectFolderRepository -import io.ignifyr.server.repository.schema.SchemaFolderRepository -import org.mockito.ArgumentCaptor -import org.mockito.MockitoSugar._ -import org.scalatest.BeforeAndAfterEach -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import scala.concurrent.Future - -/** - * Server startup: the project index is either read back from `projects.json` or, when that file is gone, - * rebuilt by scanning the repository folders. Both paths run before the first request is served, so a - * failure here is "the server does not start" rather than a failing call — and the rebuild path is the - * only recovery there is if the index file is ever lost. - * - * The repositories are mocked so the initializer's own resolution logic is what is under test, not the - * folder repositories it delegates to. - */ -class FolderDBInitializerTest extends AnyFlatSpec with Matchers with BeforeAndAfterEach { - - private val projectId = "project-1" - - private val schema = SchemaDefinition( - id = "schema-1", - url = "https://ignifyr.io/fhir/StructureDefinition/Ext-patient", - version = SchemaDefinition.VERSION_LATEST, - `type` = "Ext-patient", - name = "ext-patient", - description = None, - rootDefinition = None, - fieldDefinitions = None - ) - - private val mapping = FhirMapping( - id = "mapping-1", - url = "https://ignifyr.io/fhir/mappings/patient-mapping", - name = "patient-mapping", - source = Seq(FhirMappingSource(alias = "source", url = schema.url)), - context = Map.empty, - mapping = Seq.empty - ) - - private val job = FhirMappingJob( - id = "job-1", - sourceSettings = Map.empty, - sinkSettings = FhirRepositorySinkSettings(fhirRepoUrl = "http://localhost/fhir"), - mappings = Seq.empty - ) - - private def projectsJsonFile = FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile - - private def writeProjectsJson(content: String): Unit = { - projectsJsonFile.getParentFile.mkdirs() - Files.write(projectsJsonFile.toPath, content.getBytes(StandardCharsets.UTF_8)) - } - - override def beforeEach(): Unit = if (projectsJsonFile.exists()) projectsJsonFile.delete() - override def afterEach(): Unit = if (projectsJsonFile.exists()) projectsJsonFile.delete() - - /** - * Builds an initializer over mocked repositories, returning it with the project repository to verify. - * The `*ById` maps stub what each repository answers for a resource id the index file references; an - * id with no entry resolves to None, which is the "file is gone" case. - */ - private def initializerOver( - schemas: Map[String, Seq[SchemaDefinition]] = Map.empty, - mappings: Map[String, Seq[FhirMapping]] = Map.empty, - jobs: Map[String, Seq[FhirMappingJob]] = Map.empty, - contexts: Map[String, Seq[String]] = Map.empty, - schemasById: Map[String, Option[SchemaDefinition]] = Map.empty, - mappingsById: Map[String, Option[FhirMapping]] = Map.empty, - jobsById: Map[String, Option[FhirMappingJob]] = Map.empty - ): (FolderDBInitializer, ProjectFolderRepository) = { - val projectRepository = mock[ProjectFolderRepository] - - val schemaRepository = mock[SchemaFolderRepository] - when(schemaRepository.getProjectPairs).thenReturn(schemas) - schemasById.foreach { case (id, answer) => - when(schemaRepository.getSchema(projectId, id)).thenReturn(Future.successful(answer)) - } - - val mappingRepository = mock[ProjectMappingFolderRepository] - when(mappingRepository.getProjectPairs).thenReturn(mappings) - mappingsById.foreach { case (id, answer) => - when(mappingRepository.getMapping(projectId, id)).thenReturn(Future.successful(answer)) - } - - val jobRepository = mock[JobFolderRepository] - when(jobRepository.getProjectPairs).thenReturn(jobs) - jobsById.foreach { case (id, answer) => - when(jobRepository.getJob(projectId, id)).thenReturn(Future.successful(answer)) - } - - val contextRepository = mock[MappingContextFolderRepository] - when(contextRepository.getProjectPairs).thenReturn(contexts) - - ( - new FolderDBInitializer(projectRepository, schemaRepository, mappingRepository, jobRepository, contextRepository), - projectRepository - ) - } - - /** The projects the initializer handed to the repository. */ - private def injectedProjects(projectRepository: ProjectFolderRepository): Map[String, Project] = { - val captor: ArgumentCaptor[Map[String, Project]] = - ArgumentCaptor.forClass(classOf[Map[String, Project]]) - verify(projectRepository).setProjects(captor.capture()) - captor.getValue - } - - "init" should "rebuild the project index from the repository folders when there is no index file" in { - val (initializer, projectRepository) = initializerOver( - schemas = Map(projectId -> Seq(schema)), - mappings = Map(projectId -> Seq(mapping)), - jobs = Map(projectId -> Seq(job)), - contexts = Map(projectId -> Seq("unit-conversion")) - ) - initializer.init() - - val projects = injectedProjects(projectRepository) - projects.keySet shouldBe Set(projectId) - val project = projects(projectId) - project.schemas.map(_.id) shouldBe Seq("schema-1") - project.mappings.map(_.id) shouldBe Seq("mapping-1") - project.mappingJobs.map(_.id) shouldBe Seq("job-1") - project.mappingContexts shouldBe Seq("unit-conversion") - } - - it should "write the index file it did not find, so the rebuild happens only once" in { - val (initializer, _) = initializerOver() - projectsJsonFile should not(exist) - initializer.init() - projectsJsonFile should exist - } - - // With no index file there is no project name either, so the folder name stands in for it. - it should "name a rebuilt project after its folder and derive the url prefixes from its resources" in { - val (initializer, projectRepository) = initializerOver( - schemas = Map(projectId -> Seq(schema)), - mappings = Map(projectId -> Seq(mapping)) - ) - initializer.init() - - val project = injectedProjects(projectRepository)(projectId) - project.name shouldBe projectId - project.schemaUrlPrefix shouldBe Some("https://ignifyr.io/fhir/StructureDefinition/") - project.mappingUrlPrefix shouldBe Some("https://ignifyr.io/fhir/mappings/") - } - - it should "collect a project that owns only some of the resource kinds" in { - val (initializer, projectRepository) = initializerOver( - schemas = Map("only-schemas" -> Seq(schema)), - jobs = Map("only-jobs" -> Seq(job)) - ) - initializer.init() - - val projects = injectedProjects(projectRepository) - projects.keySet shouldBe Set("only-schemas", "only-jobs") - projects("only-schemas").mappingJobs shouldBe empty - projects("only-jobs").schemas shouldBe empty - } - - it should "resolve the resources referenced by an existing index file" in { - writeProjectsJson(s"""[{ - | "id": "$projectId", - | "name": "Example project", - | "description": "an example", - | "schemaUrlPrefix": "https://ignifyr.io/fhir/StructureDefinition/", - | "mappingUrlPrefix": "https://ignifyr.io/fhir/mappings/", - | "mappingContexts": ["unit-conversion"], - | "schemas": [{"id": "schema-1"}], - | "mappings": [{"id": "mapping-1"}], - | "mappingJobs": [{"id": "job-1"}] - |}]""".stripMargin) - - val (initializer, projectRepository) = initializerOver( - schemasById = Map("schema-1" -> Some(schema)), - mappingsById = Map("mapping-1" -> Some(mapping)), - jobsById = Map("job-1" -> Some(job)) - ) - initializer.init() - - val project = injectedProjects(projectRepository)(projectId) - project.name shouldBe "Example project" - project.description shouldBe Some("an example") - project.schemas.map(_.id) shouldBe Seq("schema-1") - project.mappings.map(_.id) shouldBe Seq("mapping-1") - project.mappingJobs.map(_.id) shouldBe Seq("job-1") - project.mappingContexts shouldBe Seq("unit-conversion") - } - - it should "read an index file that lists a project with no resources" in { - writeProjectsJson(s"""[{ - | "id": "$projectId", "name": "Empty", "mappingContexts": [], - | "schemas": [], "mappings": [], "mappingJobs": [] - |}]""".stripMargin) - - val (initializer, projectRepository) = initializerOver() - initializer.init() - - val project = injectedProjects(projectRepository)(projectId) - project.schemas shouldBe empty - project.description shouldBe None - } - - /* - * The realistic corruption: somebody deletes a mapping file by hand, leaving the index pointing at it. - * The initializer refuses to start rather than silently serving a project with a hole in it — the - * failure must name the id so the operator can find the missing file. - */ - it should "refuse to start when the index references a mapping that is not on disk" in { - writeProjectsJson(s"""[{ - | "id": "$projectId", "name": "Example", "mappingContexts": [], - | "schemas": [], "mappings": [{"id": "mapping-gone"}], "mappingJobs": [] - |}]""".stripMargin) - - val (initializer, _) = initializerOver(mappingsById = Map("mapping-gone" -> None)) - val thrown = the[IllegalStateException] thrownBy initializer.init() - thrown.getMessage should include("mapping-gone") - } - - it should "refuse to start when the index references a schema that is not on disk" in { - writeProjectsJson(s"""[{ - | "id": "$projectId", "name": "Example", "mappingContexts": [], - | "schemas": [{"id": "schema-gone"}], "mappings": [], "mappingJobs": [] - |}]""".stripMargin) - - val (initializer, _) = initializerOver(schemasById = Map("schema-gone" -> None)) - val thrown = the[IllegalStateException] thrownBy initializer.init() - thrown.getMessage should include("schema-gone") - } - - it should "refuse to start when the index references a job that is not on disk" in { - writeProjectsJson(s"""[{ - | "id": "$projectId", "name": "Example", "mappingContexts": [], - | "schemas": [], "mappings": [], "mappingJobs": [{"id": "job-gone"}] - |}]""".stripMargin) - - val (initializer, _) = initializerOver(jobsById = Map("job-gone" -> None)) - val thrown = the[IllegalStateException] thrownBy initializer.init() - thrown.getMessage should include("job-gone") - } - - "removeProjectsJsonFile" should "delete the index file, and do nothing when it is already gone" in { - val (initializer, _) = initializerOver() - writeProjectsJson("[]") - projectsJsonFile should exist - - initializer.removeProjectsJsonFile() - projectsJsonFile should not(exist) - - noException should be thrownBy initializer.removeProjectsJsonFile() - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/service/ExecutionServiceTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/service/ExecutionServiceTest.scala deleted file mode 100644 index 0e748d2a..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/service/ExecutionServiceTest.scala +++ /dev/null @@ -1,138 +0,0 @@ -package io.ignifyr.server.service - -import com.typesafe.config.ConfigFactory -import io.ignifyr.engine.config.{IgnifyrConfig, IgnifyrEngineConfig} -import io.ignifyr.engine.model._ -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.model.ExecuteJobTask -import io.ignifyr.server.repository.job.JobFolderRepository -import io.ignifyr.server.repository.mapping.ProjectMappingFolderRepository -import io.ignifyr.server.repository.schema.SchemaFolderRepository -import org.apache.commons.io -import org.apache.spark.sql.types.StructType -import org.mockito.MockitoSugar._ -import org.scalatest.BeforeAndAfterAll -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AsyncWordSpec - -import java.io.{File, FileOutputStream} -import java.nio.file.{Files, Paths} -import scala.concurrent.Future - -class ExecutionServiceTest extends AsyncWordSpec with Matchers with BeforeAndAfterAll { - - // Name of the folder to keep test data to run file mapping job - private val testDataFolder: String = "test-data" - // Ignifyr engine config - val ignifyrEngineConfig: IgnifyrEngineConfig = new IgnifyrEngineConfig(ConfigFactory.load().getConfig("ignifyr")) - // FhirMappingJob definition with streaming file source - val testJob: FhirMappingJob = FhirMappingJob( - name = Some("testJob"), - sourceSettings = Map( - "test" -> FileSystemSourceSettings( - name = "test-source", - sourceUri = "test-source", - dataFolderPath = testDataFolder, - asStream = true - ) - ), - sinkSettings = FileSystemSinkSettings(path = "http://example.com/fhir", contentType = SinkContentTypes.CSV), - mappings = Seq.apply( - FhirMappingTask( - name = "patient-mapping", - mappingRef = "https://aiccelerate.eu/fhir/mappings/patient-mapping", - sourceBinding = Map("source" -> FileSystemSource(path = ".", contentType = SourceContentTypes.CSV)) - ) - ), - dataProcessingSettings = DataProcessingSettings(archiveMode = ArchiveModes.OFF) - ) - // Job execution task to clear checkpoint folder - val testExecuteJobTask: ExecuteJobTask = ExecuteJobTask(clearCheckpoints = true, Option.empty) - // Mock repositories - val mappingRepository: ProjectMappingFolderRepository = getMockMappingRepository - val schemaRepository: SchemaFolderRepository = getMockSchemaRepository - val mappingJobRepository: JobFolderRepository = getMockMappingJobRepository - // the execution service instance for the test - val executionService: ExecutionService = - new ExecutionService(mappingJobRepository, mappingRepository, schemaRepository) - - "The Execution Service" should { - "should clear checkpoint directory" in { - // create an example file for the mapping job in the corresponding checkpoint directory - val path: String = Paths - .get(IgnifyrConfig.sparkCheckpointDirectory, testJob.id, testJob.mappings.head.name.hashCode.toString) - .toString - val testFile: File = new File(s"$path/test.txt") - io.FileUtils.createParentDirectories(testFile) - val fileOutputStream = new FileOutputStream(testFile) - fileOutputStream.write("test".getBytes("UTF-8")) - fileOutputStream.close() - - // check whether the file is written to the directory - val testDirectory: File = new File(path) - io.FileUtils.sizeOfDirectory(testDirectory) shouldBe >(0L) - - // run the job and expect to clear the created directory - executionService - .runJob("testProject", "testJob", Option.empty, Some(testExecuteJobTask)) - .map(_ => - // Check whether the directory is deleted after the job has completed - Files.exists(testDirectory.toPath) shouldBe false - ) - } - } - - /** - * Create the test folders. - */ - override def beforeAll(): Unit = { - // Deleting folders to start with a clean environment - cleanFolders() - // Create the test data folder to run mapping job - FileUtils.getPath(testDataFolder).toFile.mkdirs() - } - - /** - * Deletes the test folders after all test cases are completed. - * */ - override def afterAll(): Unit = { - cleanFolders() - } - - /** - * Deletes the context path. - * */ - private def cleanFolders(): Unit = { - org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.getPath("").toFile) - } - - private def getMockMappingJobRepository: JobFolderRepository = { - val mockMappingJobRepository: JobFolderRepository = mock[JobFolderRepository] - when(mockMappingJobRepository.getJob("testProject", "testJob")).thenReturn(Future.apply(Some(testJob))) - } - - private def getMockMappingRepository: ProjectMappingFolderRepository = { - val mockMappingRepository: ProjectMappingFolderRepository = mock[ProjectMappingFolderRepository] - when(mockMappingRepository.getFhirMappingByUrl("https://aiccelerate.eu/fhir/mappings/patient-mapping")) - .thenReturn( - FhirMapping( - id = "test", - url = "https://aiccelerate.eu/fhir/mappings/patient-mapping", - name = "test", - source = Seq(FhirMappingSource(alias = "source", url = "test-source")), - context = Map.empty, - mapping = Seq.empty - ) - ) - mockMappingRepository - } - - private def getMockSchemaRepository: SchemaFolderRepository = { - val mockSchemaRepository: SchemaFolderRepository = mock[SchemaFolderRepository] - when(mockSchemaRepository.getSchema("test-source")) - .thenReturn( - Some(StructType(fields = Seq.empty)) - ) - mockSchemaRepository - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/service/MetadataServiceTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/service/MetadataServiceTest.scala deleted file mode 100644 index dd04189c..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/service/MetadataServiceTest.scala +++ /dev/null @@ -1,81 +0,0 @@ -package io.ignifyr.server.service - -import com.typesafe.config.ConfigFactory -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.server.common.config.WebServerConfig -import io.ignifyr.server.common.spi.IgnifyrServerExtension -import io.onfhir.definitions.resource.fhir.FhirDefinitionsConfig -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import scala.concurrent.{ExecutionContext, Future} - -/** - * `/metadata` is fetched on every load of the Ignifyr frontend, so the extension lookup behind it has a - * hard 1-second bound and swallows every failure. Both properties are invisible from the endpoint test - * (which only sees a 200), and both are what keep a hung external service from blocking the UI. - * - * The lookup is also **not** generic: only the extension whose id is exactly "redcap" is ever asked. - */ -class MetadataServiceTest extends AnyFlatSpec with Matchers { - - private val rootConfig = ConfigFactory.load() - - private def serviceWith(extensions: Seq[IgnifyrServerExtension]): MetadataService = - new MetadataService( - new IgnifyrEngineConfig(rootConfig.getConfig("ignifyr")), - new WebServerConfig(rootConfig.getConfig("webserver")), - new FhirDefinitionsConfig(rootConfig.getConfig("fhir")), - extensions - ) - - /** An extension answering `externalComponentVersion` however the test needs it to. */ - private class StubExtension(val id: String, answer: ExecutionContext => Future[Option[String]]) - extends IgnifyrServerExtension { - override def externalComponentVersion()(implicit ec: ExecutionContext): Future[Option[String]] = answer(ec) - } - - private def responding(id: String, version: Option[String]) = - new StubExtension(id, _ => Future.successful(version)) - - "getMetadata" should "report the version of the redcap extension" in { - serviceWith(Seq(responding("redcap", Some("1.2.3")))).getMetadata.ignifyrRedcapVersion shouldBe Some("1.2.3") - } - - it should "report no redcap version when no extension is installed" in { - serviceWith(Seq.empty).getMetadata.ignifyrRedcapVersion shouldBe None - } - - // The seam is documented as generic but is not: MetadataService matches on the literal id "redcap", - // so implementing the hook in any other module has no effect on /metadata today. - it should "ignore the version reported by an extension other than redcap" in { - serviceWith(Seq(responding("observability", Some("9.9.9")))).getMetadata.ignifyrRedcapVersion shouldBe None - } - - it should "swallow a failing redcap lookup instead of failing the whole metadata response" in { - val failing = new StubExtension("redcap", _ => Future.failed(new RuntimeException("connection refused"))) - val metadata = serviceWith(Seq(failing)).getMetadata - metadata.ignifyrRedcapVersion shouldBe None - metadata.name shouldBe "Ignifyr" - } - - it should "give up on a redcap lookup that outlasts the one-second bound" in { - val slow = new StubExtension("redcap", ec => Future { Thread.sleep(3000); Some("too-late") }(ec)) - val startedAt = System.currentTimeMillis() - serviceWith(Seq(slow)).getMetadata.ignifyrRedcapVersion shouldBe None - (System.currentTimeMillis() - startedAt) should be < 3000L - } - - it should "report the repository folders and the archiving settings from the engine config" in { - val metadata = serviceWith(Seq.empty).getMetadata - metadata.repositoryNames.mappings should not be empty - metadata.repositoryNames.schemas should not be empty - metadata.repositoryNames.jobs should not be empty - metadata.archiving.archiveFolder should not be empty - } - - it should "publish the mapping execution configurations the UI displays" in { - val names = serviceWith(Seq.empty).getMetadata.executionConfigurations.map(_.name) - names should contain allOf ("Mapping Timeout", "Maximum Chunk Size", "Batch Group Size") - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/util/CsvUtilTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/util/CsvUtilTest.scala deleted file mode 100644 index 66240bd2..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/util/CsvUtilTest.scala +++ /dev/null @@ -1,201 +0,0 @@ -package io.ignifyr.server.util - -import akka.stream.scaladsl.{Sink, Source} -import akka.util.ByteString -import io.ignifyr.engine.Execution.actorSystem -import io.ignifyr.server.model.csv.CsvHeader -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AsyncWordSpec - -import java.io.File -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import scala.concurrent.Future - -/** - * Covers the CSV editing behind the mapping-context and terminology `content`/`header` routes. Every - * function here rewrites a file the user owns in place, so an off-by-one in the page arithmetic or a - * mis-tracked column rename is silent data loss rather than an error. - */ -class CsvUtilTest extends AsyncWordSpec with Matchers { - - /** Writes the given lines to a fresh temp CSV and returns it. */ - private def csvFile(lines: String*): File = { - val file = Files.createTempFile("ignifyr-csv-util", ".csv").toFile - Files.write(file.toPath, lines.mkString("\n").getBytes(StandardCharsets.UTF_8)) - file - } - - private def linesOf(file: File): Seq[String] = - new String(Files.readAllBytes(file.toPath), StandardCharsets.UTF_8).split("\n").map(_.trim).filter(_.nonEmpty).toSeq - - private def drain(source: Source[ByteString, Any]): Future[Seq[String]] = - source.runWith(Sink.seq).map(_.map(_.utf8String).mkString.split("\n").map(_.trim).filter(_.nonEmpty).toSeq) - - "writeCsvHeaders" should { - - "keep the values of a renamed column" in { - val file = csvFile("code,unit", "\"10839-9\",\"ng/ml\"") - CsvUtil - .writeCsvHeaders(file, Seq(CsvHeader("code", "code"), CsvHeader("targetUnit", "unit"))) - .map { _ => - linesOf(file) shouldBe Seq("code,targetUnit", "\"10839-9\",\"ng/ml\"") - } - } - - "drop the values of a removed column" in { - val file = csvFile("a,b,c", "1,2,3") - CsvUtil - .writeCsvHeaders(file, Seq(CsvHeader("a", "a"), CsvHeader("c", "c"))) - .map { _ => - linesOf(file) shouldBe Seq("a,c", "\"1\",\"3\"") - } - } - - // A newly added column has no data yet, so each row gets a visible placeholder rather than an empty - // cell — that is what tells the user in the UI which column still needs filling in. - "fill an added column with a placeholder naming it" in { - val file = csvFile("a", "1") - CsvUtil - .writeCsvHeaders(file, Seq(CsvHeader("a", "a"), CsvHeader("b", "b"))) - .map { _ => - linesOf(file) shouldBe Seq("a,b", "\"1\",\"\"") - } - } - - "reorder the columns to follow the given headers" in { - val file = csvFile("a,b", "1,2") - CsvUtil - .writeCsvHeaders(file, Seq(CsvHeader("b", "b"), CsvHeader("a", "a"))) - .map { _ => - linesOf(file) shouldBe Seq("b,a", "\"2\",\"1\"") - } - } - - "keep a value that contains the separator inside its quotes" in { - val file = csvFile("code,label", "\"1\",\"a,b\"") - CsvUtil - .writeCsvHeaders(file, Seq(CsvHeader("code", "code"), CsvHeader("label", "label"))) - .map { _ => - linesOf(file) shouldBe Seq("code,label", "\"1\",\"a,b\"") - } - } - - "write only the header row for a file that has no data rows" in { - val file = csvFile("a,b") - CsvUtil.writeCsvHeaders(file, Seq(CsvHeader("a", "a"))).map(_ => linesOf(file) shouldBe Seq("a")) - } - - // Regression: the returned Future used to complete before the write did, so a caller could observe - // the previous content right after being told the update succeeded. - "complete only after the file has been written" in { - val file = csvFile("a,b", "1,2") - CsvUtil - .writeCsvHeaders(file, Seq(CsvHeader("renamed", "a"))) - .map(_ => linesOf(file) shouldBe Seq("renamed", "\"1\"")) - } - } - - "getPaginatedCsvContent" should { - - "return the header plus the requested page" in { - val file = csvFile("h", "r1", "r2", "r3", "r4", "r5") - CsvUtil.getPaginatedCsvContent(file, pageNumber = 2, pageSize = 2).flatMap { case (source, total) => - total shouldBe 5 - drain(source).map(_ shouldBe Seq("h", "r3", "r4")) - } - } - - "return the header plus the first page" in { - val file = csvFile("h", "r1", "r2", "r3") - CsvUtil.getPaginatedCsvContent(file, pageNumber = 1, pageSize = 2).flatMap { case (source, _) => - drain(source).map(_ shouldBe Seq("h", "r1", "r2")) - } - } - - "return a short last page" in { - val file = csvFile("h", "r1", "r2", "r3") - CsvUtil.getPaginatedCsvContent(file, pageNumber = 2, pageSize = 2).flatMap { case (source, _) => - drain(source).map(_ shouldBe Seq("h", "r3")) - } - } - - "return only the header for a page past the end" in { - val file = csvFile("h", "r1") - CsvUtil.getPaginatedCsvContent(file, pageNumber = 5, pageSize = 2).flatMap { case (source, total) => - total shouldBe 1 - drain(source).map(_ shouldBe Seq("h")) - } - } - - "not count the header row in the total" in { - val file = csvFile("h") - CsvUtil.getPaginatedCsvContent(file, pageNumber = 1, pageSize = 10).map { case (_, total) => total shouldBe 0 } - } - } - - "writeCsvAndReturnRowNumber" should { - - "replace exactly the rows of the requested page" in { - val file = csvFile("h", "r1", "r2", "r3", "r4") - val replacement = Source(List(ByteString("new3"), ByteString("new4"))) - CsvUtil.writeCsvAndReturnRowNumber(file, replacement, pageNumber = 2, pageSize = 2).map { total => - linesOf(file) shouldBe Seq("h", "r1", "r2", "new3", "new4") - total shouldBe 4 - } - } - - "leave the header untouched when replacing the first page" in { - val file = csvFile("h", "r1", "r2") - val replacement = Source(List(ByteString("new1"), ByteString("new2"))) - CsvUtil.writeCsvAndReturnRowNumber(file, replacement, pageNumber = 1, pageSize = 2).map { total => - linesOf(file) shouldBe Seq("h", "new1", "new2") - total shouldBe 2 - } - } - - // The page is replaced in place, so a shorter replacement shrinks the file by the difference: the - // two rows of page 2 give way to one, and the rows before the page are untouched. - "shrink the file when the page is replaced by fewer rows" in { - val file = csvFile("h", "r1", "r2", "r3", "r4") - val replacement = Source(List(ByteString("only3"))) - CsvUtil.writeCsvAndReturnRowNumber(file, replacement, pageNumber = 2, pageSize = 2).map { total => - linesOf(file) shouldBe Seq("h", "r1", "r2", "only3") - total shouldBe 3 - } - } - - "report the row count excluding the header" in { - val file = csvFile("h", "r1") - CsvUtil - .writeCsvAndReturnRowNumber(file, Source(List(ByteString("r1'"))), pageNumber = 1, pageSize = 1) - .map(_ shouldBe 1) - } - } - - "saveFileContent" should { - - "overwrite the file with the given content" in { - val file = csvFile("old,header", "old,row") - CsvUtil - .saveFileContent(file, Source(List(ByteString("new,header\n"), ByteString("new,row")))) - .map(_ => linesOf(file) shouldBe Seq("new,header", "new,row")) - } - - "leave no trailing remnant of a longer previous content" in { - val file = csvFile("a,b,c,d,e,f,g,h", "1,2,3,4,5,6,7,8") - CsvUtil - .saveFileContent(file, Source.single(ByteString("x"))) - .map(_ => linesOf(file) shouldBe Seq("x")) - } - - "strip carriage returns so Windows line endings do not leak into the stored file" in { - val file = csvFile("placeholder") - CsvUtil - .saveFileContent(file, Source.single(ByteString("a,b\r\n1,2"))) - .map { _ => - new String(Files.readAllBytes(file.toPath), StandardCharsets.UTF_8) should not include "\r" - } - } - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/util/DataFrameUtilTest.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/util/DataFrameUtilTest.scala deleted file mode 100644 index a17d6dda..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/util/DataFrameUtilTest.scala +++ /dev/null @@ -1,58 +0,0 @@ -package io.ignifyr.server.util - -import io.ignifyr.engine.config.IgnifyrConfig -import io.ignifyr.server.model.{ResourceFilter, RowSelectionOrder} -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Row selection for the "test a mapping" route: the user picks how many source rows to run the mapping - * against and whether to take them from the start or at random. The endpoint suite only ever passes - * "start", so the random branch — which divides by `df.count()` — is only covered here. - */ -class DataFrameUtilTest extends AnyFlatSpec with Matchers { - - private val sparkSession: SparkSession = IgnifyrConfig.sparkSession - - private def rows(n: Int): DataFrame = { - import sparkSession.implicits._ - (1 to n).toDF("value") - } - - "applyResourceFilter" should "take the first rows in order for the start selection" in { - val filtered = DataFrameUtil.applyResourceFilter(rows(10), ResourceFilter(3, RowSelectionOrder.START)) - filtered.count() shouldBe 3 - filtered.collect().map(_.getInt(0)).toSeq shouldBe Seq(1, 2, 3) - } - - it should "not fail when fewer rows exist than requested for the start selection" in { - DataFrameUtil.applyResourceFilter(rows(2), ResourceFilter(10, RowSelectionOrder.START)).count() shouldBe 2 - } - - it should "return at most the requested number of rows for the random selection" in { - DataFrameUtil - .applyResourceFilter(rows(100), ResourceFilter(5, RowSelectionOrder.RANDOM)) - .count() should be <= 5L - } - - // The random branch computes numberOfRows / df.count() as a sampling fraction, so it has to cope with - // a request larger than the frame (fraction > 1, which Spark rejects) and with an empty frame - // (division by zero). - it should "clamp the sampling fraction when more rows are requested than exist" in { - DataFrameUtil - .applyResourceFilter(rows(3), ResourceFilter(10, RowSelectionOrder.RANDOM)) - .count() should be <= 3L - } - - it should "return nothing for the random selection over an empty frame" in { - val empty = rows(1).filter("value < 0") - DataFrameUtil.applyResourceFilter(empty, ResourceFilter(5, RowSelectionOrder.RANDOM)).count() shouldBe 0 - } - - "RowSelectionOrder" should "accept only the two documented orders" in { - RowSelectionOrder.isValid(RowSelectionOrder.START) shouldBe true - RowSelectionOrder.isValid(RowSelectionOrder.RANDOM) shouldBe true - RowSelectionOrder.isValid("sideways") shouldBe false - } -} diff --git a/ignifyr-server/src/test/scala/io/ignifyr/server/util/TestUtil.scala b/ignifyr-server/src/test/scala/io/ignifyr/server/util/TestUtil.scala deleted file mode 100644 index 5cdcc2be..00000000 --- a/ignifyr-server/src/test/scala/io/ignifyr/server/util/TestUtil.scala +++ /dev/null @@ -1,21 +0,0 @@ -package io.ignifyr.server.util - -import io.ignifyr.engine.config.IgnifyrEngineConfig -import io.ignifyr.engine.util.FileUtils -import io.ignifyr.server.repository.project.ProjectFolderRepository -import org.json4s.JArray - -object TestUtil { - - /** - * Reads the project json file as a [[JArray]] - * - * @param engineConfig - * @return - */ - def getProjectJsonFile(engineConfig: IgnifyrEngineConfig): JArray = { - FileOperations - .readFileIntoJson(FileUtils.getPath(ProjectFolderRepository.PROJECTS_JSON).toFile) - .asInstanceOf[JArray] - } -} diff --git a/ignifyr-sink-omop/pom.xml b/ignifyr-sink-omop/pom.xml deleted file mode 100644 index 889df271..00000000 --- a/ignifyr-sink-omop/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-sink-omop_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - - - test - - test - - - io.ignifyr.sink.omop - - - - - - - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - - io.ignifyr - ignifyr-testkit_2.13 - test - - - - diff --git a/ignifyr-sink-omop/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension b/ignifyr-sink-omop/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension deleted file mode 100644 index ed9f5368..00000000 --- a/ignifyr-sink-omop/src/main/resources/META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension +++ /dev/null @@ -1 +0,0 @@ -io.ignifyr.sink.omop.OmopSinkExtension diff --git a/ignifyr-sink-omop/src/main/scala/io/ignifyr/sink/omop/OmopSinkExtension.scala b/ignifyr-sink-omop/src/main/scala/io/ignifyr/sink/omop/OmopSinkExtension.scala deleted file mode 100644 index cc9a084c..00000000 --- a/ignifyr-sink-omop/src/main/scala/io/ignifyr/sink/omop/OmopSinkExtension.scala +++ /dev/null @@ -1,17 +0,0 @@ -package io.ignifyr.sink.omop - -import io.ignifyr.engine.spi.IgnifyrExtension - -/** - * Enterprise OMOP sink module — placeholder skeleton. Reserves the module and its ServiceLoader - * registration for the upcoming "map to OMOP" feature: an `OmopSinkSettings`-keyed sink writer - * typing rows against versioned OMOP CDM schemas (5.3/5.4/6.0) with FK-ordered table writes, plus - * an OMOP-vocabulary-backed terminology service (design recorded in the edition-split plan). - * - * Registers nothing yet — the engine-side model classes (`OmopSinkSettings`, ...) must land in the - * community engine (parse-everywhere rule) together with the implementation. - */ -class OmopSinkExtension extends IgnifyrExtension { - - override val id: String = "sink-omop" -} diff --git a/ignifyr-sink-omop/src/test/scala/io/ignifyr/sink/omop/OmopSinkExtensionSpec.scala b/ignifyr-sink-omop/src/test/scala/io/ignifyr/sink/omop/OmopSinkExtensionSpec.scala deleted file mode 100644 index bb1a8749..00000000 --- a/ignifyr-sink-omop/src/test/scala/io/ignifyr/sink/omop/OmopSinkExtensionSpec.scala +++ /dev/null @@ -1,17 +0,0 @@ -package io.ignifyr.sink.omop - -import io.ignifyr.engine.spi.ExtensionRegistry -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** - * Verifies the OMOP sink skeleton is discovered through ServiceLoader when it is on the classpath - * (no Docker required — this only inspects the extension registry). The extension contributes - * nothing yet; contribution assertions arrive with the map-to-OMOP implementation. - */ -class OmopSinkExtensionSpec extends AnyFlatSpec with Matchers { - - "The OMOP sink extension" should "be discovered through ServiceLoader" in { - ExtensionRegistry.extensions.map(_.id) should contain("sink-omop") - } -} diff --git a/ignifyr-terminology-tools/pom.xml b/ignifyr-terminology-tools/pom.xml deleted file mode 100644 index 4c47e3d6..00000000 --- a/ignifyr-terminology-tools/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - 4.0.0 - - - io.ignifyr - ignifyr_2.13 - ${revision} - - - ignifyr-terminology-tools_2.13 - jar - - - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.scalatest - scalatest-maven-plugin - - io.ignifyr.tools.terminology - - - - - - - - - io.ignifyr - ignifyr-engine_2.13 - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-csv - - - - - org.postgresql - postgresql - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.mockito - mockito-scala_2.13 - test - - - - diff --git a/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/Concept.scala b/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/Concept.scala deleted file mode 100644 index 35c904fc..00000000 --- a/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/Concept.scala +++ /dev/null @@ -1,15 +0,0 @@ -package io.ignifyr.tools.terminology - -import scala.beans.BeanProperty - -case class OmopConcept(concept_id: Int, vocabulary_id: String, concept_code: String, concept_name: String) - -case class OmopConceptRelationship(concept_id_1: Int, concept_id_2: Int, relationship_id: String) - -case class TerminologySystemMapping( - @BeanProperty source_system: String, - @BeanProperty source_code: String, - @BeanProperty target_system: String, - @BeanProperty target_code: String, - @BeanProperty target_display: String -) diff --git a/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/GeneratorDBAdapter.scala b/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/GeneratorDBAdapter.scala deleted file mode 100644 index acbba278..00000000 --- a/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/GeneratorDBAdapter.scala +++ /dev/null @@ -1,103 +0,0 @@ -package io.ignifyr.tools.terminology - -import io.ignifyr.tools.terminology.GeneratorDBAdapter._ - -import java.sql.{Connection, DriverManager, PreparedStatement, ResultSet} -import scala.collection.mutable - -object GeneratorDBAdapter { - private val DB_URL = "jdbc:postgresql://localhost:5432/ucl_omop" - private val USERNAME = "postgres" - private val PASSWORD = "password" - - private val COLUMN_CONCEPT_CODE: String = "concept_code" - private val COLUMN_CONCEPT_OMOP_ID = "concept_id" - private val COLUMN_CONCEPT_VOCABULARY_ID = "vocabulary_id" - private val COLUMN_CONCEPT_NAME = "concept_name" - - private val COLUMN_CONCEPT_RELATIONSHIP_CONCEPT_ID_1 = "concept_id_1" - private val COLUMN_CONCEPT_RELATIONSHIP_CONCEPT_ID_2 = "concept_id_2" - private val COLUMN_CONCEPT_RELATIONSHIP_RELATIONSHIP_ID = "relationship_id" - - private val getConceptsForVocabulary: String = s"SELECT * FROM concept WHERE vocabulary_id = ?" - private val getConceptRelationships: String = "SELECT * FROM concept_relationship cr" + - " WHERE cr.concept_id_1 != cr.concept_id_2" -} - -class GeneratorDBAdapter { - val connection: Connection = DriverManager.getConnection(DB_URL, USERNAME, PASSWORD) - - /** - * Get concepts for a specific terminology, wrapped as [[OmopConcept]]s - * - * @param vocabularyId OMOP vocabulary id for the terminology system, for which the concepts to be obtained - * @param conceptDomain Domain of the concepts of interests as defined in the OMOP vocabulary e.g. Conditin, Procedure, etc. - * @return - */ - def getConcepts(vocabularyId: String, conceptDomain: String): Set[OmopConcept] = { - var statement: PreparedStatement = null - try { - statement = connection.prepareStatement(s"$getConceptsForVocabulary AND domain_id = ? ") - statement.setString(1, vocabularyId) - statement.setString(2, conceptDomain) - getOmopConcepts(statement.executeQuery()) - } finally { - statement.close() - } - } - - private def getOmopConcepts(resultSet: ResultSet): Set[OmopConcept] = { - val concepts: mutable.Set[OmopConcept] = mutable.Set.empty - while (resultSet.next()) { - concepts.add( - OmopConcept( - resultSet.getInt(COLUMN_CONCEPT_OMOP_ID), - resultSet.getString(COLUMN_CONCEPT_VOCABULARY_ID), - resultSet.getString(COLUMN_CONCEPT_CODE), - resultSet.getString(COLUMN_CONCEPT_NAME) - ) - ) - } - concepts.toSet - } - - /** - * Retrieves all the relationships, specified by the input parameter, from the OMOP vocabulary - * @param relationships Names as defined in the OMOP vocabulary of the relationships to be retrieved - * @return - */ - def getOmopConceptRelationships(relationships: Set[String]): Set[OmopConceptRelationship] = { - val statement: PreparedStatement = prepareRelationshipStatement(relationships) - val resultSet: ResultSet = statement.executeQuery() - try { - val relationships: mutable.Set[OmopConceptRelationship] = mutable.Set.empty - while (resultSet.next()) { - relationships.add( - OmopConceptRelationship( - resultSet.getInt(COLUMN_CONCEPT_RELATIONSHIP_CONCEPT_ID_1), - resultSet.getInt(COLUMN_CONCEPT_RELATIONSHIP_CONCEPT_ID_2), - resultSet.getString(COLUMN_CONCEPT_RELATIONSHIP_RELATIONSHIP_ID) - ) - ) - } - relationships.toSet - - } finally { - resultSet.close() - statement.close() - } - } - - private def prepareRelationshipStatement(relationships: Set[String]): PreparedStatement = { - val relationshipCriteria: String = relationships.map(_ => s"cr.relationship_id = ?").mkString(" AND ") - val statement: PreparedStatement = - connection.prepareStatement(s"$getConceptRelationships AND $relationshipCriteria") - relationships.zipWithIndex - .foreach(r => statement.setString(r._2 + 1, r._1)) - statement - } - - def clear(): Unit = { - connection.close() - } -} diff --git a/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/TerminologyMappingGenerator.scala b/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/TerminologyMappingGenerator.scala deleted file mode 100644 index 691cf7c1..00000000 --- a/ignifyr-terminology-tools/src/main/scala/io/ignifyr/tools/terminology/TerminologyMappingGenerator.scala +++ /dev/null @@ -1,250 +0,0 @@ -package io.ignifyr.tools.terminology - -import com.fasterxml.jackson.dataformat.csv.{CsvMapper, CsvSchema} -import io.ignifyr.engine.mapping.service.LocalTerminologyService.ConceptMapFileColumns -import io.ignifyr.tools.terminology.TerminologyMappingGenerator._ - -import java.io.File -import scala.collection.mutable -import scala.jdk.CollectionConverters.IterableHasAsJava - -object TerminologyMappingGenerator { - - private val OMOP_URL: String = "https://www.ohdsi.org/omop" - - /** - * Example usage - */ - def main(args: Array[String]): Unit = { - new TerminologyMappingGenerator(new GeneratorDBAdapter()).generateMappings( - "SNOMED", - "ICD10", - "http://hl7.org/fhir/sid/icd-10", - "Condition", - Set("Mapped from"), - "icd10cm.csv" - ) - } - -} - -/** - * A class to generate file-based terminology mappings by parsing the relationships defined in the OMOP vocabulary. - * The generator creates terminology mappings from a source terminology to a target one e.g. from SNOMED to ICD10. - * - * A note on the semantics of the mappings: - *

- * The mappings captured by this generated cover all the equivalence relationships (formed by the 'Mapped from' relationship by default), - * which are not necessarily 1 to 1. For example, a single generic SNOMED code can map to thousands of ICD codes. This 1 to many - * characteristic of the mappings should be considered when the output of the generator is to be used as a terminology service. - *

- */ -class TerminologyMappingGenerator(dbAdapter: GeneratorDBAdapter) { - // All concepts defined in the OMOP vocabulary associated with the source and target terminologies. - // It is a map from the OMOP concept id to the concept object including the details about the concept. - var concepts: mutable.Map[Int, OmopConcept] = mutable.Map.empty - // All relationships defined in the OMOP vocabulary. The relationships are represented as map of (OMOP concept id to set of OMOP concepts) entries where the - // key represents the identifier of the source concept and set of concepts represent the concepts from the target terminology system equivalent to the source concept. - var relationships: mutable.Map[Int, mutable.Set[OmopConcept]] = mutable.Map.empty - - /** - * Generates a file including terminology translations from a source code system to a target code system. - * - * @param sourceSystem OMOP vocabulary ID for the source terminology - * @param targetSystem OMOP vocabulary ID for the target terminology - * @param targetSystemUrl URL for the target vocabulary. It will be set as the URL of the target codes - * @param conceptDomain OMOP domain of the concepts of interest e.g. Condition, Procedure, Measurement, etc. - * @param conceptRelationships Relationships types to be considered as defined in the OMOP vocabulary e.g. Mapped from - * @param outputFile Name of the output file - */ - def generateMappings( - sourceSystem: String, - targetSystem: String, - targetSystemUrl: String, - conceptDomain: String, - conceptRelationships: Set[String], - outputFile: String - ): Unit = { - // Retrieve all the concepts associated with the source and target code systems (terminologies) and all relationships are obtained from the OMOP vocabulary - populateConceptsAndRelationships(sourceSystem, targetSystem, conceptDomain, conceptRelationships) - - // extract mappings by traversing the relationships - val terminologySystemMappings = extractMappings(sourceSystem, targetSystem, targetSystemUrl, conceptDomain) - - // DB operations are complete, clear any resources - dbAdapter.clear() - - // Generate the file using the accumulated mappings - generateTerminologySystemFile(terminologySystemMappings, outputFile) - } - - /** - * This method traverses the direct or indirect relationships from the source system to the target system. - * For each code defined in the source terminology, all mappings leading to a code from the target terminology are identified. - * A CSV file with the given name is created containing the identified mappings. While the URL of the target terminology codes are specified explicitly, - * the URL of the source terminology is set to [[OMOP_URL]] as the source codes are in fact OMOP concept identifiers representing a standard-based code. - * (This does not have a practical implication concerning the mapping, so can be changed). - * - * The following columns are included in the CSV: "source_system","source_code","target_system","target_code","target_display" - * An example row would be: - *
    - *
  • source system: www.ohdsi.org/omop -> Omop URL
    • - *
  • source_code: 3003176 -> OMOP concept id for the LOINC code 11727-5
    • - *
  • target_system: http://loinc.org -> LOINC terminology URL
    • - *
  • target_code: 11727-5 -> LOINC code as the target code
    • - *
  • target_display: Body weight
    • - *
- * - * @param sourceSystem - * @param targetSystem - * @param targetSystemUrl - * @param conceptDomain - * @return - */ - private def extractMappings( - sourceSystem: String, - targetSystem: String, - targetSystemUrl: String, - conceptDomain: String - ): Set[TerminologySystemMapping] = { - // Each source system concept is a starting point to initiate a mapping identification process - val sourceSystemConcepts: Set[OmopConcept] = dbAdapter.getConcepts(sourceSystem, conceptDomain) - // Visited concepts will keep the concepts that are already considered so that they won't be considered again. This case might happen - // when a concept is considered already while identifiying mappings for an upstream concept. For example: - // Assuming we have such a mapping structure: SNOMED code 1 -> SNOMED code 2 -> ICD, indicating that SNOMED code 1 maps to SNOMED code 2; and SNOMED code 2 - // maps to an ICD code. While getting the mappings for SNOMED code 1, we will have processed the SNOMED code 2 as well. Therefore, it won't need to be processed once again. - val visitedConcepts: mutable.Set[OmopConcept] = mutable.Set.empty[OmopConcept] - // Keeps the terminology system concepts to be outputted to the CSV file - val terminologySystemMappings: mutable.Set[TerminologySystemMapping] = mutable.Set.empty[TerminologySystemMapping] - - // Find mappings for each source concept unless it's been processed already - sourceSystemConcepts.toSeq.foreach(concept => { - if (!visitedConcepts.contains(concept)) { - val mappings: Set[TerminologySystemMapping] = - findMappings(concept, targetSystem, targetSystemUrl, visitedConcepts) - terminologySystemMappings.addAll(mappings) - if (mappings.nonEmpty) { - println(s"Retrieved ${mappings.size} mappings for concept: $concept") - } - } - }) - terminologySystemMappings.toSet - } - - /** - * Writes the given [[TerminologySystemMapping]]s to the specified output file. - * - * @param terminologySystemConcepts Set of mappings. Each mapping will be serialized as a row in the CSV file. - * @param outputFile Name of the output file - */ - private def generateTerminologySystemFile( - terminologySystemConcepts: Set[TerminologySystemMapping], - outputFile: String - ): Unit = { - val myObjectWriter = new CsvMapper() - .writerFor(classOf[TerminologySystemMapping]) - .`with`(getCsvSchema()) - val tempFile = new File(outputFile) - myObjectWriter.writeValues(tempFile).writeAll(terminologySystemConcepts.asJava) - } - - /** - * Finds mappings for an individual source code (OMOP concept). Applies BFS to discover chains of relationships such as NOMED code 1 -> SNOMED code ->2 -> ICD. - * In this example; although SNOMED code 1 does not have direct mapping to ICD code, it has a mapping via SNOMED code 2. The search continues until all - * the relationships are processed. Codes belonging to the target terminology are accumulated and returned as [[TerminologySystemMapping]]. - * - * @param sourceConcept Source concept of the mapping - * @param targetSystem OMOP vocabulary id of the target terminology - * @param targetSystemUrl URL for the target terminology - * @param visitedConcepts A set containing the visited concepts during earlier operations - * @return - */ - private def findMappings( - sourceConcept: OmopConcept, - targetSystem: String, - targetSystemUrl: String, - visitedConcepts: mutable.Set[OmopConcept] - ): Set[TerminologySystemMapping] = { - val visitQueue: mutable.Queue[OmopConcept] = mutable.Queue.empty[OmopConcept] - val equivalentSourceConcepts: mutable.Set[OmopConcept] = mutable.Set.empty - val equivalentTargetConcepts: mutable.Set[OmopConcept] = mutable.Set.empty - - visitQueue.enqueue(sourceConcept) - while (visitQueue.nonEmpty) { - val currentCode: OmopConcept = visitQueue.dequeue() - equivalentSourceConcepts.add(currentCode) - if (!visitedConcepts.contains(currentCode)) { - visitedConcepts.add(currentCode) - relationships.get(currentCode.concept_id) match { - case None => - case Some(mappings) => - mappings.foreach(concept => { - if (concept.vocabulary_id.equals(targetSystem)) { - equivalentTargetConcepts.add(concept) - } else { - visitQueue.enqueue(concept) - } - }) - } - } - } - equivalentSourceConcepts - .flatMap(sourceConcept => { - equivalentTargetConcepts.map(targetConcept => { - TerminologySystemMapping( - OMOP_URL, - sourceConcept.concept_id.toString, - targetSystemUrl, - targetConcept.concept_code, - targetConcept.concept_name - ) - }) - }) - .toSet - } - - /** - * Obtains concepts for the source and target terminologies and relationships as defined in the OMOP vocabulary. - * - * @param sourceSystem - * @param targetSystem - * @param conceptDomain - * @param conceptRelationships - */ - private def populateConceptsAndRelationships( - sourceSystem: String, - targetSystem: String, - conceptDomain: String, - conceptRelationships: Set[String] - ): Unit = { - dbAdapter.getConcepts(sourceSystem, conceptDomain).foreach(concept => concepts.put(concept.concept_id, concept)) - dbAdapter.getConcepts(targetSystem, conceptDomain).foreach(concept => concepts.put(concept.concept_id, concept)) - dbAdapter - .getOmopConceptRelationships(conceptRelationships) - .foreach(relationship => { - val conceptMappings: mutable.Set[OmopConcept] = - relationships.getOrElseUpdate(relationship.concept_id_1, mutable.Set.empty[OmopConcept]) - if (concepts.contains(relationship.concept_id_2)) { - conceptMappings.add(concepts(relationship.concept_id_2)) - } - }) - } - - /** - * Creates a [[CsvSchema]] conforming to the structure of a concept map that will be used as a terminology mapping service. - * @return - */ - private def getCsvSchema(): CsvSchema = { - CsvSchema - .builder() - .addColumn(ConceptMapFileColumns.SOURCE_SYSTEM) - .addColumn(ConceptMapFileColumns.SOURCE_CODE) - .addColumn(ConceptMapFileColumns.TARGET_SYSTEM) - .addColumn(ConceptMapFileColumns.TARGET_CODE) - .addColumn(ConceptMapFileColumns.TARGET_DISPLAY) - .build() - .withHeader() - .withEscapeChar('\\') - } - -} diff --git a/ignifyr-terminology-tools/src/test/scala/io/ignifyr/tools/terminology/TerminologyMappingGeneratorTest.scala b/ignifyr-terminology-tools/src/test/scala/io/ignifyr/tools/terminology/TerminologyMappingGeneratorTest.scala deleted file mode 100644 index fa377905..00000000 --- a/ignifyr-terminology-tools/src/test/scala/io/ignifyr/tools/terminology/TerminologyMappingGeneratorTest.scala +++ /dev/null @@ -1,57 +0,0 @@ -package io.ignifyr.tools.terminology - -import io.ignifyr.tools.terminology.{ - GeneratorDBAdapter, - OmopConcept, - OmopConceptRelationship, - TerminologyMappingGenerator, - TerminologySystemMapping -} -import org.mockito.MockitoSugar._ -import org.scalatest.PrivateMethodTester -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -class TerminologyMappingGeneratorTest extends AnyFlatSpec with Matchers with PrivateMethodTester { - - val mockDbAdapter: GeneratorDBAdapter = mock[GeneratorDBAdapter] - when(mockDbAdapter.getConcepts("SNOMED", "Some domain")).thenReturn( - Set( - OmopConcept(1, "SNOMED", "SNOMED-1", "SNOMED-1"), - OmopConcept(2, "SNOMED", "SNOMED-2", "SNOMED-2"), - OmopConcept(3, "SNOMED", "SNOMED-3", "SNOMED-3") - ) - ) - when(mockDbAdapter.getConcepts("ICD10", "Some domain")).thenReturn( - Set( - OmopConcept(4, "ICD10", "ICD10-1", "ICD10-1") - ) - ) - when(mockDbAdapter.getOmopConceptRelationships(Set("Mapped from"))).thenReturn( - Set( - OmopConceptRelationship(1, 2, "Mapped from"), - OmopConceptRelationship(2, 4, "Mapped from"), - OmopConceptRelationship(3, 4, "Mapped from") - ) - ) - - val generator: TerminologyMappingGenerator = new TerminologyMappingGenerator(mockDbAdapter) - - "TerminologySystemMappingGenerator" should "construct concepts and relationships" in { - val populateConceptsAndRelationships = - PrivateMethod[Set[TerminologySystemMapping]](Symbol("populateConceptsAndRelationships")) - generator invokePrivate populateConceptsAndRelationships("SNOMED", "ICD10", "Some domain", Set("Mapped from")) - generator.concepts.size shouldEqual 4 - generator.relationships.size shouldEqual 3 - } - - "TerminologySystemMappingGenerator" should "extract implied mappings" in { - val extractMappings = PrivateMethod[Set[TerminologySystemMapping]](Symbol("extractMappings")) - val mappings = generator invokePrivate extractMappings("SNOMED", "ICD10", "SomeUrl", "Some domain") - mappings.size shouldEqual 3 - mappings.exists(mapping => mapping.source_code.equals("1") && mapping.target_code.equals("ICD10-1")) shouldBe true - mappings.exists(mapping => mapping.source_code.equals("2") && mapping.target_code.equals("ICD10-1")) shouldBe true - mappings.exists(mapping => mapping.source_code.equals("3") && mapping.target_code.equals("ICD10-1")) shouldBe true - } - -} diff --git a/pom.xml b/pom.xml index 9665e6e4..fe5d5e25 100644 --- a/pom.xml +++ b/pom.xml @@ -82,20 +82,7 @@ ignifyr-connector-file ignifyr-sink-fhir ignifyr-sink-file - ignifyr-sink-omop - ignifyr-format-json - ignifyr-format-delta - ignifyr-connector-fhir-server - ignifyr-connector-kafka - ignifyr-runtime-streaming - ignifyr-runtime-scheduling ignifyr-cli - ignifyr-redcap - ignifyr-server - ignifyr-server-common - ignifyr-rxnorm - ignifyr-terminology-tools - ignifyr-observability ignifyr-testkit @@ -615,71 +602,11 @@ ignifyr-sink-file_2.13 ${revision} - - io.ignifyr - ignifyr-sink-omop_2.13 - ${revision} - - - io.ignifyr - ignifyr-format-json_2.13 - ${revision} - - - io.ignifyr - ignifyr-format-delta_2.13 - ${revision} - - - io.ignifyr - ignifyr-connector-fhir-server_2.13 - ${revision} - - - io.ignifyr - ignifyr-connector-kafka_2.13 - ${revision} - - - io.ignifyr - ignifyr-runtime-streaming_2.13 - ${revision} - - - io.ignifyr - ignifyr-runtime-scheduling_2.13 - ${revision} - - - io.ignifyr - ignifyr-redcap_2.13 - ${revision} - - - io.ignifyr - ignifyr-terminology-tools_2.13 - ${revision} - - - io.ignifyr - ignifyr-observability_2.13 - ${revision} - io.ignifyr ignifyr-common ${revision} - - io.ignifyr - ignifyr-server-common - ${revision} - - - io.ignifyr - ignifyr-rxnorm_2.13 - ${revision} - diff --git a/test-flow/.gitignore b/test-flow/.gitignore index 27aff80d..ed269ba1 100644 --- a/test-flow/.gitignore +++ b/test-flow/.gitignore @@ -1,7 +1,2 @@ -# Runtime scratch generated by run-manual-flow.sh -/workspace/ -/watch/ -/web-dist/ - # pom backup left by check-enforcer-gate.sh if it were interrupted mid-run *.enforcer-test.bak diff --git a/test-flow/check-editions.sh b/test-flow/check-editions.sh index 5d1aa26e..c0e488ba 100644 --- a/test-flow/check-editions.sh +++ b/test-flow/check-editions.sh @@ -1,22 +1,23 @@ #!/usr/bin/env bash # -# Enterprise/Community edition separation check (packaged artifacts). +# Community edition boundary check (packaged artifact). # # Complements the CommunityEditionSeparationSpec (which checks the registry on the community *test* -# classpath) by inspecting the actually-shipped fat jars and the community CLI's runtime behavior: +# classpath) by inspecting the actually-shipped community fat jar and the CLI's runtime behavior: # -# SPI manifest — the merged META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension in the community -# jar lists only community extensions; the server jar lists the rest. +# jar content — no enterprise library (Kafka, Delta, cron4j, Logstash) got shaded in. +# SPI manifest — the merged META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension lists exactly +# the community extensions. # behavior — the community CLI refuses enterprise jobs with an actionable MissingX error # (streaming -> MissingCapabilityException, Kafka -> MissingConnectorException). # -# Needs both fat jars; builds them (tests skipped) if missing. Usage: test-flow/check-editions.sh +# The enterprise modules live in their own (private) repository, which checks the server jar's side of +# the boundary. Builds the community jar (tests skipped). Usage: test-flow/check-editions.sh set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" CLI_JAR="$REPO_ROOT/ignifyr-cli/target/ignifyr-engine-standalone.jar" -SRV_JAR="$REPO_ROOT/ignifyr-server/target/ignifyr-server-standalone.jar" PASS=0; FAILC=0 ok() { printf ' \033[1;32mPASS\033[0m %s\n' "$*"; PASS=$((PASS+1)); } @@ -24,18 +25,17 @@ bad() { printf ' \033[1;31mFAIL\033[0m %s\n' "$*"; FAILC=$((FAILC+1)); } warn() { printf ' \033[1;33mWARN\033[0m %s\n' "$*"; } # non-fatal (packaged-runtime behavior is environment-sensitive) log() { printf '\n\033[1;34m== %s ==\033[0m\n' "$*"; } -# Enterprise footprints that must be absent from the community jar and present in the server jar. +# Enterprise footprints that must be absent from the community jar. The enterprise *code* cannot be in +# this repository any more, so the class markers guard against a vendored copy or a stray dependency; +# the library markers are the ones the enforcer gate bans. ENTERPRISE_CLASSES=( io/ignifyr/runtime/streaming io/ignifyr/runtime/scheduling io/ignifyr/connector/kafka io/ignifyr/connector/fhirserver io/ignifyr/redcap io/ignifyr/format/delta io/ignifyr/format/json io/ignifyr/observability - io/ignifyr/sink/omop + io/ignifyr/sink/omop io/ignifyr/server/ ) ENTERPRISE_LIBS=( org/apache/spark/sql/kafka io/delta/ it/sauronsoftware/cron4j net/logstash/logback ) -# Every enterprise module that registers a top-level IgnifyrExtension, so the SPI-manifest section -# below (the authoritative one -- the jar-content section above is best-effort and skippable) covers -# all of them. format-json and observability are absent on purpose: neither declares an -# IgnifyrExtension, so neither can ever appear in this manifest. +# Every enterprise extension id prefix; none may appear in the community SPI manifest. ENTERPRISE_EXT=( io.ignifyr.runtime.streaming io.ignifyr.runtime.scheduling io.ignifyr.connector.kafka io.ignifyr.connector.fhirserver io.ignifyr.redcap io.ignifyr.sink.omop @@ -44,13 +44,13 @@ ENTERPRISE_EXT=( # Community distribution now includes the split-out FHIR + file sinks (sink-fhir / sink-file). COMMUNITY_EXT=( io.ignifyr.connector.sql io.ignifyr.connector.file io.ignifyr.sink.fhir io.ignifyr.sink.file ) -log "(Re)building the community + server fat jars (always fresh — stale jars give false results)" -( cd "$REPO_ROOT" && mvn -q -DskipTests -pl ignifyr-cli,ignifyr-server -am package ) || { echo "build failed" >&2; exit 1; } -[ -f "$CLI_JAR" ] && [ -f "$SRV_JAR" ] || { echo "jars missing after build" >&2; exit 1; } +log "(Re)building the community fat jar (always fresh — a stale jar gives false results)" +( cd "$REPO_ROOT" && mvn -q -DskipTests -pl ignifyr-cli -am package ) || { echo "build failed" >&2; exit 1; } +[ -f "$CLI_JAR" ] || { echo "jar missing after build" >&2; exit 1; } command -v unzip >/dev/null 2>&1 || { echo "unzip required" >&2; exit 1; } # ---- jar content (best-effort, non-fatal) ------------------------------------ -log "Jar content: enterprise code/libs excluded from community, present in server" +log "Jar content: enterprise code/libs excluded from the community jar" # The community fat jar is a zip64 archive with ~96k entries, which Info-ZIP cannot enumerate; the JDK # `jar` tool and Python's zipfile can. Collect candidate `jar` binaries (PATH, then the running JVM's # java.home, then $JAVA_HOME); cygpath/CRLF handling keeps it working under Git Bash. @@ -80,7 +80,6 @@ list_jar() { return 1 } CLI_ENTRIES="$(list_jar "$CLI_JAR")" -SRV_ENTRIES="$(list_jar "$SRV_JAR")" # Match with bash's own substring test, NOT `echo "$VAR" | grep -q`. These listings are ~6 MB, and # `grep -q` exits at the first match and closes the pipe, so `echo` dies of SIGPIPE (141); under # `pipefail` (set at the top) that makes a *successful* match evaluate as false. Every check below @@ -92,28 +91,22 @@ has() { [[ "$1" == *"$2"* ]]; } # Skip (non-fatal) unless a lister actually enumerated each jar. guard_ok=1 has "$CLI_ENTRIES" "io/ignifyr/engine/" || { warn "no jar lister could enumerate the community jar here — skipping this best-effort section (the SPI manifest + enforcer gate are authoritative)"; guard_ok=0; } -has "$SRV_ENTRIES" "io/ignifyr/server/" || { guard_ok=0; } if [ "$guard_ok" -eq 0 ]; then echo " (skipping jar-content checks — non-fatal; the ServiceLoader manifest and behavior checks below still run)" else for marker in "${ENTERPRISE_CLASSES[@]}" "${ENTERPRISE_LIBS[@]}"; do if has "$CLI_ENTRIES" "$marker"; then bad "community jar unexpectedly contains '$marker'"; else ok "community jar excludes '$marker'"; fi done -for marker in "${ENTERPRISE_CLASSES[@]}"; do - if has "$SRV_ENTRIES" "$marker"; then ok "server jar contains '$marker'"; else bad "server jar missing '$marker'"; fi -done fi # ---- SPI manifest ------------------------------------------------------------ log "ServiceLoader manifest (io.ignifyr.engine.spi.IgnifyrExtension)" CLI_SPI="$(unzip -p "$CLI_JAR" META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension 2>/dev/null)" -SRV_SPI="$(unzip -p "$SRV_JAR" META-INF/services/io.ignifyr.engine.spi.IgnifyrExtension 2>/dev/null)" for ext in "${COMMUNITY_EXT[@]}"; do echo "$CLI_SPI" | grep -q "$ext" && ok "community SPI lists '$ext'" || bad "community SPI missing '$ext'" done for ext in "${ENTERPRISE_EXT[@]}"; do echo "$CLI_SPI" | grep -q "$ext" && bad "community SPI unexpectedly lists '$ext'" || ok "community SPI excludes '$ext'" - echo "$SRV_SPI" | grep -q "$ext" && ok "server SPI lists '$ext'" || bad "server SPI missing '$ext'" done # ---- community CLI runtime behavior ----------------------------------------- diff --git a/test-flow/check-release-ready.sh b/test-flow/check-release-ready.sh index 1fca4913..0f9e46c4 100644 --- a/test-flow/check-release-ready.sh +++ b/test-flow/check-release-ready.sh @@ -2,18 +2,19 @@ # # Release-readiness gate. # -# Ignifyr publishes no Maven artifacts -- nothing depends on it as a library. A release is a tag, -# the two fat jars, and the Docker images built from them. That makes the interesting question -# "what is inside the jars" rather than "what does the pom declare", because a shaded jar -# *redistributes* every dependency and inherits its obligations. Five invariants: +# A Community release is a tag, the library modules deployed to SRDC Nexus (for the enterprise +# edition to build on), the standalone fat jar, and the Docker image built from it. The fat jar is +# the interesting part: a shaded jar *redistributes* every dependency and inherits its obligations, +# so the question is "what is inside the jar" rather than "what does the pom declare". Four +# invariants: # -# 1. Nothing resolves to a -SNAPSHOT, so the jars are rebuildable from the tag. -# 2. Both jars carry aggregated third-party NOTICEs and Ignifyr's own LICENSE (Apache-2.0 4(d)). +# 1. Nothing resolves to a -SNAPSHOT, so the jar and the deployed modules are rebuildable from the tag. +# 2. The jar carries aggregated third-party NOTICEs and Ignifyr's own LICENSE (Apache-2.0 4(d)). # 3. No copyleft artifact reaches the Apache-2.0 community distribution. Repofyr -- the onFHIR # server continuation -- is GPL-3.0 and sits one dependency edge from code we already use. -# 4. ignifyr-terminology-tools is in neither distribution: it embeds hard-coded dev Postgres -# credentials, which is exactly why it ships nowhere. -# 5. The working tree is clean and the tag is free, so the tag names what was verified. +# 4. The working tree is clean and the tag is free, so the tag names what was verified. +# +# The enterprise server jar is checked by its own copy of this gate in the enterprise repository. # # Two modes. Bare, it is a per-commit guard: version checks WARN, because a development ${revision} # is legitimately a snapshot. With --release they are hard failures. @@ -21,7 +22,7 @@ # test-flow/check-release-ready.sh # dev guard (mainly invariants 2-4) # test-flow/check-release-ready.sh --release # release gate; everything must pass # -# Builds and installs the two distributions and their upstream modules (tests skipped) before +# Builds and installs the distribution and its upstream modules (tests skipped) before # checking, so the dependency listings resolve on a clean checkout. No Docker. set -uo pipefail @@ -30,7 +31,6 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$REPO_ROOT" CLI_JAR="$REPO_ROOT/ignifyr-cli/target/ignifyr-engine-standalone.jar" -SRV_JAR="$REPO_ROOT/ignifyr-server/target/ignifyr-server-standalone.jar" RELEASE_MODE=0 [ "${1:-}" = "--release" ] && RELEASE_MODE=1 @@ -62,12 +62,12 @@ jar_read() { # -> raw bytes on stdout, empty if absent else unzip -p "$1" "$2" 2>/dev/null; fi } -log "(Re)building the community + server fat jars" +log "(Re)building the community fat jar" # `install`, not `package`: the dependency:list calls below resolve ignifyr-cli's sibling # modules from the local repository, and CI only ever runs `verify`/`package`, which do not # put them there. Without this the listings come back empty on a clean runner. -mvn -q -DskipTests -pl ignifyr-cli,ignifyr-server -am install || { echo "build failed" >&2; exit 1; } -[ -f "$CLI_JAR" ] && [ -f "$SRV_JAR" ] || { echo "jars missing after build" >&2; exit 1; } +mvn -q -DskipTests -pl ignifyr-cli -am install || { echo "build failed" >&2; exit 1; } +[ -f "$CLI_JAR" ] || { echo "jar missing after build" >&2; exit 1; } # ---- 1. no -SNAPSHOT anywhere ------------------------------------------------ log "1. Nothing resolves to a -SNAPSHOT" @@ -79,7 +79,7 @@ esac DEPS="$(mktemp)"; CLI_DEPS="$(mktemp)" trap 'rm -f "$DEPS" "$CLI_DEPS"' EXIT -mvn -q dependency:list -pl ignifyr-cli,ignifyr-server -DincludeScope=runtime \ +mvn -q dependency:list -pl ignifyr-cli -DincludeScope=runtime \ -DoutputFile="$DEPS" -DappendOutput=true >/dev/null 2>&1 # An empty listing would make every dependency check below pass vacuously. if ! grep -qE ':[^:]+:[^:]+:' "$DEPS" 2>/dev/null; then @@ -87,7 +87,7 @@ if ! grep -qE ':[^:]+:[^:]+:' "$DEPS" 2>/dev/null; then fi SNAPS="$(sed 's/^ *//' "$DEPS" 2>/dev/null | grep -E ':[^:]*-SNAPSHOT' | sort -u || true)" if [ -z "$SNAPS" ]; then - ok "no -SNAPSHOT on either distribution runtime classpath" + ok "no -SNAPSHOT on the distribution runtime classpath" else # Ignifyr's own modules carry ${revision}; they stop being snapshots once the check above passes. OWN="$(echo "$SNAPS" | grep -c '^io\.ignifyr:' || true)" @@ -101,12 +101,12 @@ else fi # ---- 2. attribution inside the shaded jars ----------------------------------- -log "2. Both jars carry aggregated NOTICEs and Ignifyr's own LICENSE" +log "2. The jar carries aggregated NOTICEs and Ignifyr's own LICENSE" REPO_LICENSE_BYTES="$(wc -c < LICENSE | tr -d ' ')" # ~75 bundled jars ship a NOTICE; one un-merged copy is a few hundred bytes, the merged file tens of # kB. The floor catches an attribution transformer silently dropping out of the shade config. NOTICE_MIN_BYTES=10000 -for pair in "community:$CLI_JAR" "server:$SRV_JAR"; do +for pair in "community:$CLI_JAR"; do name="${pair%%:*}"; jar="${pair#*:}" n="$(jar_read "$jar" META-INF/NOTICE | wc -c | tr -d ' ')" if [ "${n:-0}" -ge "$NOTICE_MIN_BYTES" ]; then @@ -203,19 +203,8 @@ else warn "python not found; skipping the declared-license scan (coordinate denylist still ran)" fi -# ---- 4. the credential-carrying module ships nowhere ------------------------- -log "4. ignifyr-terminology-tools is in neither distribution" -for pair in "community:$CLI_JAR" "server:$SRV_JAR"; do - name="${pair%%:*}"; jar="${pair#*:}" - if jar_list "$jar" | grep -q '^io/ignifyr/terminology/'; then - bad "$name jar bundles ignifyr-terminology-tools, which embeds hard-coded dev Postgres credentials" - else - ok "$name jar excludes ignifyr-terminology-tools" - fi -done - -# ---- 5. release hygiene ------------------------------------------------------ -log "5. Release hygiene" +# ---- 4. release hygiene ------------------------------------------------------ +log "4. Release hygiene" if [ -z "$(git status --porcelain 2>/dev/null)" ]; then ok "working tree is clean" else diff --git a/test-flow/config/ignifyr-server.conf b/test-flow/config/ignifyr-server.conf deleted file mode 100644 index edb91c17..00000000 --- a/test-flow/config/ignifyr-server.conf +++ /dev/null @@ -1,85 +0,0 @@ -# Ignifyr server config for the manual test-flow stack. -# Workspace-rooted: all repositories, the db folder, and Spark checkpoints live under /workspace -# (a bind mount), and the FHIR-definitions endpoint points at the repofyr container. -ignifyr { - # Base for relative file-system readings (job dataFolderPath, terminology folders). - context-path = "/workspace/" - - mappings = { - repository = { folder-path = "/workspace/mappings" } - schemas = { - repository = { folder-path = "/workspace/schemas" } - fhir-version = "5.0.0" - } - contexts = { repository = { folder-path = "/workspace/mapping-contexts" } } - timeout = 30 seconds - } - - mapping-jobs = { - repository = { folder-path = "/workspace/mapping-jobs" } - } - - terminology-systems = { folder-path = "/workspace/terminology-systems" } - - archiving = { - erroneous-records-folder = "/workspace/erroneous-records-folder" - archive-folder = "/workspace/archive-folder" - stream-archiving-frequency = 5000 - } - - fhir-server-writer { batch-group-size = 50 } - - db-path = "/workspace/ignifyr-db" - - # Keep only the custom mapping functions library (no external RxNorm calls during the flow). - functionLibraries { - cst { className = "io.ignifyr.common.util.CustomMappingFunctionsFactory" } - } -} - -fhir = { - fhir-version = "R5" - definitions-root-urls = null - # Use the server's bundled base FHIR R5 definitions. Setting a definitions-fhir-endpoint here would - # switch to "read definitions from a remote FHIR server" mode, which additionally REQUIRES - # definitions-root-urls to be set, and aborts startup otherwise. The flow's mapping uses the - # Ext-patient schema from the schema repository, so no remote definitions source is needed. - definitions-fhir-endpoint = null - fhir-endpoint-auth = { method = null } - context-path = "/workspace/" - profiles-path = null - valuesets-path = null - codesystems-path = null -} - -webserver = { - host = 0.0.0.0 - port = 8085 - base-uri = ignifyr - ssl { keystore = null, password = null } -} - -spark = { - app.name = "Ignifyr Test-Flow Server" - master = "local[4]" - ui.enabled = "false" - checkpoint-dir = "/workspace/checkpoint" -} - -akka = { - daemonic = "on" - http = { - parsing = { max-header-value-length = 82k } - server = { - server-header = Ignifyr Server - parsing { uri-parsing-mode = relaxed, max-content-length = 8m } - request-timeout = 120 s - idle-timeout = 120 s - remote-address-header = on - } - } -} - -# NOTE: the `ignifyr-redcap { }` block is intentionally omitted. Its presence would add the /redcap -# proxy routes (and a startup probe against the external tofhir-redcap service). This flow simulates -# REDCap by publishing directly to Kafka, so the proxy is not needed. diff --git a/test-flow/config/jobs/archive-file-job.json b/test-flow/config/jobs/archive-file-job.json deleted file mode 100644 index c5e3b5c8..00000000 --- a/test-flow/config/jobs/archive-file-job.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "id": "archive-file-job", - "sourceSettings": { - "source": { - "jsonClass": "FileSystemSourceSettings", - "name": "archive-source", - "sourceUri": "https://ignifyr.io/test-flow/archive", - "dataFolderPath": "/workspace/archive-input" - } - }, - "sinkSettings": { - "jsonClass": "FhirRepositorySinkSettings", - "fhirRepoUrl": "http://repofyr:8080/fhir" - }, - "mappings": [ - { - "name": "patient-mapping", - "mappingRef": "https://aiccelerate.eu/fhir/mappings/patient-mapping", - "sourceBinding": { - "source": { - "jsonClass": "FileSystemSource", - "path": "patients.csv", - "contentType": "csv" - } - } - } - ], - "dataProcessingSettings": { - "saveErroneousRecords": false, - "archiveMode": "archive" - } -} diff --git a/test-flow/config/jobs/batch-file-job.json b/test-flow/config/jobs/batch-file-job.json deleted file mode 100644 index ddd2c84a..00000000 --- a/test-flow/config/jobs/batch-file-job.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "id": "batch-file-job", - "sourceSettings": { - "source": { - "jsonClass": "FileSystemSourceSettings", - "name": "batch-source", - "sourceUri": "https://ignifyr.io/test-flow/batch", - "dataFolderPath": "/workspace/data" - } - }, - "sinkSettings": { - "jsonClass": "FhirRepositorySinkSettings", - "fhirRepoUrl": "http://repofyr:8080/fhir" - }, - "mappings": [ - { - "name": "patient-mapping", - "mappingRef": "https://aiccelerate.eu/fhir/mappings/patient-mapping", - "sourceBinding": { - "source": { - "jsonClass": "FileSystemSource", - "path": "patients.csv", - "contentType": "csv" - } - } - } - ], - "dataProcessingSettings": { - "saveErroneousRecords": false, - "archiveMode": "off" - } -} diff --git a/test-flow/config/jobs/scheduling-file-job.json b/test-flow/config/jobs/scheduling-file-job.json deleted file mode 100644 index 861623be..00000000 --- a/test-flow/config/jobs/scheduling-file-job.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "id": "scheduling-file-job", - "schedulingSettings": { - "jsonClass": "SchedulingSettings", - "cronExpression": "* * * * *" - }, - "sourceSettings": { - "source": { - "jsonClass": "FileSystemSourceSettings", - "name": "scheduled-source", - "sourceUri": "https://ignifyr.io/test-flow/scheduled", - "dataFolderPath": "/workspace/data" - } - }, - "sinkSettings": { - "jsonClass": "FhirRepositorySinkSettings", - "fhirRepoUrl": "http://repofyr:8080/fhir" - }, - "mappings": [ - { - "name": "patient-mapping", - "mappingRef": "https://aiccelerate.eu/fhir/mappings/patient-mapping", - "sourceBinding": { - "source": { - "jsonClass": "FileSystemSource", - "path": "patients.csv", - "contentType": "csv" - } - } - } - ], - "dataProcessingSettings": { - "saveErroneousRecords": false, - "archiveMode": "off" - } -} diff --git a/test-flow/config/jobs/sql-patient-job.json b/test-flow/config/jobs/sql-patient-job.json deleted file mode 100644 index e1cddb50..00000000 --- a/test-flow/config/jobs/sql-patient-job.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "sql-patient-job", - "sourceSettings": { - "source": { - "jsonClass": "SqlSourceSettings", - "name": "sql-source", - "sourceUri": "https://ignifyr.io/test-flow/sql", - "databaseUrl": "jdbc:postgresql://postgres:5432/ignifyr", - "username": "ignifyr", - "password": "ignifyr" - } - }, - "sinkSettings": { - "jsonClass": "FhirRepositorySinkSettings", - "fhirRepoUrl": "http://repofyr:8080/fhir" - }, - "mappings": [ - { - "name": "patient-mapping", - "mappingRef": "https://aiccelerate.eu/fhir/mappings/patient-mapping", - "sourceBinding": { - "source": { - "jsonClass": "SqlSource", - "tableName": "patients" - } - } - } - ], - "dataProcessingSettings": { - "saveErroneousRecords": false, - "archiveMode": "off" - } -} diff --git a/test-flow/create-ui-testflow.sh b/test-flow/create-ui-testflow.sh deleted file mode 100644 index 699849e8..00000000 --- a/test-flow/create-ui-testflow.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bash -# -# Create a "test-flow" project in the RUNNING ignifyr-server via its REST API, so a SQL job (Postgres -# 'patients' table) and a Kafka job (redcap-patients topic) show up in the web UI and can be run and -# observed from there. Idempotent-ish: re-running reports 409 (already exists) and continues. -# -# Prereq: the manual stack is up (test-flow/run-manual-flow.sh --with-web [--with-efk]) so the server, -# Postgres and Kafka containers are running. Needs curl. -# -# Usage: test-flow/create-ui-testflow.sh -set -uo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -BASE="http://localhost:8085/ignifyr" -PROJECT="test-flow" - -SCHEMA_FILE="$REPO_ROOT/ignifyr-testkit/src/main/resources/test-schemas/some-folder-1/Ext-patient.StructureDefinition.json" -MAPPING_FILE="$REPO_ROOT/ignifyr-testkit/src/main/resources/test-mappings/some-folder-1/patient-mapping.json" -TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT -FAILC=0 - -command -v curl >/dev/null 2>&1 || { echo "curl required" >&2; exit 1; } -curl -fsS -X OPTIONS "$BASE/projects" >/dev/null 2>&1 || { echo "ignifyr-server not reachable at $BASE — bring the stack up first" >&2; exit 1; } - -# POST