diff --git a/server.js b/server.js index 14cb176..498e898 100644 --- a/server.js +++ b/server.js @@ -568,6 +568,19 @@ async function startServer() { if (modules.packages && config.modules.packages.enabled) { modules.packages.startInitialCrawler(); } + // Run usage tracker in background after startup + if (modules.tx && modules.tx.library) { + setImmediate(async () => { + try { + serverLog.info('Starting ConceptUsageTracker scan...'); + let count = await modules.tx.usageTracker.scanValueSets(modules.tx.library); + serverLog.info(`ConceptUsageTracker scan complete: ${count} valuesets list codes`); + } catch (err) { + console.log(err); + serverLog.error('ConceptUsageTracker scan failed:', err); + } + }); + } } catch (error) { console.error('FATAL - Failed to start server:', error); serverLog.error('FATAL - Failed to start server:', error); diff --git a/tests/cs/cs-areacode.test.js b/tests/cs/cs-areacode.test.js index 71bde13..43d1700 100644 --- a/tests/cs/cs-areacode.test.js +++ b/tests/cs/cs-areacode.test.js @@ -266,7 +266,7 @@ describe('AreaCodeServices', () => { test('should throw error for search filter', async () => { await expect( provider.searchFilter(await provider.getPrepContext(false), 'test', false) - ).rejects.toThrow('not implemented'); + ).rejects.toThrow('Text Search is not supported'); }); }); diff --git a/tests/cs/cs-country.test.js b/tests/cs/cs-country.test.js index 0795fc2..94cd326 100644 --- a/tests/cs/cs-country.test.js +++ b/tests/cs/cs-country.test.js @@ -403,7 +403,7 @@ describe('CountryCodeServices', () => { test('should throw error for search filter', async () => { await expect( provider.searchFilter(await provider.getPrepContext(false), 'test', false) - ).rejects.toThrow('not implemented'); + ).rejects.toThrow('Text Search is not supported'); }); }); diff --git a/tests/cs/cs-cpt.test.js b/tests/cs/cs-cpt.test.js index ad18101..9a5adea 100644 --- a/tests/cs/cs-cpt.test.js +++ b/tests/cs/cs-cpt.test.js @@ -635,7 +635,7 @@ describe('CPT Provider', () => { await expect( provider.searchFilter(filterContext, 'test', false) - ).rejects.toThrow('not implemented'); + ).rejects.toThrow('Text Search is not supported'); }); }); }); \ No newline at end of file diff --git a/tests/cs/cs-cs.test.js b/tests/cs/cs-cs.test.js index 39029c1..f8ad39c 100644 --- a/tests/cs/cs-cs.test.js +++ b/tests/cs/cs-cs.test.js @@ -5,7 +5,7 @@ const {CodeSystem} = require('../../tx/library/codesystem'); const {FhirCodeSystemFactory, FhirCodeSystemProvider, FhirCodeSystemProviderContext} = require('../../tx/cs/cs-cs'); const {Languages, Language} = require('../../library/languages'); const {OperationContext} = require("../../tx/operation-context"); -const {Designations} = require("../../tx/library/designations"); +const {Designations, SearchFilterText} = require("../../tx/library/designations"); const {TestUtilities} = require("../test-utilities"); describe('FHIR CodeSystem Provider', () => { @@ -1416,7 +1416,7 @@ describe('FHIR CodeSystem Provider', () => { describe('Search Filter', () => { test('should find concepts by exact code match', async () => { - const results = await simpleProvider.searchFilter(filterContext, 'code1', true); + const results = await simpleProvider.searchFilter(filterContext, new SearchFilterText('code1'), true); expect(results.size()).toBeGreaterThan(0); const concept = results.findConceptByCode('code1'); @@ -1425,7 +1425,7 @@ describe('FHIR CodeSystem Provider', () => { }); test('should find concepts by display text match', async () => { - const results = await simpleProvider.searchFilter(filterContext, 'Display 1', true); + const results = await simpleProvider.searchFilter(filterContext, new SearchFilterText('Display 1'), true); expect(results.size()).toBeGreaterThan(0); const concept = results.findConceptByCode('code1'); @@ -1433,22 +1433,22 @@ describe('FHIR CodeSystem Provider', () => { }); test('should find concepts by partial match', async () => { - const results = await simpleProvider.searchFilter(filterContext, 'Display', true); + const results = await simpleProvider.searchFilter(filterContext, new SearchFilterText('Display'), true); expect(results.size()).toBeGreaterThan(1); // Should find multiple concepts }); test('should find concepts by definition match', async () => { - const results = await simpleProvider.searchFilter(filterContext, 'first', true); + const results = await simpleProvider.searchFilter(filterContext, new SearchFilterText('first'), true); expect(results.size()).toBeGreaterThan(0); }); test('should return empty results for non-matching search', async () => { - const results = await simpleProvider.searchFilter(filterContext, 'nonexistent', true); + const results = await simpleProvider.searchFilter(filterContext, new SearchFilterText('nonexistent'), true); expect(results.size()).toBe(0); }); test('should sort results by relevance when requested', async () => { - const results = await simpleProvider.searchFilter(filterContext, 'code', true); + const results = await simpleProvider.searchFilter(filterContext, new SearchFilterText('code'), true); expect(results.size()).toBeGreaterThan(1); // Results should be sorted by rating (exact matches first) @@ -1683,7 +1683,7 @@ describe('FHIR CodeSystem Provider', () => { describe('Complex Filter Scenarios', () => { test('should work with German CodeSystem', async () => { - const results = await deProvider.searchFilter(filterContext, 'Anzeige', true); + const results = await deProvider.searchFilter(filterContext, new SearchFilterText('Anzeige'), true); expect(results.size()).toBeGreaterThan(0); }); diff --git a/tests/cs/cs-currency.test.js b/tests/cs/cs-currency.test.js index 8f7edad..86ef4ec 100644 --- a/tests/cs/cs-currency.test.js +++ b/tests/cs/cs-currency.test.js @@ -201,7 +201,7 @@ describe('Iso4217Services', () => { const ctxt = await provider.getPrepContext(false); await expect( provider.searchFilter(ctxt, 'dollar', false) - ).rejects.toThrow('not implemented'); + ).rejects.toThrow('Text Search is not supported'); }); test('should throw error for unsupported filter', async () => { diff --git a/tests/cs/cs-hgvs.test.js b/tests/cs/cs-hgvs.test.js index 05a5c0b..1c622ea 100644 --- a/tests/cs/cs-hgvs.test.js +++ b/tests/cs/cs-hgvs.test.js @@ -243,7 +243,7 @@ describe('HGVS Provider', () => { test('should throw errors for filter operations', async () => { await expect(provider.getPrepContext(true)).rejects.toThrow('not supported for HGVS'); - await expect(provider.searchFilter(null, 'filter', false)).rejects.toThrow('not supported for HGVS'); + await expect(provider.searchFilter(null, 'filter', false)).rejects.toThrow('Text Search is not supported'); await expect(provider.filter(null, 'prop', 'equal', 'value')).rejects.toThrow('not supported for HGVS'); await expect(provider.prepare(null)).rejects.toThrow('not supported for HGVS'); await expect(provider.executeFilters(null)).rejects.toThrow('not supported for HGVS'); diff --git a/tests/cs/cs-lang.test.js b/tests/cs/cs-lang.test.js index 6370b9c..45872d7 100644 --- a/tests/cs/cs-lang.test.js +++ b/tests/cs/cs-lang.test.js @@ -343,7 +343,7 @@ describe('IETF Language CodeSystem Provider', () => { test('should not support text search', async () => { await expect( provider.searchFilter(new FilterExecutionContext(), 'english', false) - ).rejects.toThrow('Text search not supported'); + ).rejects.toThrow('Text Search is not supported'); }); test('should indicate filters are not closed', async () => { diff --git a/tests/cs/cs-omop.test.js b/tests/cs/cs-omop.test.js index ec9240a..e868009 100644 --- a/tests/cs/cs-omop.test.js +++ b/tests/cs/cs-omop.test.js @@ -506,7 +506,7 @@ describe('OMOP Provider', () => { await expect( provider.searchFilter(filterContext, 'test', false) - ).rejects.toThrow('not implemented'); + ).rejects.toThrow('Text Search is not supported'); }); }); diff --git a/tests/tx/test-cases.test.js b/tests/tx/test-cases.test.js index 0545d84..5d4f6b5 100644 --- a/tests/tx/test-cases.test.js +++ b/tests/tx/test-cases.test.js @@ -3881,6 +3881,67 @@ describe('exclude', () => { await runTest({"suite":"exclude","test":"exclude-zero"}, "4.0"); }); + it('exclude-allR5', async () => { + await runTest({"suite":"exclude","test":"exclude-all"}, "5.0"); + }); + + it('exclude-allR4', async () => { + await runTest({"suite":"exclude","test":"exclude-all"}, "4.0"); + }); + +}); + +describe('search', () => { + // Tests for proper functioning of text search. Note what we're not interested in the implementation of the text search itself, so we only test very obvious results. We're just interested in testing support for the parameter + + it('search-all-yesR5', async () => { + await runTest({"suite":"search","test":"search-all-yes"}, "5.0"); + }); + + it('search-all-yesR4', async () => { + await runTest({"suite":"search","test":"search-all-yes"}, "4.0"); + }); + + it('search-all-noR5', async () => { + await runTest({"suite":"search","test":"search-all-no"}, "5.0"); + }); + + it('search-all-noR4', async () => { + await runTest({"suite":"search","test":"search-all-no"}, "4.0"); + }); + + it('search-filter-yesR5', async () => { + await runTest({"suite":"search","test":"search-filter-yes"}, "5.0"); + }); + + it('search-filter-yesR4', async () => { + await runTest({"suite":"search","test":"search-filter-yes"}, "4.0"); + }); + + it('search-filter-noR5', async () => { + await runTest({"suite":"search","test":"search-filter-no"}, "5.0"); + }); + + it('search-filter-noR4', async () => { + await runTest({"suite":"search","test":"search-filter-no"}, "4.0"); + }); + + it('search-enum-yesR5', async () => { + await runTest({"suite":"search","test":"search-enum-yes"}, "5.0"); + }); + + it('search-enum-yesR4', async () => { + await runTest({"suite":"search","test":"search-enum-yes"}, "4.0"); + }); + + it('search-enum-noR5', async () => { + await runTest({"suite":"search","test":"search-enum-no"}, "5.0"); + }); + + it('search-enum-noR4', async () => { + await runTest({"suite":"search","test":"search-enum-no"}, "4.0"); + }); + }); describe('default-valueset-version', () => { @@ -4440,6 +4501,102 @@ describe('snomed', () => { await runTest({"suite":"snomed","test":"snomed-expand-procedures"}, "4.0"); }); + it('lookupR5', async () => { + await runTest({"suite":"snomed","test":"lookup"}, "5.0"); + }); + + it('lookupR4', async () => { + await runTest({"suite":"snomed","test":"lookup"}, "4.0"); + }); + + it('lookup-pcR5', async () => { + await runTest({"suite":"snomed","test":"lookup-pc"}, "5.0"); + }); + + it('lookup-pcR4', async () => { + await runTest({"suite":"snomed","test":"lookup-pc"}, "4.0"); + }); + + it('validate-code-pc-goodR5', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-good"}, "5.0"); + }); + + it('validate-code-pc-goodR4', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-good"}, "4.0"); + }); + + it('validate-code-pc-bad1R5', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-bad1"}, "5.0"); + }); + + it('validate-code-pc-bad1R4', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-bad1"}, "4.0"); + }); + + it('validate-code-pc-bad2R5', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-bad2"}, "5.0"); + }); + + it('validate-code-pc-bad2R4', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-bad2"}, "4.0"); + }); + + it('validate-code-pc-noneR5', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-none"}, "5.0"); + }); + + it('validate-code-pc-noneR4', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-none"}, "4.0"); + }); + + it('validate-code-pc-listR5', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-list"}, "5.0"); + }); + + it('validate-code-pc-listR4', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-list"}, "4.0"); + }); + + it('validate-code-pc-list-no-pcR5', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-list-no-pc"}, "5.0"); + }); + + it('validate-code-pc-list-no-pcR4', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-list-no-pc"}, "4.0"); + }); + + it('validate-code-pc-filterR5', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-filter"}, "5.0"); + }); + + it('validate-code-pc-filterR4', async () => { + await runTest({"suite":"snomed","test":"validate-code-pc-filter"}, "4.0"); + }); + + it('expand-pc-noneR5', async () => { + await runTest({"suite":"snomed","test":"expand-pc-none"}, "5.0"); + }); + + it('expand-pc-noneR4', async () => { + await runTest({"suite":"snomed","test":"expand-pc-none"}, "4.0"); + }); + + it('expand-pc-listR5', async () => { + await runTest({"suite":"snomed","test":"expand-pc-list"}, "5.0"); + }); + + it('expand-pc-listR4', async () => { + await runTest({"suite":"snomed","test":"expand-pc-list"}, "4.0"); + }); + + it('expand-pc-filterR5', async () => { + await runTest({"suite":"snomed","test":"expand-pc-filter"}, "5.0"); + }); + + it('expand-pc-filterR4', async () => { + await runTest({"suite":"snomed","test":"expand-pc-filter"}, "4.0"); + }); + }); describe('batch', () => { @@ -4745,6 +4902,235 @@ describe('UCUM', () => { }); +describe('related', () => { + // Tests for candidate new 'related' operation + + it('related-allR5', async () => { + await runTest({"suite":"related","test":"related-all"}, "5.0"); + }); + + it('related-allR4', async () => { + await runTest({"suite":"related","test":"related-all"}, "4.0"); + }); + + it('related-activeR5', async () => { + await runTest({"suite":"related","test":"related-active"}, "5.0"); + }); + + it('related-activeR4', async () => { + await runTest({"suite":"related","test":"related-active"}, "4.0"); + }); + + it('related-inactiveR5', async () => { + await runTest({"suite":"related","test":"related-inactive"}, "5.0"); + }); + + it('related-inactiveR4', async () => { + await runTest({"suite":"related","test":"related-inactive"}, "4.0"); + }); + + it('related-enumeratedR5', async () => { + await runTest({"suite":"related","test":"related-enumerated"}, "5.0"); + }); + + it('related-enumeratedR4', async () => { + await runTest({"suite":"related","test":"related-enumerated"}, "4.0"); + }); + + it('related-is-aR5', async () => { + await runTest({"suite":"related","test":"related-is-a"}, "5.0"); + }); + + it('related-is-aR4', async () => { + await runTest({"suite":"related","test":"related-is-a"}, "4.0"); + }); + + it('related-regex-1R5', async () => { + await runTest({"suite":"related","test":"related-regex-1"}, "5.0"); + }); + + it('related-regex-1R4', async () => { + await runTest({"suite":"related","test":"related-regex-1"}, "4.0"); + }); + + it('related-regex-2R5', async () => { + await runTest({"suite":"related","test":"related-regex-2"}, "5.0"); + }); + + it('related-regex-2R4', async () => { + await runTest({"suite":"related","test":"related-regex-2"}, "4.0"); + }); + + it('related-listsR5', async () => { + await runTest({"suite":"related","test":"related-lists"}, "5.0"); + }); + + it('related-listsR4', async () => { + await runTest({"suite":"related","test":"related-lists"}, "4.0"); + }); + + it('related-lists-moreR5', async () => { + await runTest({"suite":"related","test":"related-lists-more"}, "5.0"); + }); + + it('related-lists-moreR4', async () => { + await runTest({"suite":"related","test":"related-lists-more"}, "4.0"); + }); + + it('related-lists-lessR5', async () => { + await runTest({"suite":"related","test":"related-lists-less"}, "5.0"); + }); + + it('related-lists-lessR4', async () => { + await runTest({"suite":"related","test":"related-lists-less"}, "4.0"); + }); + + it('related-lists-overR5', async () => { + await runTest({"suite":"related","test":"related-lists-over"}, "5.0"); + }); + + it('related-lists-overR4', async () => { + await runTest({"suite":"related","test":"related-lists-over"}, "4.0"); + }); + + it('related-lists-disjR5', async () => { + await runTest({"suite":"related","test":"related-lists-disj"}, "5.0"); + }); + + it('related-lists-disjR4', async () => { + await runTest({"suite":"related","test":"related-lists-disj"}, "4.0"); + }); + + it('related-systemsR5', async () => { + await runTest({"suite":"related","test":"related-systems"}, "5.0"); + }); + + it('related-systemsR4', async () => { + await runTest({"suite":"related","test":"related-systems"}, "4.0"); + }); + + it('related-systemsR5', async () => { + await runTest({"suite":"related","test":"related-systems"}, "5.0"); + }); + + it('related-systemsR4', async () => { + await runTest({"suite":"related","test":"related-systems"}, "4.0"); + }); + + it('related-systems-lessR5', async () => { + await runTest({"suite":"related","test":"related-systems-less"}, "5.0"); + }); + + it('related-systems-lessR4', async () => { + await runTest({"suite":"related","test":"related-systems-less"}, "4.0"); + }); + + it('related-systems-moreR5', async () => { + await runTest({"suite":"related","test":"related-systems-more"}, "5.0"); + }); + + it('related-systems-moreR4', async () => { + await runTest({"suite":"related","test":"related-systems-more"}, "4.0"); + }); + + it('related-system-disjR5', async () => { + await runTest({"suite":"related","test":"related-system-disj"}, "5.0"); + }); + + it('related-system-disjR4', async () => { + await runTest({"suite":"related","test":"related-system-disj"}, "4.0"); + }); + + it('related-system-overR5', async () => { + await runTest({"suite":"related","test":"related-system-over"}, "5.0"); + }); + + it('related-system-overR4', async () => { + await runTest({"suite":"related","test":"related-system-over"}, "4.0"); + }); + + it('related-filters-1R5', async () => { + await runTest({"suite":"related","test":"related-filters-1"}, "5.0"); + }); + + it('related-filters-1R4', async () => { + await runTest({"suite":"related","test":"related-filters-1"}, "4.0"); + }); + + it('related-filters-2R5', async () => { + await runTest({"suite":"related","test":"related-filters-2"}, "5.0"); + }); + + it('related-filters-2R4', async () => { + await runTest({"suite":"related","test":"related-filters-2"}, "4.0"); + }); + + it('related-filters-3R5', async () => { + await runTest({"suite":"related","test":"related-filters-3"}, "5.0"); + }); + + it('related-filters-3R4', async () => { + await runTest({"suite":"related","test":"related-filters-3"}, "4.0"); + }); + + it('related-mixed-1R5', async () => { + await runTest({"suite":"related","test":"related-mixed-1"}, "5.0"); + }); + + it('related-mixed-1R4', async () => { + await runTest({"suite":"related","test":"related-mixed-1"}, "4.0"); + }); + + it('related-mixed-1-lessR5', async () => { + await runTest({"suite":"related","test":"related-mixed-1-less"}, "5.0"); + }); + + it('related-mixed-1-lessR4', async () => { + await runTest({"suite":"related","test":"related-mixed-1-less"}, "4.0"); + }); + + it('related-mixed-1-moreR5', async () => { + await runTest({"suite":"related","test":"related-mixed-1-more"}, "5.0"); + }); + + it('related-mixed-1-moreR4', async () => { + await runTest({"suite":"related","test":"related-mixed-1-more"}, "4.0"); + }); + + it('related-mixed-1-disjR5', async () => { + await runTest({"suite":"related","test":"related-mixed-1-disj"}, "5.0"); + }); + + it('related-mixed-1-disjR4', async () => { + await runTest({"suite":"related","test":"related-mixed-1-disj"}, "4.0"); + }); + + it('related-mixed-1-overR5', async () => { + await runTest({"suite":"related","test":"related-mixed-1-over"}, "5.0"); + }); + + it('related-mixed-1-overR4', async () => { + await runTest({"suite":"related","test":"related-mixed-1-over"}, "4.0"); + }); + + it('related-filters-lessR5', async () => { + await runTest({"suite":"related","test":"related-filters-less"}, "5.0"); + }); + + it('related-filters-lessR4', async () => { + await runTest({"suite":"related","test":"related-filters-less"}, "4.0"); + }); + + it('related-filters-moreR5', async () => { + await runTest({"suite":"related","test":"related-filters-more"}, "5.0"); + }); + + it('related-filters-moreR4', async () => { + await runTest({"suite":"related","test":"related-filters-more"}, "4.0"); + }); + +}); + describe('bugs', () => { // A series of tests that deal with discovered bugs in FHIRsmith. These tests are specific to FHIRsmith - internal QA diff --git a/tx/cs/cs-api.js b/tx/cs/cs-api.js index ea6a050..aa35d58 100644 --- a/tx/cs/cs-api.js +++ b/tx/cs/cs-api.js @@ -30,7 +30,7 @@ class CodeSystemProvider { */ supplements; - constructor(opContext, supplements) { + constructor(opContext, supplements = null) { this.opContext = opContext; this.supplements = supplements; this._ensureOpContext(opContext); @@ -488,6 +488,39 @@ class CodeSystemProvider { // procedure getCDSInfo(card : TCDSHookCard; langList : THTTPLanguageList; baseURL, code, display : String); virtual; + /** + * There are two models for handling concepts and filters. The first is where the logic is entirely + * handled by worker classes; this is needed for value sets that select codes across systems, and + * with references to other value sets. This workflow consists of calling GetPrepContext, followed + * by some combination of filter+searchFilter, and then executeFilters + * + * followed by filterMore/filterConcept. All code system providers have to support this workflow + * + * But an important subset of value sets simply select codes from one codeSystem, from large + * code systems. Such processing can be done much more efficiently by the code system provider. + * providers that do this should return handlesSelecting() = true, and then for suitable valuesets, + * the method processSelection() will be called + */ + handlesSelecting() { + return false; + } + + /** + * Process a set of includes and excludes for the code system + * + * @param {TxParameters} params: information from the request that the user made, to help optimise loading + * @param {Object[]} includes - a list of includes from the code system. Each include may contain just the system(+version), concepts and/or filters (but won't contain value sets) + * @param {Object[]} excludes - a list of excludes from the code system. Each include may contain just the system(+version), concepts and/or filters (but won't contain value sets) + * @param {boolean} excludeInactive: whether the server will use inactive codes or not + * @param {int} offset if handlesOffset() and !iterate, and if the value set is a simple one that only uses this provider, then this is the applicable offset. -1 if not applicable + * @param {int} count if handlesOffset() and !iterate, and if the value set is a simple one that only uses this provider, then this is the applicable count. -1 if not applicable + * @returns {FilterConceptSet[]} filter sets. In general, it wouldn't make sense to return more than one, but providers can do if they want to. See futher comments on executeFilters + */ + processSelection(params, includes, excludes, excludeInactive, offset, count) { + // well, you only need to override if handlesSelecting=true, but that's the only time this will be called + throw new Error("Must override"); + } + /** * returns true if a filter is supported * @@ -499,14 +532,16 @@ class CodeSystemProvider { async doesFilter(prop, op, value) { return false; } /** - * gets a single context in which filters will be evaluated. The application doesn't make use of this context; - * it's only use is to be passed back to the CodeSystem provider so it can make use of it - if it wants + * gets a single context in which filters will be evaluated. The server doesn't doesn't make use of this context; + * it's only use is to be passed back to the CodeSystem provider so it can make use of it to organise the filter process * * @param {boolean} iterate true if the conceptSets that result from this will be iterated, and false if they'll be used to locate a single code - * @returns {FilterExecutionContext} filter (or null, it no use for this) - * */ + * @returns {FilterExecutionContext} filter + * + **/ async getPrepContext(iterate) { return new FilterExecutionContext(iterate); } + /** * executes a text search filter (whatever that means) and returns a FilterConceptSet * @@ -516,7 +551,7 @@ class CodeSystemProvider { * @param {String} filter user entered text search * @param {boolean} sort ? **/ - async searchFilter(filterContext, filter, sort) { throw new Error("Must override"); } // ? must override? + async searchFilter(filterContext, filter, sort) { throw new Error("Text Search is not supported"); } // ? must override? /** * Used for searching ucum (see specialEnumeration) @@ -532,7 +567,7 @@ class CodeSystemProvider { } // ? must override? /** - * Get a FilterConceptSet for a value set filter + * inform the CS provider about a filter * * throws an exception if the search filter can't be handled * @@ -549,6 +584,9 @@ class CodeSystemProvider { * one FilterConceptSet, then the code system provider has done the join across the * filters, otherwise the engine will do so as required * + * The first in the set of returned FilterConceptSet is used for iterating; other + * FilterConceptSets are used for filterCheck(); + * * @param {FilterExecutionContext} filterContext filtering context * @returns {FilterConceptSet[]} filter sets **/ @@ -679,7 +717,22 @@ class CodeSystemProvider { return null; } - + /** + * a record of observed usages of codes from this code system + * - a map of code and object which has count, an integer count + * of frequency of use (this server iteration, for now) + * + * Only populated when expanding, and read-only to the CS Provider + * + * @type {Map} + */ + usages() { + if (this.usagesObj == undefined) { + this.usagesObj = this.opContext.usageTracker ? this.opContext.usageTracker.usages(this.system()) : null; + } + return this.usagesObj; + } + usagesObj = undefined; } class CodeSystemFactoryProvider { diff --git a/tx/cs/cs-areacode.js b/tx/cs/cs-areacode.js index b636f00..ee5c9ea 100644 --- a/tx/cs/cs-areacode.js +++ b/tx/cs/cs-areacode.js @@ -169,15 +169,6 @@ class AreaCodeServices extends CodeSystemProvider { return (prop === 'type' || prop === 'class') && op === '='; } - async searchFilter(filterContext, filter, sort) { - - assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); - assert(filter && typeof filter === 'string', 'filter must be a non-null string'); - assert(typeof sort === 'boolean', 'sort must be a boolean'); - - throw new Error('Search filter not implemented for AreaCode'); - } - async filter(filterContext, prop, op, value) { assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); diff --git a/tx/cs/cs-country.js b/tx/cs/cs-country.js index 2f690a0..fde6bd8 100644 --- a/tx/cs/cs-country.js +++ b/tx/cs/cs-country.js @@ -179,15 +179,6 @@ class CountryCodeServices extends CodeSystemProvider { } - async searchFilter(filterContext, filter, sort) { - - assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); - assert(filter && typeof filter === 'string', 'filter must be a non-null string'); - assert(typeof sort === 'boolean', 'sort must be a boolean'); - - throw new Error('Search filter not implemented for CountryCode'); - } - async filter(filterContext, prop, op, value) { assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); diff --git a/tx/cs/cs-cpt.js b/tx/cs/cs-cpt.js index 62e6b25..d5c3c96 100644 --- a/tx/cs/cs-cpt.js +++ b/tx/cs/cs-cpt.js @@ -569,12 +569,7 @@ class CPTServices extends BaseCSServices { return filterContext.filters.some(f => !f.closed); } - // Search filter - not implemented - // eslint-disable-next-line no-unused-vars - async searchFilter(filterContext, filter, sort) { - - throw new Error('Text search not implemented yet'); - } + // Subsumption testing - not implemented async subsumesTest(codeA, codeB) { diff --git a/tx/cs/cs-cs.js b/tx/cs/cs-cs.js index f9f8707..f5177ff 100644 --- a/tx/cs/cs-cs.js +++ b/tx/cs/cs-cs.js @@ -109,7 +109,6 @@ class FhirCodeSystemProvider extends BaseCSServices { */ constructor(opContext, codeSystem, supplements) { super(opContext, supplements); - if (codeSystem.content == 'supplements') { throw new Issue('error', 'invalid', null, 'CODESYSTEM_CS_NO_SUPPLEMENT', opContext.i18n.translate('CODESYSTEM_CS_NO_SUPPLEMENT', opContext.langs, codeSystem.vurl)); } @@ -1134,7 +1133,7 @@ class FhirCodeSystemProvider extends BaseCSServices { const results = new FhirCodeSystemProviderFilterContext(); - const searchTerm = filter.toLowerCase(); + const searchTerm = filter.filter.toLowerCase(); // Search through all concepts const allConcepts = this.codeSystem.getAllConcepts(); diff --git a/tx/cs/cs-currency.js b/tx/cs/cs-currency.js index c63c2c1..e07fc36 100644 --- a/tx/cs/cs-currency.js +++ b/tx/cs/cs-currency.js @@ -176,15 +176,6 @@ class Iso4217Services extends CodeSystemProvider { return prop === 'decimals' && op === 'equals'; } - async searchFilter(filterContext, filter, sort) { - - assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); - assert(filter && typeof filter === 'string', 'filter must be a non-null string'); - assert(typeof sort === 'boolean', 'sort must be a boolean'); - - throw new Error('Search filter not implemented for ISO 4217'); - } - async filter(filterContext, prop, op, value) { assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); diff --git a/tx/cs/cs-db.js b/tx/cs/cs-db.js deleted file mode 100644 index 10317f0..0000000 --- a/tx/cs/cs-db.js +++ /dev/null @@ -1,1308 +0,0 @@ -const sqlite3 = require('sqlite3').verbose(); -const assert = require('assert'); -const { CodeSystem } = require('../library/codesystem'); -const { Language, Languages} = require('../../library/languages'); -const { CodeSystemProvider, CodeSystemFactoryProvider} = require('./cs-api'); -const { validateOptionalParameter, validateArrayParameter} = require("../../library/utilities"); - -class CachedDesignation { - constructor(display, language, use) { - this.display = display; - this.language = language; - this.use = use; - } -} - -class CodeDBProviderContext { - constructor(key, code, display, definition, status) { - this.key = key; - this.code = code; - this.display = display; - this.definition = definition; - this.status = status; - this.designations = null; // Array of CachedDesignation - this.children = null; // Will be Set of keys if this has children - } - - addChild(key) { - if (!this.children) { - this.children = new Set(); - } - this.children.add(key); - } -} - - -class CodeDBIteratorContext { - constructor(context, keys) { - this.context = context; - this.keys = keys || []; - this.current = 0; - this.total = this.keys.length; - } - - more() { - return this.current < this.total; - } - - next() { - this.current++; - } -} - -class CodeDBFilterHolder { - constructor() { - this.keys = []; - this.cursor = 0; - this.lsql = ''; - } - - hasKey(key) { - // Binary search since keys are sorted - let l = 0; - let r = this.keys.length - 1; - while (l <= r) { - const m = Math.floor((l + r) / 2); - if (this.keys[m] < key) { - l = m + 1; - } else if (this.keys[m] > key) { - r = m - 1; - } else { - return true; - } - } - return false; - } -} - -class CodeDBPrep { - constructor() { - this.filters = []; - } -} - -class CodeDBServices extends CodeSystemProvider { - constructor(opContext, supplements, db, sharedData) { - super(opContext, supplements); - this.db = db; - - // Shared data from factory - this.langs = sharedData.langs; - this.codes = sharedData.codes; - this.codeList = sharedData.codeList; - this.codeSystem = sharedData.codeSystem; - this.root = sharedData.root; - this.firstCodeKey = sharedData.firstCodeKey; - this.relationships = sharedData.relationships; - this.propertyList = sharedData.propertyList; - this.statusKeys = sharedData.statusKeys; - this.statusCodes = sharedData.statusCodes; - } - - close() { - if (this.db) { - this.db.close(); - this.db = null; - } - } - - // Metadata methods - system() { - return this.codeSystem.url; - } - - version() { - return this.codeSystem.version; - } - - name() { - return this.codeSystem.name; - } - - description() { - return this.codeSystem.description; - } - - async totalCount() { - return this.codes.size; - } - - hasParents() { - return this.codeSystem.hierarchical; - } - - hasAnyDisplays(languages) { - const langs = this._ensureLanguages(languages); - - // Check supplements first - if (this._hasAnySupplementDisplays(langs)) { - return true; - } - - // Check if any requested languages are available in code system data - for (const requestedLang of langs.languages) { - for (const [codeDBLangCode] of this.langs) { - const codeDBLang = new Language(codeDBLangCode); - if (codeDBLang.matchesForDisplay(requestedLang)) { - return true; - } - } - } - - return super.hasAnyDisplays(langs); - } - - // Core concept methods - async code(context) { - - const ctxt = await this.#ensureContext(context); - return ctxt ? ctxt.code : null; - } - - async display(context) { - - const ctxt = await this.#ensureContext(context); - if (!ctxt) { - return null; - } - - // Check supplements first - let disp = this._displayFromSupplements(ctxt.code); - if (disp) { - return disp; - } - - // Use language-aware display logic - if (this.opContext.langs && !this.opContext.langs.isEnglishOrNothing()) { - await this.#loadDesignationsForContext(ctxt); - - // Try to find exact language match - for (const lang of this.opContext.langs.langs) { - for (const display of ctxt.designations) { - if (lang.matches(display.language, true)) { - return display.value; - } - } - } - - // Try partial language match - for (const lang of this.opContext.langs.langs) { - for (const display of ctxt.designations) { - if (lang.matches(display.language, false)) { - return display.value; - } - } - } - } - - return ctxt.display || ''; - } - - async definition(context) { - const ctxt = await this.#ensureContext(context); - return ctxt.definition; - } - - async isAbstract(context) { - const ctxt = await this.#ensureContext(context); - return ctxt.abstract; - } - - async isInactive(context) { - const ctxt = await this.#ensureContext(context); - return ctxt.status == 'inactive'; - } - - async getStatus(context) { - const ctxt = await this.#ensureContext(context); - return ctxt.status; - } - - async isDeprecated(context) { - const ctxt = await this.#ensureContext(context); - return ctxt.status == 'deprecated'; - } - - async designations(context, displays) { - const ctxt = await this.#ensureContext(context); - if (ctxt) { - await this.#loadDesignationsForContext(ctxt); - ctxt.designations - // Add main display - displays.addDesignation(true, 'active', this.codeSystem.language, CodeSystem.makeUseForDisplay(), ctxt.desc.trim()); - - // Add cached designations - if (ctxt.displays.length === 0) { - await this.#loadDesignationsForContext(ctxt); - } - - for (const entry of ctxt.designations) { - let use = undefined; - if (entry.type) { - use = { - system: this.codeSystem.url, - code: entry.type - } - } - if (!use) { - use = entry.display ? CodeSystem.makeUseForDisplay() : null; - } - displays.addDesignation(false, 'active', entry.lang, use, entry.value.trim()); - } - - // Add supplement designations - this._listSupplementDesignations(ctxt.code, displays); - } - - } - - async extendLookup(ctxt, props, params) { - validateArrayParameter(props, 'props', String); - validateArrayParameter(params, 'params', Object); - - - if (typeof ctxt === 'string') { - const located = await this.locate(ctxt); - if (!located.context) { - throw new Error(located.message); - } - ctxt = located.context; - } - - if (!(ctxt instanceof CodeDBProviderContext)) { - throw new Error('Invalid context for CodeDB lookup'); - } - - await this.#addConceptProperties(ctxt, params); - await this.#addStatusProperty(ctxt, params); - } - - async #addConceptProperties(ctxt, params) { - return new Promise((resolve, reject) => { - const sql = ` - SELECT PropertyTypes.Description, PropertyValues.Value - FROM Properties, PropertyTypes, PropertyValues - WHERE Properties.CodeKey = ? - AND Properties.PropertyTypeKey = PropertyTypes.PropertyTypeKey - AND Properties.PropertyValueKey = PropertyValues.PropertyValueKey - `; - - this.db.all(sql, [ctxt.key], (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - this.#addStringProperty(params, 'property', row.Description, row.Value); - } - resolve(); - } - }); - }); - } - - async #addStatusProperty(ctxt, params) { - if (ctxt.status) { - this.#addStringProperty(params, 'property', 'STATUS', ctxt.status); - } - } - - #addProperty(params, type, name, value, language = null) { - - const property = { - name: type, - part: [ - { name: 'code', valueCode: name }, - { name: 'value', valueString: value } - ] - }; - - if (language) { - property.part.push({ name: 'language', valueCode: language }); - } - - params.push(property); - } - - #addCodeProperty(params, type, name, value, language = null) { - - const property = { - name: type, - part: [ - { name: 'code', valueCode: name }, - { name: 'value', valueCode: value } - ] - }; - - if (language) { - property.part.push({ name: 'language', valueCode: language }); - } - - params.push(property); - } - - #addStringProperty(params, type, name, value, language = null) { - - const property = { - name: type, - part: [ - { name: 'code', valueCode: name }, - { name: 'value', valueString: value } - ] - }; - - if (language) { - property.part.push({ name: 'language', valueCode: language }); - } - - params.push(property); - } - - #addSupplementDisplays(displays, code) { - if (this.supplements) { - for (const supplement of this.supplements) { - const concept = supplement.getConceptByCode(code); - if (concept) { - if (concept.display) { - displays.push(new LoincDisplay(supplement.jsonObj.language || 'en', concept.display)); - } - if (concept.designation) { - for (const designation of concept.designation) { - const lang = designation.language || supplement.jsonObj.language || 'en'; - displays.push(new LoincDisplay(lang, designation.value)); - } - } - } - } - } - } - - async #loadDesignationsForContext(ctxt) { - if (!ctxt.designations) { - ctxt.designations = []; - return new Promise((resolve, reject) => { - const sql = ` - SELECT Languages.Code as Lang, DescriptionTypes.Description as DType, Descriptions.Value - FROM Descriptions, - Languages, - DescriptionTypes - WHERE Descriptions.CodeKey = ? - AND Descriptions.DescriptionTypeKey != 4 - AND Descriptions.DescriptionTypeKey = DescriptionTypes.DescriptionTypeKey - AND Descriptions.LanguageKey = Languages.LanguageKey - `; - - this.db.all(sql, [ctxt.key], (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - const isDisplay = row.DType === 'LONG_COMMON_NAME'; - ctxt.designations.push(new CachedDesignation(row.Value, , row.Lang, row.DType)); - } - resolve(); - } - }); - }); - } - } - - async #ensureContext(context) { - if (!context) { - return null; - } - if (typeof context === 'string') { - const ctxt = await this.locate(context); - if (!ctxt.context) { - throw new Error(ctxt.message); - } else { - return ctxt.context; - } - } - if (context instanceof CodeDBProviderContext) { - return context; - } - throw new Error("Unknown Type at #ensureContext: " + (typeof context)); - } - - // Lookup methods - async locate(code) { - - assert(!code || typeof code === 'string', 'code must be string'); - if (!code) return { context: null, message: 'Empty code' }; - - const context = this.codes.get(code); - if (context) { - return { context: context, message: null }; - } - - return { context: null, message: undefined }; - } - - // Iterator methods - async iterator(context) { - - - if (!context) { - // Iterate all codes starting from first code - const keys = Array.from({ length: this.codeList.length - this.firstCodeKey }, (_, i) => i + this.firstCodeKey); - return new LoincIteratorContext(null, keys); - } else { - const ctxt = await this.#ensureContext(context); - if (ctxt.kind === LoincProviderContextKind.PART && ctxt.children) { - return new LoincIteratorContext(ctxt, Array.from(ctxt.children)); - } else { - return new LoincIteratorContext(ctxt, []); - } - } - } - - async nextContext(iteratorContext) { - - - if (!iteratorContext.more()) { - return null; - } - - const key = iteratorContext.keys[iteratorContext.current]; - iteratorContext.next(); - - return this.codeList[key]; - } - - // Filter support - async doesFilter(prop, op, value) { - // Relationship filters - if (this.relationships.has(prop) && ['=', 'in', 'exists', 'regex'].includes(op)) { - return true; - } - - // Property filters - if (this.propertyList.has(prop) && ['=', 'in', 'exists', 'regex'].includes(op)) { - return true; - } - - // Status filter - if (prop === 'STATUS' && op === '=' && this.statusKeys.has(value)) { - return true; - } - - // LIST filter - if (prop === 'LIST' && op === '=' && this.codes.has(value)) { - return true; - } - - // CLASSSTYPE filter - if (prop === 'CLASSTYPE' && op === '=' && ["1", "2", "3", "4"].includes(value)) { - return true; - } - - // answers-for filter - if (prop === 'answers-for' && op === '=') { - return true; - } - - // concept filters - if (prop === 'concept' && ['is-a', 'descendent-of', '=', 'in', 'not-in'].includes(op)) { - return true; - } - - // code filters (VSAC workaround) - if (prop === 'code' && ['is-a', 'descendent-of', '='].includes(op)) { - return true; - } - - // copyright filter - if (prop === 'copyright' && op === '=' && ['LOINC', '3rdParty'].includes(value)) { - return true; - } - - return false; - } - - async getPrepContext(iterate) { - return new LoincPrep(iterate); - } - - async filter(filterContext, prop, op, value) { - - - const filter = new LoincFilterHolder(); - await this.#executeFilterQuery(prop, op, value, filter); - filterContext.filters.push(filter); - } - - async #executeFilterQuery(prop, op, value, filter) { - let sql = ''; - let lsql = ''; - - // LIST filter - if (prop === 'LIST' && op === '=' && this.codes.has(value)) { - sql = `SELECT TargetKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('Answer')} - AND SourceKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(TargetKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('Answer')} - AND SourceKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - AND TargetKey = `; - } - // answers-for filter - else if (prop === 'answers-for' && op === '=') { - if (value.startsWith('LL')) { - sql = `SELECT TargetKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('Answer')} - AND SourceKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(TargetKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('Answer')} - AND SourceKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - AND TargetKey = `; - } else { - sql = `SELECT TargetKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('Answer')} - AND SourceKey IN ( - SELECT SourceKey FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('answers-for')} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - ) - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(TargetKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('Answer')} - AND SourceKey IN (SELECT SourceKey FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get('answers-for')} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}')) - AND TargetKey = `; - } - } - // Relationship equal filter - else if (this.relationships.has(prop) && op === '=') { - if (this.codes.has(value)) { - sql = `SELECT SourceKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(SourceKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - AND SourceKey = `; - } else { - sql = `SELECT SourceKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Description = '${this.#sqlWrapString(value)}' COLLATE NOCASE) - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(SourceKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Description = '${this.#sqlWrapString(value)}' COLLATE NOCASE) - AND SourceKey = `; - } - } - // Relationship 'in' filter - else if (this.relationships.has(prop) && op === 'in') { - const codes = this.#commaListOfCodes(value); - sql = `SELECT SourceKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Code IN (${codes})) - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(SourceKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (SELECT CodeKey FROM Codes WHERE Code IN (${codes})) - AND SourceKey = `; - } - // Relationship 'exists' filter - else if (this.relationships.has(prop) && op === 'exists') { - if (this.codes.has(value)) { - sql = `SELECT SourceKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND EXISTS (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(SourceKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND EXISTS (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - AND SourceKey = `; - } else { - sql = `SELECT SourceKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND EXISTS (SELECT CodeKey FROM Codes WHERE Description = '${this.#sqlWrapString(value)}' COLLATE NOCASE) - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(SourceKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND EXISTS (SELECT CodeKey FROM Codes WHERE Description = '${this.#sqlWrapString(value)}' COLLATE NOCASE) - AND SourceKey = `; - } - } - // Relationship regex filter - else if (this.relationships.has(prop) && op === 'regex') { - const matchingKeys = await this.#findRegexMatches( - `SELECT CodeKey as Key, Description FROM Codes - WHERE CodeKey IN (SELECT TargetKey FROM Relationships WHERE RelationshipTypeKey = ${this.relationships.get(prop)})`, - value, - 'Description' - ); - if (matchingKeys.length > 0) { - sql = `SELECT SourceKey as Key FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (${matchingKeys.join(',')}) - ORDER BY SourceKey ASC`; - lsql = `SELECT COUNT(SourceKey) FROM Relationships - WHERE RelationshipTypeKey = ${this.relationships.get(prop)} - AND TargetKey IN (${matchingKeys.join(',')}) - AND SourceKey = `; - } - } - // Property equal filter (with CLASSTYPE handling) - else if (this.propertyList.has(prop) && op === '=') { - let actualValue = value; - if (prop === 'CLASSTYPE' && ['1', '2', '3', '4'].includes(value)) { - const classTypes = { - '1': 'Laboratory class', - '2': 'Clinical class', - '3': 'Claims attachments', - '4': 'Surveys' - }; - actualValue = classTypes[value]; - } - sql = `SELECT CodeKey as Key FROM Properties, PropertyValues - WHERE Properties.PropertyTypeKey = ${this.propertyList.get(prop)} - AND Properties.PropertyValueKey = PropertyValues.PropertyValueKey - AND PropertyValues.Value = '${this.#sqlWrapString(actualValue)}' COLLATE NOCASE - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Properties, PropertyValues - WHERE Properties.PropertyTypeKey = ${this.propertyList.get(prop)} - AND Properties.PropertyValueKey = PropertyValues.PropertyValueKey - AND PropertyValues.Value = '${this.#sqlWrapString(actualValue)}' COLLATE NOCASE - AND CodeKey = `; - } - // Property 'in' filter - else if (this.propertyList.has(prop) && op === 'in') { - const codes = this.#commaListOfCodes(value); - sql = `SELECT CodeKey as Key FROM Properties, PropertyValues - WHERE Properties.PropertyTypeKey = ${this.propertyList.get(prop)} - AND Properties.PropertyValueKey = PropertyValues.PropertyValueKey - AND PropertyValues.Value IN (${codes}) COLLATE NOCASE - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Properties, PropertyValues - WHERE Properties.PropertyTypeKey = ${this.propertyList.get(prop)} - AND Properties.PropertyValueKey = PropertyValues.PropertyValueKey - AND PropertyValues.Value IN (${codes}) COLLATE NOCASE - AND CodeKey = `; - } - // Property 'exists' filter - else if (this.propertyList.has(prop) && op === 'exists') { - sql = `SELECT DISTINCT CodeKey as Key FROM Properties - WHERE Properties.PropertyTypeKey = ${this.propertyList.get(prop)} - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Properties - WHERE Properties.PropertyTypeKey = ${this.propertyList.get(prop)} - AND CodeKey = `; - } - // Property regex filter - else if (this.propertyList.has(prop) && op === 'regex') { - const matchingKeys = await this.#findRegexMatches( - `SELECT PropertyValueKey, Value FROM PropertyValues - WHERE PropertyValueKey IN (SELECT PropertyValueKey FROM Properties WHERE PropertyTypeKey = ${this.propertyList.get(prop)})`, - value, - 'Value', - 'PropertyValueKey' - ); - if (matchingKeys.length > 0) { - sql = `SELECT CodeKey as Key FROM Properties - WHERE PropertyTypeKey = ${this.propertyList.get(prop)} - AND PropertyValueKey IN (${matchingKeys.join(',')}) - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Properties - WHERE PropertyTypeKey = ${this.propertyList.get(prop)} - AND PropertyValueKey IN (${matchingKeys.join(',')}) - AND CodeKey = `; - } - } - // Status filter - else if (prop === 'STATUS' && op === '=' && this.statusKeys.has(value)) { - sql = `SELECT CodeKey as Key FROM Codes - WHERE StatusKey = ${this.statusKeys.get(value)} - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Codes - WHERE StatusKey = ${this.statusKeys.get(value)} - AND CodeKey = `; - } - // Concept hierarchy filters (is-a, descendent-of) - else if (prop === 'concept' && ['is-a', 'descendent-of'].includes(op)) { - sql = `SELECT DescendentKey as Key FROM Closure - WHERE AncestorKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - ORDER BY DescendentKey ASC`; - lsql = `SELECT COUNT(DescendentKey) FROM Closure - WHERE AncestorKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - AND DescendentKey = `; - } - // Concept equal filter (workaround for VSAC misuse) - else if (prop === 'concept' && op === '=') { - sql = `SELECT CodeKey as Key FROM Codes - WHERE Code = '${this.#sqlWrapString(value)}' - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Codes - WHERE Code = '${this.#sqlWrapString(value)}' - AND CodeKey = `; - } - // Concept 'in' filter (workaround for VSAC misuse) - else if (prop === 'concept' && op === 'in') { - const codes = this.#commaListOfCodes(value); - sql = `SELECT CodeKey as Key FROM Codes - WHERE Code IN (${codes}) - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Codes - WHERE Code IN (${codes}) - AND CodeKey = `; - } - // Code property filters (workaround for VSAC misuse) - else if (prop === 'code' && ['is-a', 'descendent-of'].includes(op)) { - sql = `SELECT DescendentKey as Key FROM Closure - WHERE AncestorKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - ORDER BY DescendentKey ASC`; - lsql = `SELECT COUNT(DescendentKey) FROM Closure - WHERE AncestorKey IN (SELECT CodeKey FROM Codes WHERE Code = '${this.#sqlWrapString(value)}') - AND DescendentKey = `; - } - else if (prop === 'code' && op === '=') { - sql = `SELECT CodeKey as Key FROM Codes - WHERE Code = '${this.#sqlWrapString(value)}' - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Codes - WHERE Code = '${this.#sqlWrapString(value)}' - AND CodeKey = `; - } - // Copyright filters - else if (prop === 'copyright' && op === '=') { - if (value === 'LOINC') { - sql = `SELECT CodeKey as Key FROM Codes - WHERE NOT CodeKey IN (SELECT CodeKey FROM Properties WHERE PropertyTypeKey = 9) - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Codes - WHERE NOT CodeKey IN (SELECT CodeKey FROM Properties WHERE PropertyTypeKey = 9) - AND CodeKey = `; - } else if (value === '3rdParty') { - sql = `SELECT CodeKey as Key FROM Codes - WHERE CodeKey IN (SELECT CodeKey FROM Properties WHERE PropertyTypeKey = 9) - ORDER BY CodeKey ASC`; - lsql = `SELECT COUNT(CodeKey) FROM Codes - WHERE CodeKey IN (SELECT CodeKey FROM Properties WHERE PropertyTypeKey = 9) - AND CodeKey = `; - } - } - - if (sql) { - await this.#executeSQL(sql, filter); - filter.lsql = lsql; - } else { - throw new Error(`The filter "${prop} ${op} ${value}" is not supported for LOINC`); - } - } - -// Helper method for regex matching - async #findRegexMatches(sql, pattern, valueColumn, keyColumn = 'Key') { - return new Promise((resolve, reject) => { - const regex = new RegExp(pattern); - const matchingKeys = []; - - this.db.all(sql, (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - if (regex.test(row[valueColumn])) { - matchingKeys.push(row[keyColumn]); - } - } - resolve(matchingKeys); - } - }); - }); - } - -// Helper method for comma-separated code lists - #commaListOfCodes(source) { - const codes = source.split(',') - .filter(s => this.codes.has(s.trim())) - .map(s => `'${this.#sqlWrapString(s.trim())}'`); - return codes.join(','); - } - - async #executeSQL(sql, filter) { - return new Promise((resolve, reject) => { - this.db.all(sql, (err, rows) => { - if (err) { - reject(err); - } else { - filter.keys = rows.map(row => row.Key).filter(key => key !== 0); - resolve(); - } - }); - }); - } - - #sqlWrapString(str) { - return str.replace(/'/g, "''"); - } - - async executeFilters(filterContext) { - - return filterContext.filters; - } - - async filterSize(filterContext, set) { - - return set.keys.length; - } - - async filterMore(filterContext, set) { - - set.cursor = set.cursor || 0; - return set.cursor < set.keys.length; - } - - async filterConcept(filterContext, set) { - - - if (set.cursor >= set.keys.length) { - return null; - } - - const key = set.keys[set.cursor]; - set.cursor++; - - return this.codeList[key]; - } - - async filterLocate(filterContext, set, code) { - const context = this.codes.get(code); - if (!context) { - return `Not a valid code: ${code}`; - } - - if (!set.lsql) { - return 'Filter not understood'; - } - - // Check if this context's key is in the filter - if (set.hasKey(context.key)) { - return context; - } else { - return null; // `Code ${code} is not in the specified filter`; - } - } - - async filterCheck(filterContext, set, concept) { - if (!(concept instanceof LoincProviderContext)) { - return false; - } - - return set.hasKey(concept.key); - } - - // Search filter - placeholder for text search - // eslint-disable-next-line no-unused-vars - async searchFilter(filterContext, filter, sort) { - - throw new Error('Text search not implemented yet'); - } - - // Subsumption testing - async subsumesTest(codeA, codeB) { - await this.#ensureContext(codeA); - await this.#ensureContext(codeB); - - return 'not-subsumed'; // Not implemented yet - } - - versionAlgorithm() { - return 'natural'; - } - - isDisplay(designation) { - return designation.use.code == "SHORTNAME" || designation.use.code == "LONG_COMMON_NAME" || designation.use.code == "LinguisticVariantDisplayName"; - } -} - -class LoincServicesFactory extends CodeSystemFactoryProvider { - constructor(i18n, dbPath) { - super(i18n); - this.dbPath = dbPath; - this.uses = 0; - this._loaded = false; - this._sharedData = null; - } - - system() { - return 'http://loinc.org'; - } - - version() { - return this._sharedData._version; - } - - name() { - return 'LOINC'; - } - - async #ensureLoaded() { - if (!this._loaded) { - await this.load(); - } - } - - async load() { - const db = new sqlite3.Database(this.dbPath); - - // Enable performance optimizations - await this.#optimizeDatabase(db); - - try { - this._sharedData = { - langs: new Map(), - codes: new Map(), - codeList: [null], - relationships: new Map(), - propertyList: new Map(), - statusKeys: new Map(), - statusCodes: new Map(), - _version: '', - root: '', - firstCodeKey: 0 - }; - - // Load small lookup tables in parallel - // eslint-disable-next-line no-unused-vars - const [langs, statusCodes, relationships, propertyList, config] = await Promise.all([ - this.#loadLanguages(db), - this.#loadStatusCodes(db), - this.#loadRelationshipTypes(db), - this.#loadPropertyTypes(db), - this.#loadConfig(db) - ]); - - // Load codes (largest operation) - await this.#loadCodes(db); - - // Load dependent data in parallel - await Promise.all([ - // this.#loadDesignationsCache(db), - this.#loadHierarchy(db) - ]); - - } finally { - db.close(); - } - this._loaded = true; - } - - async #optimizeDatabase(db) { - return new Promise((resolve) => { - db.serialize(() => { - db.run('PRAGMA journal_mode = WAL'); - db.run('PRAGMA synchronous = NORMAL'); - db.run('PRAGMA cache_size = 10000'); - db.run('PRAGMA temp_store = MEMORY'); - db.run('PRAGMA mmap_size = 268435456'); // 256MB - resolve(); - }); - }); - } - - async #loadLanguages(db) { - return new Promise((resolve, reject) => { - db.all('SELECT LanguageKey, Code FROM Languages', (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - this._sharedData.langs.set(row.Code, row.LanguageKey); - } - resolve(); - } - }); - }); - } - - async #loadStatusCodes(db) { - return new Promise((resolve, reject) => { - db.all('SELECT StatusKey, Description FROM StatusCodes', (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - this._sharedData.statusKeys.set(row.Description, row.StatusKey.toString()); - this._sharedData.statusCodes.set(row.StatusKey.toString(), row.Description); - } - resolve(); - } - }); - }); - } - - async #loadRelationshipTypes(db) { - return new Promise((resolve, reject) => { - db.all('SELECT RelationshipTypeKey, Description FROM RelationshipTypes', (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - this._sharedData.relationships.set(row.Description, row.RelationshipTypeKey.toString()); - } - resolve(); - } - }); - }); - } - - async #loadPropertyTypes(db) { - return new Promise((resolve, reject) => { - db.all('SELECT PropertyTypeKey, Description FROM PropertyTypes', (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - this._sharedData.propertyList.set(row.Description, row.PropertyTypeKey.toString()); - } - resolve(); - } - }); - }); - } - - async #loadCodes(db) { - return new Promise((resolve, reject) => { - // First get the count to pre-allocate array - db.get('SELECT MAX(CodeKey) as maxKey FROM Codes', (err, row) => { - if (err) return reject(err); - - // Pre-allocate the array to avoid repeated resizing - const maxKey = row.maxKey || 0; - this._sharedData.codeList = new Array(maxKey + 1).fill(null); - - // Now load all codes - db.all('SELECT CodeKey, Code, Type, Codes.Description, StatusCodes.Description as Status FROM Codes, StatusCodes where StatusCodes.StatusKey = Codes.StatusKey order by CodeKey Asc', (err, rows) => { - if (err) return reject(err); - - // Batch process rows - for (const row of rows) { - const context = new LoincProviderContext( - row.CodeKey, - row.Type - 1, - row.Code, - row.Description, - row.Status - ); - - this._sharedData.codes.set(row.Code, context); - this._sharedData.codeList[row.CodeKey] = context; - - if (this._sharedData.firstCodeKey === 0 && context.kind === LoincProviderContextKind.CODE) { - this._sharedData.firstCodeKey = context.key; - } - } - resolve(); - }); - }); - }); - } - - async #loadDesignationsCache(db) { - return new Promise((resolve, reject) => { - const sql = ` - SELECT - d.CodeKey, - l.Code as Lang, - dt.Description as DType, - d.Value, - dt.Description = 'LONG_COMMON_NAME' as IsDisplay - FROM Descriptions d - JOIN Languages l ON d.LanguageKey = l.LanguageKey - JOIN DescriptionTypes dt ON d.DescriptionTypeKey = dt.DescriptionTypeKey - WHERE d.DescriptionTypeKey != 4 - ORDER BY d.CodeKey - `; - - db.all(sql, (err, rows) => { - if (err) return reject(err); - - // Batch process by CodeKey to reduce lookups - let currentKey = null; - let currentContext = null; - - for (const row of rows) { - if (row.CodeKey !== currentKey) { - currentKey = row.CodeKey; - currentContext = this._sharedData.codeList[currentKey]; - } - - if (currentContext) { - currentContext.displays.push( - new DescriptionCacheEntry(row.IsDisplay, row.Lang, row.Value, row.DType) - ); - } - } - resolve(); - }); - }); - } - - async #loadHierarchy(db) { - const childRelKey = this._sharedData.relationships.get('child'); - if (!childRelKey) { - return; // No child relationships defined - } - - return new Promise((resolve, reject) => { - const sql = ` - SELECT SourceKey, TargetKey FROM Relationships - WHERE RelationshipTypeKey = ${childRelKey} - `; - - db.all(sql, (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - if (row.SourceKey !== 0 && row.TargetKey !== 0) { - const parentContext = this._sharedData.codeList[row.SourceKey]; - if (parentContext) { - parentContext.addChild(row.TargetKey); - } - } - } - resolve(); - } - }); - }); - } - - async #loadConfig(db) { - return new Promise((resolve, reject) => { - db.all('SELECT ConfigKey, Value FROM Config WHERE ConfigKey IN (2, 3)', (err, rows) => { - if (err) { - reject(err); - } else { - for (const row of rows) { - if (row.ConfigKey === 2) { - this._sharedData._version = row.Value; - } else if (row.ConfigKey === 3) { - this._sharedData.root = row.Value; - } - } - resolve(); - } - }); - }); - } - - defaultVersion() { - return this._sharedData?._version || 'unknown'; - } - - async build(opContext, supplements) { - await this.#ensureLoaded(); - this.recordUse(); - - // Create fresh database connection for this provider instance - const db = new sqlite3.Database(this.dbPath); - - return new LoincServices(opContext, supplements, db, this._sharedData); - } - - useCount() { - return this.uses; - } - - recordUse() { - this.uses++; - } - - async buildKnownValueSet(url, version) { - - if (version && version != this.version()) { - return null; - } - if (!url.startsWith('http://loinc.org/vs')) { - return null; - } - if (url == 'http://loinc.org/vs') { - // All LOINC codes - return { - resourceType: 'ValueSet', url: 'http://loinc.org/vs', version: this.version(), status: 'active', - name: 'LOINC Value Set - all LOINC codes', description: 'All LOINC codes', - date: new Date().toISOString(), experimental: false, - compose: { include: [{ system: this.system() }] } - }; - } - - if (url.startsWith('http://loinc.org/vs/')) { - const code = url.substring(20); - const ci = this._sharedData.codes.get(code); - if (!ci) { - return null; - } - - if (ci.kind === LoincProviderContextKind.PART) { - // Part-based value set with ancestor filter - return { - resourceType: 'ValueSet', url: url, version: this.version(), status: 'active', - name: 'LOINCValueSetFor' + ci.code.replace(/-/g, '_'), description: 'LOINC value set for code ' + ci.code + ': ' + ci.desc, - date: new Date().toISOString(), experimental: false, - compose: { include: [{ system: this.system(), filter: [{ property: 'ancestor', op: '=', value: code }] }] - } - }; - } - - if (ci.kind === LoincProviderContextKind.LIST) { - // Answer list - enumerate concepts from database - const concepts = await this.#getAnswerListConcepts(ci.key); - return { - resourceType: 'ValueSet', url: url, version: this.version(), status: 'active', - name: 'LOINCAnswerList' + ci.code.replace(/-/g, '_'), description: 'LOINC Answer list for code ' + ci.code + ': ' + ci.desc, - date: new Date().toISOString(), experimental: false, - compose: { include: [{ system: this.system(), concept: concepts }] } - }; - } - } - - return null; - } - - /** - * Get answer list concepts from database - * @param {number} sourceKey - Key of the answer list - * @returns {Promise} Array of {code, display} objects - */ - async #getAnswerListConcepts(sourceKey) { - return new Promise((resolve, reject) => { - let db = new sqlite3.Database(this.dbPath); - const sql = ` - SELECT Code, Description - FROM Relationships, Codes - WHERE SourceKey = ? - AND RelationshipTypeKey = 40 - AND Relationships.TargetKey = Codes.CodeKey - `; - - db.all(sql, [sourceKey], (err, rows) => { - if (err) { - reject(err); - } else { - const concepts = rows.map(row => ({ - code: row.Code - })); - resolve(concepts); - } - }); - }); - } - - id() { - return "loinc"+this.version(); - } -} - -module.exports = { - LoincServices, - LoincServicesFactory, - LoincProviderContext, - LoincProviderContextKind -}; \ No newline at end of file diff --git a/tx/cs/cs-hgvs.js b/tx/cs/cs-hgvs.js index 500647f..d2a90ca 100644 --- a/tx/cs/cs-hgvs.js +++ b/tx/cs/cs-hgvs.js @@ -200,11 +200,6 @@ class HGVSServices extends CodeSystemProvider { throw new Error('Filters are not supported for HGVS'); } - async searchFilter(filterContext, filter, sort) { - - throw new Error('Filters are not supported for HGVS'); - } - async filter(filterContext, prop, op, value) { throw new Error('Filters are not supported for HGVS'); diff --git a/tx/cs/cs-lang.js b/tx/cs/cs-lang.js index 35d152c..774a620 100644 --- a/tx/cs/cs-lang.js +++ b/tx/cs/cs-lang.js @@ -259,16 +259,6 @@ class IETFLanguageCodeProvider extends CodeSystemProvider { return false; } - async searchFilter(filterContext, filter, sort) { - - assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); - assert(filter && typeof filter === 'string', 'filter must be a non-null string'); - assert(typeof sort === 'boolean', 'sort must be a boolean'); - - throw new Error('Text search not supported'); - } - - async filter(filterContext, prop, op, value) { assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); diff --git a/tx/cs/cs-loinc.js b/tx/cs/cs-loinc.js index 7b58f9b..fc718c3 100644 --- a/tx/cs/cs-loinc.js +++ b/tx/cs/cs-loinc.js @@ -1020,13 +1020,6 @@ class LoincServices extends BaseCSServices { return set.hasKey(concept.key); } - // Search filter - placeholder for text search - // eslint-disable-next-line no-unused-vars - async searchFilter(filterContext, filter, sort) { - - throw new Error('Text search not implemented yet'); - } - // Subsumption testing async subsumesTest(codeA, codeB) { await this.#ensureContext(codeA); diff --git a/tx/cs/cs-omop.js b/tx/cs/cs-omop.js index cf50673..3349b1f 100644 --- a/tx/cs/cs-omop.js +++ b/tx/cs/cs-omop.js @@ -650,12 +650,6 @@ class OMOPServices extends BaseCSServices { return false; // OMOP filters are closed } - // Search filter - not implemented - // eslint-disable-next-line no-unused-vars - async searchFilter(filterContext, filter, sort) { - - throw new Error('Search filter not implemented yet'); - } // Subsumption testing - not implemented async subsumesTest(codeA, codeB) { diff --git a/tx/cs/cs-ucum.js b/tx/cs/cs-ucum.js index b00e225..973cdb6 100644 --- a/tx/cs/cs-ucum.js +++ b/tx/cs/cs-ucum.js @@ -245,15 +245,6 @@ class UcumCodeSystemProvider extends BaseCSServices { return (prop === 'canonical' && op === '='); } - async searchFilter(filterContext, filter, sort) { - - assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); - assert(filter && typeof filter === 'string', 'filter must be a non-null string'); - assert(typeof sort === 'boolean', 'sort must be a boolean'); - - throw new Error('Search filter not implemented for UCUM'); - } - async specialFilter(filterContext, filter, sort) { assert(filterContext && filterContext instanceof FilterExecutionContext, 'filterContext must be a FilterExecutionContext'); diff --git a/tx/library/designations.js b/tx/library/designations.js index 4bd5d24..426ab8e 100644 --- a/tx/library/designations.js +++ b/tx/library/designations.js @@ -608,7 +608,11 @@ class Designations { } } } - + for (const cd of this.designations) { + if (!cd.language && this.isDisplay(cd)) { + return cd; + } + } return null; } diff --git a/tx/library/renderer.js b/tx/library/renderer.js index 5648fe4..49feef9 100644 --- a/tx/library/renderer.js +++ b/tx/library/renderer.js @@ -904,7 +904,7 @@ class Renderer { this.renderProperty(tbl, 'Expansion Total', vs.expansion.total); this.renderProperty(tbl, 'Expansion Offset', vs.expansion.offset); for (let p of vs.expansion.parameter || []) { - await this.renderPropertyLink(tbl, "Parameter: " + p.name, getValuePrimitive(p)); + await this.renderPropertyLink(tbl, "Parameter: " + p.name, String(getValuePrimitive(p))); } if (!vs.expansion.contains || vs.expansion.contains.length === 0) { diff --git a/tx/operation-context.js b/tx/operation-context.js index e83f2e9..f3c948d 100644 --- a/tx/operation-context.js +++ b/tx/operation-context.js @@ -447,6 +447,7 @@ class OperationContext { newContext.timeTracker = this.timeTracker.link(); newContext.logEntries = [...this.logEntries]; newContext.debugging = this.debugging; + newContext.usageTracker = this.usageTracker; return newContext; } diff --git a/tx/params.js b/tx/params.js index 2c0528e..c37524d 100644 --- a/tx/params.js +++ b/tx/params.js @@ -36,6 +36,7 @@ class TxParameters { validating = false; abstractOk = true; // note true! inferSystem = false; + sort = 'design'; constructor(languages, i18n, validating) { validateParameter(languages, 'languages', LanguageDefinitions); @@ -242,6 +243,10 @@ class TxParameters { if (getValuePrimitive(p) == true) this.inferSystem = true; break; } + case 'sort': { + this.sort = getValuePrimitive(p); + break; + } case "exclude-system": { throw new Issue('error', 'not-supported', null, null, "The parameter 'exclude-system' is not supported by this system", null, 400); } @@ -524,7 +529,7 @@ class TxParameters { this.FUid + '|' + b(this.FMembershipOnly) + '|' + this.FProperties.join(',') + '|' + b(this.FActiveOnly) + b(this.FDisplayWarning) + b(this.FExcludeNested) + b(this.FGenerateNarrative) + b(this.FExcludeNotForUI) + b(this.FExcludePostCoordinated) + b(this.FIncludeDesignations) + b(this.FIncludeDefinition) + b(this.hasActiveOnly) + b(this.hasExcludeNested) + b(this.hasGenerateNarrative) + - b(this.hasExcludeNotForUI) + b(this.hasExcludePostCoordinated) + b(this.hasIncludeDesignations) + + b(this.hasExcludeNotForUI) + b(this.hasExcludePostCoordinated) + b(this.hasIncludeDesignations) + this.sort+'|'+ b(this.hasIncludeDefinition) + b(this.hasDefaultToLatestVersion) + b(this.hasDisplayWarning) + b(this.hasExcludeNotForUI) + b(this.hasMembershipOnly) + b(this.FDefaultToLatestVersion); if (this.hasHTTPLanguages) { @@ -585,6 +590,7 @@ class TxParameters { this.hasDefaultToLatestVersion = other.hasDefaultToLatestVersion; this.hasMembershipOnly = other.hasMembershipOnly; this.hasDisplayWarning = other.hasDisplayWarning; + this.sort = other.sort; if (other.FProperties) { this.FProperties = [...other.FProperties]; diff --git a/tx/tests/test-cases-version.js b/tx/tests/test-cases-version.js index a5c4a17..a1964e9 100644 --- a/tx/tests/test-cases-version.js +++ b/tx/tests/test-cases-version.js @@ -3,6 +3,6 @@ // Regenerate with: node generate-tests.js function txTestVersion() { - return '1.9.0-SNAPSHOT'; + return '1.9.0'; } module.exports = { txTestVersion }; diff --git a/tx/tx.js b/tx/tx.js index 95a697e..b10fcf7 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -48,6 +48,7 @@ const {bundleFromR5} = require("./xversion/xv-bundle"); const {convertResourceToR5} = require("./xversion/xv-resource"); const ClosureWorker = require("./workers/closure"); const {BundleXML} = require("./xml/bundle-xml"); +const ConceptUsageTracker = require("./usage-tracker"); // const {writeFileSync} = require("fs"); class TXModule { @@ -138,6 +139,7 @@ class TXModule { consoleErrors: config.consoleErrors, telnetErrors: config.telnetErrors }); + this.usageTracker = new ConceptUsageTracker(); this.log.info('Initializing TX module'); @@ -280,6 +282,7 @@ class TXModule { acceptLanguage, this.i18n, requestId, 30, endpointInfo.resourceCache, endpointInfo.expansionCache ); + opContext.usageTracker = this.usageTracker; // Attach everything to request req.txProvider = endpointInfo.provider; diff --git a/tx/usage-tracker.js b/tx/usage-tracker.js new file mode 100644 index 0000000..0768d4b --- /dev/null +++ b/tx/usage-tracker.js @@ -0,0 +1,58 @@ + +class ConceptUsageTracker { + + constructor() { + this.map = new Map(); + } + + async scanValueSets(library) { + let c = 0; + for (let vsp of library.valueSetProviders) { + let list = await vsp.listAllValueSets(); + for (let url of list) { + let vs = await vsp.fetchValueSet(url); + if (vs && vs.jsonObj.compose) { + if (await this.scanValueSet(vs.jsonObj.compose)) { + c++; + } + } + } + } + return c; + } + + async scanValueSet(compose) { + let ok = false; + for (let inc of compose.include || []) { + if (inc.system) { + for (let c of inc.concept || []) { + if (c.code) { + ok = true; + this.seeConcept(inc.system, c.code); + } + } + } + } + return ok; + } + + seeConcept(system, code) { + let cs = this.map.get(system); + if (!cs) { + cs = new Map(); + this.map.set(system, cs); + } + let ci = cs.get(code); + if (!ci) { + ci = { count : 0 } + cs.set(code, ci); + } + ci.count++; + } + + usages(system) { + return this.map.get(system) || null; + } +} + +module.exports = ConceptUsageTracker; \ No newline at end of file diff --git a/tx/vs/vs-vsac.js b/tx/vs/vs-vsac.js index 773426f..3739f6e 100644 --- a/tx/vs/vs-vsac.js +++ b/tx/vs/vs-vsac.js @@ -4,7 +4,6 @@ const { AbstractValueSetProvider } = require('./vs-api'); const { ValueSetDatabase } = require('./vs-database'); const { VersionUtilities } = require('../../library/version-utilities'); const folders = require('../../library/folder-setup'); -const ValueSet = require("../library/valueset"); /** * VSAC (Value Set Authority Center) ValueSet provider @@ -460,17 +459,18 @@ class VSACValueSetProvider extends AbstractValueSetProvider { // are fetched, we check to see if we've got the compose, and if we // haven't, then we fetch it and store it async checkFullVS(vs) { - if (!vs) { - return null; - } - if (vs.jsonObj.compose) { - return vs; - } - console.log('get a full copy for the ValueSet '+vs.url+'|'+vs.version); - let vsNew = await this._fetchValueSet(vs.id); - await this.database.upsertValueSet(vsNew); - this.database.addToMap(this.valueSetMap, vsNew.id, vsNew.url, vsNew.version, vsNew); - return new ValueSet(vsNew); + // if (!vs) { + // return null; + // } + // if (vs.jsonObj?.compose) { + // return vs; + // } + // console.log('get a full copy for the ValueSet '+vs.url+'|'+vs.version); + // let vsNew = await this._fetchValueSet(vs.id); + // await this.database.upsertValueSet(vsNew); + // this.database.addToMap(this.valueSetMap, vsNew.id, vsNew.url, vsNew.version, vsNew); + // return new ValueSet(vsNew); + return vs; } async processContentAndHistory(q, tracking, length) { diff --git a/tx/workers/expand.js b/tx/workers/expand.js index b227e45..6262a96 100644 --- a/tx/workers/expand.js +++ b/tx/workers/expand.js @@ -198,6 +198,7 @@ class ValueSetCounter { class ValueSetExpander { worker; params; + excludedSystems = new Set(); excluded = new Set(); hasExclusions = false; requiredSupplements = new Set(); @@ -210,21 +211,6 @@ class ValueSetExpander { this.csCounter = new Map(); } - addDefinedCode(cs, system, c, imports, parent, excludeInactive, srcURL) { - this.worker.deadCheck('addDefinedCode'); - let n = null; - if (!this.params.excludeNotForUI || !cs.isAbstract(c)) { - const cds = new Designations(this.worker.opContext.i18n.languageDefinitions); - this.listDisplays(cds, c); - n = this.includeCode(null, parent, system, '', c.code, cs.isAbstract(c), cs.isInactive(c), cs.isDeprecated(c), cs.codeStatus(c), cds, c.definition, c.itemWeight, - null, imports, c.getAllExtensionsW(), null, c.properties, null, excludeInactive, srcURL); - } - for (let i = 0; i < c.concept.length; i++) { - this.worker.deadCheck('addDefinedCode'); - this.addDefinedCode(cs, system, c.concept[i], imports, n, excludeInactive, srcURL); - } - } - async listDisplaysFromProvider(displays, cs, context) { await cs.designations(context, displays); displays.source = cs; @@ -446,6 +432,9 @@ class ValueSetExpander { if (definition) { this.defineProperty(expansion, n, 'http://hl7.org/fhir/concept-properties#definition', pn, "valueString", definition); } + } else if (pn === 'usage-count') { + let counter = cs.usages().get(code); + this.defineProperty(expansion, n, 'http://fhir.org/FHIRsmith/CodeSystem/concept-properties#usage-count', pn, "valueInteger", counter ? counter.count : 0); } else if (csProps != null && cs != null) { for (const cp of csProps) { if (cp.code === pn) { @@ -601,7 +590,7 @@ class ValueSetExpander { } } - async checkSource(cset, exp, filter, srcURL, ts) { + async checkSource(cset, exp, filter, srcURL, ts, vsInfo) { this.worker.deadCheck('checkSource'); Extensions.checkNoModifiers(cset, 'ValueSetExpander.checkSource', 'set'); let imp = false; @@ -628,6 +617,9 @@ class ValueSetExpander { if (cs == null) { // nothing } else { + if (vsInfo && vsInfo.isSimple) { + vsInfo.handleByCS = cs.handlesSelecting(); + } if (cs.contentMode() !== 'complete') { if (cs.contentMode() === 'not-present') { throw new Issue('error', 'business-rule', null, null, 'The code system definition for ' + cset.system + ' has no content, so this expansion cannot be performed', 'invalid'); @@ -660,7 +652,18 @@ class ValueSetExpander { } } - async includeCodes(cset, path, vsSrc, filter, expansion, excludeInactive, notClosed) { + async processCodes(path, vsSrc, compose, filter, expansion, excludeInactive, notClosed, vsInfo) { + const cs = await this.worker.findCodeSystem(vsInfo.system, vsInfo.version, this.params, ['complete', 'fragment'], + false, false, true, null, this.requiredSupplements); + if (cs != null) { + + // set up the call to the provider + // call the provider + // include the codes + } + } + + async includeCodes(cset, path, vsSrc, compose, filter, expansion, excludeInactive, notClosed) { this.worker.deadCheck('processCodes#1'); const valueSets = []; @@ -730,7 +733,7 @@ class ValueSetExpander { } const iter = await cs.iterator(null); - if (valueSets.length === 0 && this.limitCount > 0 && (iter && iter.total > this.limitCount) && this.offset < 0) { + if (valueSets.length === 0 && this.limitCount > 0 && (iter && iter.total > this.limitCount) && this.offset < 0) { throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [vsSrc.vurl, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics()); } @@ -754,14 +757,17 @@ class ValueSetExpander { const ctxt = await cs.searchFilter(prep, filter, false); let set = await cs.executeFilters(prep); this.worker.opContext.log('iterate filters'); - while (await cs.filterMore(ctxt, set)) { + while (await cs.filterMore(ctxt, set[0])) { this.worker.deadCheck('processCodes#4'); - const c = await cs.filterConcept(ctxt, set); - if (await this.passesFilters(cs, c, prep, filters, 0)) { + const c = await cs.filterConcept(ctxt, set[0]); + if (await this.passesFilters(cs, c, prep, set, 1)) { const cds = new Designations(this.worker.i18n.languageDefinitions); await this.listDisplaysFromProvider(cds, cs, c); - await this.includeCode(cs, null, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c), await cs.deprecated(c), await cs.getCodeStatus(c), - cds, await cs.definition(c), await cs.itemWeight(c), expansion, valueSets, await cs.getExtensions(c), null, await cs.getProperties(c), null, excludeInactive, vsSrc.url); + let added = await this.includeCode(cs, null, await cs.system(), await cs.version(), await cs.code(c), await cs.isAbstract(c), await cs.isInactive(c), await cs.isDeprecated(c), await cs.getStatus(c), + cds, await cs.definition(c), await cs.itemWeight(c), expansion, valueSets, await cs.extensions(c), null, await cs.properties(c), null, excludeInactive, vsSrc.url); + if (added) { + this.addToTotal(); + } } } this.worker.opContext.log('iterate filters done'); @@ -800,8 +806,9 @@ class ValueSetExpander { this.worker.opContext.log('prepare filters'); const fcl = cset.filter; const prep = await cs.getPrepContext(true); + if (!filter.isNull) { - await cs.searchFilter(filter, prep, true); + await cs.searchFilter(prep, filter, true); } if (cs.specialEnumeration()) { @@ -813,7 +820,7 @@ class ValueSetExpander { this.worker.deadCheck('processCodes#4a'); const fc = fcl[i]; if (!fc.value) { - throw new Issue('error', 'invalid', path+".filter["+i+"]", 'UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.worker.i18n.translate('UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.params.httpLanguages, [cs.system(), fc.property, fc.op]), 'vs-invalid', 400); + throw new Issue('error', 'invalid', path + ".filter[" + i + "]", 'UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.worker.i18n.translate('UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE', this.params.httpLanguages, [cs.system(), fc.property, fc.op]), 'vs-invalid', 400); } Extensions.checkNoModifiers(fc, 'ValueSetExpander.processCodes', 'filter'); await cs.filter(prep, fc.property, fc.op, fc.value); @@ -860,22 +867,10 @@ class ValueSetExpander { async passesFilters(cs, c, prep, filters, offset) { for (let j = offset; j < filters.length; j++) { const f = filters[j]; - // if (f instanceof SpecialProviderFilterContextNothing) { - // return false; - // } else if (f instanceof SpecialProviderFilterContextConcepts) { - // let ok = false; - // for (const t of f.list) { - // if (cs.sameContext(t, c)) { - // ok = true; - // } - // } - // if (!ok) return false; - // } else { - let ok = await cs.filterCheck(prep, f, c); - if (ok != true) { - return false; - } - // } + let ok = await cs.filterCheck(prep, f, c); + if (ok != true) { + return false; + } } return true; } @@ -930,13 +925,11 @@ class ValueSetExpander { } if (!cset.concept && !cset.filter) { - this.opContext.log('handle system'); - if (cs.specialEnumeration() && filters.length === 0) { - const base = await this.expandValueSet(cs.specialEnumeration(), '', filter, notClosed); - Extensions.addBoolean(expansion, 'http://hl7.org/fhir/StructureDefinition/valueset-toocostly', true); - this.excludeValueSet(base, expansion, valueSets, 0); - notClosed.value = true; - } else if (filter.isNull) { + this.worker.opContext.log('handle system'); + if (!cset.valueSet) { + // excluding a whole system - we don't list the codes in this case + this.excludedSystems.add(cset.system + (cset.version ? '|'+cset.version : '')); + } else { if (cs.isNotClosed(filter)) { if (cs.specialEnumeration()) { Extensions.addString(expansion, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed", 'The code System "' + cs.system() + " has a grammar and so has infinite members. This extension is based on " + cs.specialEnumeration()); @@ -945,30 +938,15 @@ class ValueSetExpander { } } - const iter = await cs.getIterator(null); - if (valueSets.length === 0 && this.limitCount > 0 && iter.count > this.limitCount) { - throw new Issue("error", "too-costly", null, 'VALUESET_TOO_COSTLY', this.worker.i18n.translate('VALUESET_TOO_COSTLY', this.params.httpLanguages, [vsSrc.url, '>' + this.limitCount]), null, 422).withDiagnostics(this.worker.opContext.diagnostics()); - } - while (iter.more()) { - this.worker.deadCheck('processCodes#3a'); - const c = await cs.getNextContext(iter); - if (await this.passesFilters(cs, c, prep, filters, 0)) { - await this.excludeCodeAndDescendants(cs, c, expansion, valueSets, excludeInactive, vsSrc.url); - } - } - } else { - this.noTotal(); - if (cs.isNotClosed(filter)) { - notClosed.value = true; - } - const prep = await cs.getPrepContext(true); - const ctxt = await cs.searchFilter(filter, prep, false); - await cs.prepare(prep); - while (await cs.filterMore(ctxt)) { - this.worker.deadCheck('processCodes#4'); - const c = await cs.filterConcept(ctxt); - if (await this.passesFilters(cs, c, prep, filters, 0)) { - this.excludeCode(cs, await cs.system(), await cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url); + const iter = await cs.iteratorAll(); + if (iter) { + let c = await cs.nextContext(iter); + while (c) { + this.worker.deadCheck('processCodes#3a'); + if (await this.passesFilters(cs, c, prep, filters, 0)) { + this.excludeCode(cs, cs.system(), cs.version(), await cs.code(c), expansion, valueSets, vsSrc.url); + } + c = await cs.nextContext(iter); } } } @@ -1092,47 +1070,50 @@ class ValueSetExpander { if ((!this.params.excludeNotForUI || !await cs.isAbstract(context)) && (!this.params.activeOnly || !await cs.isInactive(context))) { const cds = new Designations(this.worker.i18n.languageDefinitions); await this.listDisplaysFromProvider(cds, cs, context); - for (const code of await cs.listCodes(context, this.params.altCodeRules)) { - this.worker.deadCheck('processCodeAndDescendants#2'); - this.excludeCode(cs, await cs.system(), await cs.version(), code, expansion, imports, srcUrl); - } + this.worker.deadCheck('processCodeAndDescendants#2'); + this.excludeCode(cs, await cs.system(), await cs.version(), context, expansion, imports, srcUrl); } - const iter = await cs.getIterator(context); - while (iter.more()) { + const iter = await cs.iterator(context); + let c = await cs.nextContext(iter); + while (c) { this.worker.deadCheck('processCodeAndDescendants#3'); - const c = await cs.getNextContext(iter); await this.excludeCodeAndDescendants(cs, c, expansion, imports, excludeInactive, srcUrl); + c = await cs.nextContext(iter); } } - async handleCompose(source, filter, expansion, notClosed) { + async handleCompose(source, filter, expansion, notClosed, vsInfo) { this.worker.opContext.log('compose #1'); const ts = new Map(); for (const c of source.jsonObj.compose.include || []) { this.worker.deadCheck('handleCompose#2'); - await this.checkSource(c, expansion, filter, source.url, ts); + await this.checkSource(c, expansion, filter, source.url, ts, vsInfo); } for (const c of source.jsonObj.compose.exclude || []) { this.worker.deadCheck('handleCompose#3'); this.hasExclusions = true; - await this.checkSource(c, expansion, filter, source.url, ts); + await this.checkSource(c, expansion, filter, source.url, ts, null); } this.worker.opContext.log('compose #2'); - let i = 0; - for (const c of source.jsonObj.compose.exclude || []) { - this.worker.deadCheck('handleCompose#4'); - await this.excludeCodes(c, "ValueSet.compose.exclude["+i+"]", source, filter, expansion, this.excludeInactives(source), notClosed); - } + if (vsInfo.handleByCS) { + await this.processCodes("ValueSet.compose", source, source.jsonObj.compose, filter, expansion, this.excludeInactives(source), notClosed, vsInfo); + } else { + let i = 0; + for (const c of source.jsonObj.compose.exclude || []) { + this.worker.deadCheck('handleCompose#4'); + await this.excludeCodes(c, "ValueSet.compose.exclude[" + i + "]", source, filter, expansion, this.excludeInactives(source), notClosed); + } - i = 0; - for (const c of source.jsonObj.compose.include || []) { - this.worker.deadCheck('handleCompose#5'); - await this.includeCodes(c, "ValueSet.compose.include["+i+"]", source, filter, expansion, this.excludeInactives(source), notClosed); - i++; + i = 0; + for (const c of source.jsonObj.compose.include || []) { + this.worker.deadCheck('handleCompose#5'); + await this.includeCodes(c, "ValueSet.compose.include[" + i + "]", source, source.jsonObj.compose, filter, expansion, this.excludeInactives(source), notClosed); + i++; + } } } @@ -1259,10 +1240,11 @@ class ValueSetExpander { let notClosed = { value : false}; + let vsInfo = this.scanValueSet(source.jsonObj.compose); try { if (source.jsonObj.compose && Extensions.checkNoModifiers(source.jsonObj.compose, 'ValueSetExpander.Expand', 'compose') && this.worker.checkNoLockedDate(source.url, source.jsonObj.compose)) { - await this.handleCompose(source, filter, exp, notClosed); + await this.handleCompose(source, filter, exp, notClosed, vsInfo); } const unused = new Set([...this.requiredSupplements].filter(s => !this.usedSupplements.has(s))); @@ -1338,7 +1320,7 @@ class ValueSetExpander { const c = list[i]; if (this.map.has(this.keyC(c))) { o++; - if (o > this.offset && (this.count < 0 || t < this.count)) { + if ((vsInfo.csDoOffset) || (o > this.offset && (this.count < 0 || t < this.count))) { t++; if (!exp.contains) { exp.contains = []; @@ -1362,6 +1344,43 @@ class ValueSetExpander { } } + if (result.expansion.contains && result.expansion.contains.length > 0) { + let hasChildren = false; + for (let c of result.expansion.contains) { + if (c.contains) { + hasChildren = true; + break; + } + } + + if (!hasChildren) { + let sort = this.params.sort; + let order = 1; + if (sort.startsWith('-')) { + order = -1; + sort = sort.substring(1); + } + switch (sort) { + case "design": + break; // do nothing - that's the natural order of this class + case "code" : + result.expansion.contains.sort((a, b) => order * (a.code ?? 'zzz').localeCompare(b.code ?? 'zzz')); + break; + case "display" : + result.expansion.contains.sort((a, b) => order * (a.display ?? 'zzz').localeCompare(b.display ?? 'zzz')); + break; + case "codesystem" : + // do nothing about that here + break; + default: + if (sort.startsWith("prop:")) { + result.expansion.contains.sort((a, b) => order * this.sortByProp(a, b, sort.substring(5))); + } else { + // do nothing? + } + } + } + } return result; } @@ -1454,6 +1473,12 @@ class ValueSetExpander { } isExcluded(system, version, code) { + if (this.excludedSystems.has(system)) { + return true; + } + if (this.excludedSystems.has(system+'|'+version)) { + return true; + } return this.excluded.has(system+'|'+version+'#'+code); } @@ -1533,6 +1558,79 @@ class ValueSetExpander { return undefined; } + /** + * we have a look at the value set compose to see what we have. + * If it's all one code system(|version), and has no value set dependencies, + * then we call it simple - this will affect how it can be handled later + * + * @param compose + * @returns {undefined} + */ + scanValueSet(compose) { + let result = { isSimple : false, hasExcludes : true, csset : new Set(), csDoExcludes : false, csDoOffset : false}; + let simple = true; + for (let inc of compose.include || {}) { + if (!this.isSimpleSelect(inc, result.csset)) { + simple = false; + } + } + for (let exc of compose.exclude || []) { + if (!this.isSimpleSelect(exc, result.csset)) { + simple = false; + } + result.hasExcludes = true; + } + if (simple && result.csset.size == 1) { + result.isSimple = true; + } + return result; + } + + isSimpleSelect(inc, set) { + set.add(inc.system+"|"+inc.version); + return !inc.valueset || inc.valueset.length == 0; + } + + excludeFilterList(exc) { + const results = []; + + for (const f of exc.filter || []) { + results.push({ prop: f.property, op: f.op, value: f.value }); + } + + return results; + } + + sortByProp(a, b, name) { + let pA = this.getPropValue(a, name); + let pB = this.getPropValue(b, name); + + // nulls sort last + if (pA == null && pB == null) return 0; + if (pA == null) return 1; + if (pB == null) return -1; + + // unwrap objects with a code property + if (typeof pA === 'object') pA = pA.code ?? ''; + if (typeof pB === 'object') pB = pB.code ?? ''; + + // numbers and booleans: subtract + if (typeof pA === 'number' || typeof pA === 'boolean') { + return pA - pB; + } else { + // strings + return pA.localeCompare(pB); + } + } + + getPropValue(cc, name) { + for (let p of cc.property) { + if (p.code == name) { + return getValuePrimitive(p); + } + } + return null; + } } class ExpandWorker extends TerminologyWorker { diff --git a/tx/workers/validate.js b/tx/workers/validate.js index 36ba555..7d1deea 100644 --- a/tx/workers/validate.js +++ b/tx/workers/validate.js @@ -1033,6 +1033,9 @@ class ValueSetChecker { let i = 0; let impliedSystem = { value: '' }; for (let c of code.coding || []) { + if (this.worker.opContext.usageTracker) { + this.worker.opContext.usageTracker.seeConcept(c.system, c.code); + } const csd = await this.worker.findCodeSystem(c.system, null, this.params, ['complete', 'fragment'], false, true, false, false, this.worker.requiredSupplements); this.worker.seeSourceProvider(csd, c.system); this.worker.deadCheck('check-b#1');