diff --git a/registry/api.js b/registry/api.js index 3d352da..25c9771 100644 --- a/registry/api.js +++ b/registry/api.js @@ -23,57 +23,51 @@ class RegistryAPI { const rows = []; const data = this.crawler.getData(); - // Lock for thread safety during read - data.lock('buildRowsCS'); - try { - data.registries.forEach(registry => { - if (registryCode && registry.code !== registryCode) return; - - registry.servers.forEach(server => { - if (serverCode && server.code !== serverCode) return; + data.registries.forEach(registry => { + if (registryCode && registry.code !== registryCode) return; - // Check if server is authoritative for this code system - const isAuth = codeSystem ? ServerRegistryUtilities.hasMatchingCodeSystem( - codeSystem, - server.authCSList, - true // support wildcards - ) : false; - - server.versions.forEach(versionInfo => { - if (version && !ServerRegistryUtilities.versionMatches(version, versionInfo.version)) { - return; - } + registry.servers.forEach(server => { + if (serverCode && server.code !== serverCode) return; + + // Check if server is authoritative for this code system + const isAuth = codeSystem ? ServerRegistryUtilities.hasMatchingCodeSystem( + codeSystem, + server.authCSList, + true // support wildcards + ) : false; + + server.versions.forEach(versionInfo => { + if (version && !ServerRegistryUtilities.versionMatches(version, versionInfo.version)) { + return; + } - // Always skip servers with errors - they can't serve requests - if (versionInfo.error) { - return; - } + // Always skip servers with errors - they can't serve requests + if (versionInfo.error) { + return; + } - // Include if: - // 1. Authoritative for the requested code system - // 2. No filter specified - // 3. Has the code system in its list - if (isAuth || - !codeSystem || - (codeSystem && ServerRegistryUtilities.hasMatchingCodeSystem( - codeSystem, - versionInfo.codeSystems, - false // no wildcards for actual content - ))) { - const row = ServerRegistryUtilities.createRow( - registry, - server, - versionInfo, - isAuth - ); - rows.push(row); - } - }); + // Include if: + // 1. Authoritative for the requested code system + // 2. No filter specified + // 3. Has the code system in its list + if (isAuth || + !codeSystem || + (codeSystem && ServerRegistryUtilities.hasMatchingCodeSystem( + codeSystem, + versionInfo.codeSystems, + false // no wildcards for actual content + ))) { + const row = ServerRegistryUtilities.createRow( + registry, + server, + versionInfo, + isAuth + ); + rows.push(row); + } }); }); - } finally { - data.unlock(); - } + }); return this._sortAndRankRows(rows); } @@ -93,70 +87,65 @@ class RegistryAPI { const rows = []; const data = this.crawler.getData(); - data.lock('buildRowsVS'); - try { - data.registries.forEach(registry => { - if (registryCode && registry.code !== registryCode) return; - - registry.servers.forEach(server => { - if (serverCode && server.code !== serverCode) return; + data.registries.forEach(registry => { + if (registryCode && registry.code !== registryCode) return; - // Check if server is authoritative for this value set - const isAuth = valueSet ? ServerRegistryUtilities.hasMatchingValueSet( - valueSet, - server.authVSList, - true // support wildcards - ) : false; - - server.versions.forEach(versionInfo => { - if (version && !ServerRegistryUtilities.versionMatches(version, versionInfo.version)) { - return; - } + registry.servers.forEach(server => { + if (serverCode && server.code !== serverCode) return; + + // Check if server is authoritative for this value set + const isAuth = valueSet ? ServerRegistryUtilities.hasMatchingValueSet( + valueSet, + server.authVSList, + true // support wildcards + ) : false; + + server.versions.forEach(versionInfo => { + if (version && !ServerRegistryUtilities.versionMatches(version, versionInfo.version)) { + return; + } - // Always skip servers with errors - they can't serve requests - if (versionInfo.error) { - return; - } + // Always skip servers with errors - they can't serve requests + if (versionInfo.error) { + return; + } - // Include if: - // 1. No filter specified - // 2. Authoritative for the value set (even via wildcard) - // 3. Has the value set in its list - let includeRow = false; + // Include if: + // 1. No filter specified + // 2. Authoritative for the value set (even via wildcard) + // 3. Has the value set in its list + let includeRow = false; - if (!valueSet) { - // No filter, include all working servers + if (!valueSet) { + // No filter, include all working servers + includeRow = true; + } else { + // Check if actually has the value set + const hasValueSet = ServerRegistryUtilities.hasMatchingValueSet( + valueSet, + versionInfo.valueSets, + false // no wildcards for actual content + ); + + // Include if authoritative OR has the value set + // This matches the Pascal logic: if auth or hasMatchingValueSet + if (isAuth || hasValueSet) { includeRow = true; - } else { - // Check if actually has the value set - const hasValueSet = ServerRegistryUtilities.hasMatchingValueSet( - valueSet, - versionInfo.valueSets, - false // no wildcards for actual content - ); - - // Include if authoritative OR has the value set - // This matches the Pascal logic: if auth or hasMatchingValueSet - if (isAuth || hasValueSet) { - includeRow = true; - } } + } - if (includeRow) { - const row = ServerRegistryUtilities.createRow( - registry, - server, - versionInfo, - isAuth - ); - rows.push(row); - } - }); + if (includeRow) { + const row = ServerRegistryUtilities.createRow( + registry, + server, + versionInfo, + isAuth + ); + rows.push(row); + } }); }); - } finally { - data.unlock(); - } + }); return this._sortAndRankRows(rows); } @@ -254,7 +243,7 @@ class RegistryAPI { workingVersions++; } - version.codeSystems.forEach(cs => totalCodeSystems.add(cs)); + version.codeSystems.forEach(cs => totalCodeSystems.add(cs.uri+(cs.version ? '|'+cs.version : ''))); version.valueSets.forEach(vs => totalValueSets.add(vs)); }); }); @@ -415,13 +404,49 @@ class RegistryAPI { baseCodeSystem = codeSystem.substring(0, codeSystem.indexOf('|')); } - // Lock for thread safety during read - data.lock('resolveCS'); - try { + data.registries.forEach(registry => { + registry.servers.forEach(server => { + let added = false; + + // Check if server supports the requested usage tag + if (server.usageList.length === 0 || + (usage && server.usageList.includes(usage))) { + + // Check if server is authoritative for this code system + const isAuth = server.isAuthCS(codeSystem); + + server.versions.forEach(version => { + if (ServerRegistryUtilities.versionMatches(normalizedVersion, version.version)) { + // Check if the server has the code system + // Test against both the full URL and the base URL + let content = {}; + const hasMatchingCS = + ServerRegistryUtilities.hasMatchingCodeSystem(baseCodeSystem, version.codeSystems, false, content) || + (baseCodeSystem !== codeSystem && + ServerRegistryUtilities.hasMatchingCodeSystem(codeSystem, version.codeSystems, false, content)); + + if (hasMatchingCS) { + if (isAuth) { + result.authoritative.push(this.createServerEntry(server, version)); + } else if (!authoritativeOnly) { + result.candidates.push(this.createServerEntry(server, version, content.content)); + } + added = true; + } + } + }); + + if (added) { + matchedServers.push(server.code); + } + } + }); + }); + + // NEW: Fallback - if no matches found, check for authoritative pattern matches + if (result.authoritative.length === 0 && result.candidates.length === 0) { data.registries.forEach(registry => { registry.servers.forEach(server => { - let added = false; - // Check if server supports the requested usage tag if (server.usageList.length === 0 || (usage && server.usageList.includes(usage))) { @@ -429,64 +454,23 @@ class RegistryAPI { // Check if server is authoritative for this code system const isAuth = server.isAuthCS(codeSystem); - server.versions.forEach(version => { - if (ServerRegistryUtilities.versionMatches(normalizedVersion, version.version)) { - // Check if the server has the code system - // Test against both the full URL and the base URL - const hasMatchingCS = - ServerRegistryUtilities.hasMatchingCodeSystem(baseCodeSystem, version.codeSystems, false) || - (baseCodeSystem !== codeSystem && - ServerRegistryUtilities.hasMatchingCodeSystem(codeSystem, version.codeSystems, false)); - - if (hasMatchingCS) { - if (isAuth) { - result.authoritative.push(this.createServerEntry(server, version)); - } else if (!authoritativeOnly) { - result.candidates.push(this.createServerEntry(server, version)); + if (isAuth) { + server.versions.forEach(version => { + if (ServerRegistryUtilities.versionMatches(normalizedVersion, version.version)) { + result.authoritative.push(this.createServerEntry(server, version)); + if (!matchedServers.includes(server.code)) { + matchedServers.push(server.code); } - added = true; } - } - }); - - if (added) { - matchedServers.push(server.code); + }); } } }); }); - - // NEW: Fallback - if no matches found, check for authoritative pattern matches - if (result.authoritative.length === 0 && result.candidates.length === 0) { - data.registries.forEach(registry => { - registry.servers.forEach(server => { - // Check if server supports the requested usage tag - if (server.usageList.length === 0 || - (usage && server.usageList.includes(usage))) { - - // Check if server is authoritative for this code system - const isAuth = server.isAuthCS(codeSystem); - - if (isAuth) { - server.versions.forEach(version => { - if (ServerRegistryUtilities.versionMatches(normalizedVersion, version.version)) { - result.authoritative.push(this.createServerEntry(server, version)); - if (!matchedServers.includes(server.code)) { - matchedServers.push(server.code); - } - } - }); - } - } - }); - }); - } - } finally { - data.unlock(); } return { - result : this._cleanEmptyArrays(result), + result: this._cleanEmptyArrays(result), matches: matchedServers.length > 0 ? matchedServers.join(',') : '--' }; } @@ -522,50 +506,45 @@ class RegistryAPI { } // Lock for thread safety during read - data.lock('resolveVS'); - try { - data.registries.forEach(registry => { - registry.servers.forEach(server => { - let added = false; + data.registries.forEach(registry => { + registry.servers.forEach(server => { + let added = false; - // Check if server supports the requested usage tag - if (server.usageList.length === 0 || - (usage && server.usageList.includes(usage))) { + // Check if server supports the requested usage tag + if (server.usageList.length === 0 || + (usage && server.usageList.includes(usage))) { - // Check if server is authoritative for this value set - const isAuth = server.isAuthVS(baseValueSet); + // Check if server is authoritative for this value set + const isAuth = server.isAuthVS(baseValueSet); - server.versions.forEach(version => { - if (ServerRegistryUtilities.versionMatches(normalizedVersion, version.version)) { - // For authoritative servers, we don't need to check if they have the value set - if (isAuth) { - result.authoritative.push(this.createServerEntry(server, version)); - added = true; - } - // For non-authoritative servers, check if they have the value set - else if (ServerRegistryUtilities.hasMatchingValueSet(baseValueSet, version.valueSets, false) || - (baseValueSet !== valueSet && - ServerRegistryUtilities.hasMatchingValueSet(valueSet, version.valueSets, false))) { - if (!authoritativeOnly) { - result.candidates.push(this.createServerEntry(server, version)); - } - added = true; - } + server.versions.forEach(version => { + if (ServerRegistryUtilities.versionMatches(normalizedVersion, version.version)) { + // For authoritative servers, we don't need to check if they have the value set + if (isAuth) { + result.authoritative.push(this.createServerEntry(server, version)); + added = true; + } + // For non-authoritative servers, check if they have the value set + else if (ServerRegistryUtilities.hasMatchingValueSet(baseValueSet, version.valueSets, false) || + (baseValueSet !== valueSet && + ServerRegistryUtilities.hasMatchingValueSet(valueSet, version.valueSets, false))) { + if (!authoritativeOnly) { + result.candidates.push(this.createServerEntry(server, version)); } - }); - - if (added) { - matchedServers.push(server.code); + added = true; + } } + }); + + if (added) { + matchedServers.push(server.code); } - }); + } }); - } finally { - data.unlock(); - } + }); return { - result : this._cleanEmptyArrays(result), + result: this._cleanEmptyArrays(result), matches: matchedServers.length > 0 ? matchedServers.join(',') : '--' }; } @@ -598,6 +577,9 @@ class RegistryAPI { if (server.accessInfo) { entry.access_info = server.accessInfo; } + if (version.content) { + entry.content = version.content; + } return entry; } diff --git a/registry/crawler.js b/registry/crawler.js index 26a7c56..d01fe97 100644 --- a/registry/crawler.js +++ b/registry/crawler.js @@ -8,6 +8,7 @@ const { ServerInformation, ServerVersionInformation, } = require('./model'); +const {Extensions} = require("../tx/library/extensions"); const MASTER_URL = 'https://fhir.github.io/ig-registry/tx-servers.json'; @@ -36,32 +37,6 @@ class RegistryCrawler { this.log = logv; } - // /** - // * Start the crawler with periodic updates - // */ - // start() { - // if (this.crawlTimer) { - // return; // Already running - // } - // - // // Initial crawl - // this.crawl(); - // - // // Set up periodic crawling - // this.crawlTimer = setInterval(() => { - // this.crawl(); - // }, this.config.crawlInterval); - // } - // - // /** - // * Stop the crawler - // */ - // stop() { - // if (this.crawlTimer) { - // clearInterval(this.crawlTimer); - // this.crawlTimer = null; - // } - // } /** * Main entry point - crawl the registry starting from the master URL @@ -111,6 +86,7 @@ class RegistryCrawler { // Update the current data this.currentData = newData; } catch (error) { + console.log(error); this.addLogEntry('error', 'Exception Scanning:', error); this.currentData.outcome = `Error: ${error.message}`; this.errors.push({ @@ -165,6 +141,7 @@ class RegistryCrawler { } } catch (error) { + console.log(error); registry.error = error.message; this.addLogEntry('error', `Exception processing registry ${registry.name}: ${error.message}`, registry.address); } @@ -181,7 +158,7 @@ class RegistryCrawler { server.name = serverConfig.name; server.address = serverConfig.url || ''; server.accessInfo = serverConfig.access_info || ''; - + if (!server.name) { this.addLogEntry('error', 'No name provided for server', source); return server; @@ -200,7 +177,7 @@ class RegistryCrawler { // Process each FHIR version const fhirVersions = serverConfig.fhirVersions || []; for (const versionConfig of fhirVersions) { - const version = await this.processServerVersion(versionConfig, server); + const version = await this.processServerVersion(versionConfig, server, serverConfig.exclusions); if (version) { server.versions.push(version); } @@ -212,7 +189,7 @@ class RegistryCrawler { /** * Process a single server version */ - async processServerVersion(versionConfig, server) { + async processServerVersion(versionConfig, server, exclusions) { const version = new ServerVersionInformation(); version.version = versionConfig.version; version.address = versionConfig.url; @@ -233,20 +210,20 @@ class RegistryCrawler { switch (majorVersion) { case 3: - await this.processServerVersionR3(version, server); + await this.processServerVersionR3(version, server, exclusions); break; case 4: - await this.processServerVersionR4(version, server); + await this.processServerVersionR4or5(version, server, '4.0.1', exclusions); break; case 5: - await this.processServerVersionR5(version, server); + await this.processServerVersionR4or5(version, server, '5.0.0', exclusions); break; default: throw new Error(`Version ${version.version} not supported`); } // Sort and deduplicate - version.codeSystems = [...new Set(version.codeSystems)].sort(); + version.codeSystems.sort((a, b) => this.compareCS(a, b)); version.valueSets = [...new Set(version.valueSets)].sort(); version.lastSuccess = new Date(); version.lastTat = `${Date.now() - startTime}ms`; @@ -254,6 +231,7 @@ class RegistryCrawler { this.addLogEntry('info', ` Server ${version.address}: ${version.lastTat} for ${version.codeSystems.length} CodeSystems and ${version.valueSets.length} ValueSets`); } catch (error) { + console.log(error); const elapsed = Date.now() - startTime; this.addLogEntry('error', `Server ${version.address}: Error after ${elapsed}ms: ${error.message}`); version.error = error.message; @@ -266,7 +244,7 @@ class RegistryCrawler { /** * Process an R3 server */ - async processServerVersionR3(version, server) { + async processServerVersionR3(version, server, exclusions) { // Get capability statement const capabilityUrl = `${version.address}/metadata`; const capability = await this.fetchJson(capabilityUrl, server.name); @@ -283,12 +261,12 @@ class RegistryCrawler { termCap.parameter.forEach(param => { if (param.name === 'system') { const uri = param.valueUri || param.valueString; - if (uri) { + if (uri && !this.isExcluded(uri, exclusions)) { version.codeSystems.push(uri); // Look for version parts if (param.part) { param.part.forEach(part => { - if (part.name === 'version' && part.valueString) { + if (part.name === 'version' && part.valueString && !this.isExcluded(uri+'|'+part.valueString, exclusions)) { version.codeSystems.push(`${uri}|${part.valueString}`); } }); @@ -298,24 +276,27 @@ class RegistryCrawler { }); } } catch (error) { + console.log(error); this.addLogEntry('error', `Could not fetch terminology capabilities: ${error.message}`); } // Search for value sets - await this.fetchValueSets(version, server); + await this.fetchValueSets(version, server, exclusions); } /** * Process an R4 server */ - async processServerVersionR4(version, server) { + async processServerVersionR4or5(version, server, defVersion, exclusions) { // Get capability statement const capabilityUrl = `${version.address}/metadata`; const capability = await this.fetchJson(capabilityUrl, server.code); - version.version = capability.fhirVersion || '4.0.1'; + version.version = capability.fhirVersion || defVersion; version.software = capability.software ? capability.software.name : "unknown"; - + + let set = new Set(); + // Get terminology capabilities try { const termCapUrl = `${version.address}/metadata?mode=terminology`; @@ -323,12 +304,19 @@ class RegistryCrawler { if (termCap.codeSystem) { termCap.codeSystem.forEach(cs => { - if (cs.uri) { - version.codeSystems.push(cs.uri); + let content = cs.content || Extensions.readString(cs, "http://hl7.org/fhir/5.0/StructureDefinition/extension-TerminologyCapabilities.codeSystem.content"); + if (cs.uri && !this.isExcluded(cs.uri, exclusions)) { + if (!set.has(cs.uri)) { + set.add(cs.uri); + version.codeSystems.push(this.addContent({uri: cs.uri}, content)); + } if (cs.version) { cs.version.forEach(v => { - if (v.code) { - version.codeSystems.push(`${cs.uri}|${v.code}`); + if (v.code && !this.isExcluded(cs.uri+"|"+v.code, exclusions)) { + if (!set.has(cs.uri+"|"+v.code)) { + version.codeSystems.push(this.addContent({uri: cs.uri, version: v.code}, content)); + set.add(cs.uri+"|"+v.code); + } } }); } @@ -336,20 +324,12 @@ class RegistryCrawler { }); } } catch (error) { + console.log(error); this.addLogEntry('error', `Could not fetch terminology capabilities: ${error.message}`); } // Search for value sets - await this.fetchValueSets(version, server); - } - - /** - * Process an R5 server - */ - async processServerVersionR5(version, server) { - // R5 is essentially the same as R4 for our purposes - await this.processServerVersionR4(version, server); - version.version = version.version || '5.0.0'; + await this.fetchValueSets(version, server, exclusions); } /** @@ -360,9 +340,9 @@ class RegistryCrawler { * @param {Object} version - The server version information * @param {Object} server - The server information */ - async fetchValueSets(version, server) { + async fetchValueSets(version, server, exclusions) { // Initial search URL - let searchUrl = `${version.address}/ValueSet?_elements=url,version`; + let searchUrl = `${version.address}/ValueSet?_elements=url,version`+(version.address.includes("fhir.org") ? "&_count=200" : ""); try { // Set of URLs to avoid duplicates const valueSetUrls = new Set(); @@ -378,9 +358,9 @@ class RegistryCrawler { bundle.entry.forEach(entry => { if (entry.resource) { const vs = entry.resource; - if (vs.url) { + if (vs.url && !this.isExcluded(vs.url, exclusions)) { valueSetUrls.add(vs.url); - if (vs.version) { + if (vs.version && !this.isExcluded(vs.url+'|'+vs.version, exclusions)) { valueSetUrls.add(`${vs.url}|${vs.version}`); } } @@ -402,6 +382,7 @@ class RegistryCrawler { version.valueSets = Array.from(valueSetUrls).sort(); } catch (error) { + console.log(error); this.addLogEntry('error', `Could not fetch value sets: ${error.message} from ${searchUrl}`); } } @@ -478,6 +459,7 @@ class RegistryCrawler { return response.data; } catch (error) { + console.log(error); if (error.response) { throw new Error(`HTTP ${error.response.status}: ${error.response.statusText}`); } else if (error.request) { @@ -634,6 +616,38 @@ class RegistryCrawler { return filteredLogs.slice(-limit); } + addContent(param, content) { + if (content) { + param.content = content; + } + return param; + } + + compareCS(a, b) { + if (a.version || b.version) { + let s = (a.uri+'|'+a.version) || ''; + return s.localeCompare(b.uri+'|'+b.version); + } else { + return (a.uri || '').localeCompare(b.uri); + } + } + + isExcluded(url, exclusions) { + for (let exclusion of exclusions || []) { + let match = false; + if (exclusion.endsWith('*')) { + const prefix = exclusion.slice(0, -1); + match = url.startsWith(prefix); + } else { + // Otherwise do exact matching on both full and base URL + match = url === exclusion; + } + if (match) { + return true; + } + } + return false; + } } module.exports = RegistryCrawler; \ No newline at end of file diff --git a/registry/model.js b/registry/model.js index c364f03..cbdc085 100644 --- a/registry/model.js +++ b/registry/model.js @@ -37,7 +37,7 @@ class ServerVersionInformation { getCsListHtml() { if (this.codeSystems.length === 0) return '