Add external language support (#82)

Add external language support - users can add a language_name to run on an already installed external language.

Author: JonathanZhu11

Python/buildandinstall.cmd

@@ -1,3 +1,4 @@
 del /q dist\*
 python.exe setup.py sdist --formats=zip
-python.exe -m pip install --upgrade --upgrade-strategy only-if-needed --find-links=dist sqlmlutils
+python.exe setup.py bdist_wheel
+python.exe -m pip install --upgrade --upgrade-strategy only-if-needed --find-links=dist sqlmlutils

Python/dist/sqlmlutils-1.0.3.zip

@@ -6,7 +6,7 @@
 setup(
     name='sqlmlutils',
     packages=['sqlmlutils', 'sqlmlutils/packagemanagement'],
-    version='1.0.3',
+    version='1.1.0',
     url='https://github.com/Microsoft/sqlmlutils/Python',
     license='MIT License',
     description='A client side package for working with SQL Server',

Python/dist/sqlmlutils-1.1.0.zip

@@ -9,8 +9,9 @@
 
 class CreateLibraryBuilder(SQLBuilder):
 
-    def __init__(self, pkg_name: str, pkg_filename: str, scope: Scope):
+    def __init__(self, pkg_name: str, pkg_filename: str, scope: Scope, language_name: str):
         self._name = clean_library_name(pkg_name)
+        self._language_name = language_name
         self._filename = pkg_filename
         self._scope = scope
 
@@ -23,9 +24,8 @@ def params(self):
 
     @property
     def base_script(self) -> str:
-        sqlpkgname = self._name
         authorization = _get_authorization(self._scope)
-        dummy_spees = _get_dummy_spees()
+        dummy_spees = _get_dummy_spees(self._language_name)
 
         return """
 set NOCOUNT on  
@@ -38,30 +38,39 @@ def base_script(self) -> str:
         
 -- Create the library
 CREATE EXTERNAL LIBRARY [{sqlpkgname}] {authorization}
-FROM (CONTENT = ?) WITH (LANGUAGE = 'Python');
+FROM (CONTENT = ?) WITH (LANGUAGE = '{language_name}');
 
 -- Dummy SPEES
 {dummy_spees}
 """.format(
-    sqlpkgname=sqlpkgname,
+    sqlpkgname=self._name,
     authorization=authorization,
-    dummy_spees=dummy_spees
+    dummy_spees=dummy_spees,
+    language_name=self._language_name
 )
 
 
 class CheckLibraryBuilder(SQLBuilder):
 
-    def __init__(self, pkg_name: str, scope: Scope):
+    def __init__(self, pkg_name: str, scope: Scope, language_name: str):
         self._name = clean_library_name(pkg_name)
+        self._language_name = language_name
         self._scope = scope
+        
+        if self._language_name == "Python":
+            self._private_path_env = "MRS_EXTLIB_USER_PATH"
+            self._public_path_env = "MRS_EXTLIB_SHARED_PATH"
+        else:
+            self._private_path_env = "PRIVATELIBPATH"
+            self._public_path_env = "PUBLICLIBPATH"
 
     @property
     def params(self):
         return """ 
 import os
 import re
-_ENV_NAME_USER_PATH = "MRS_EXTLIB_USER_PATH"
-_ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
+_ENV_NAME_USER_PATH = "{private_path_env}"
+_ENV_NAME_SHARED_PATH = "{public_path_env}"
 
 def _is_dist_info_file(name, file):
     return re.match(name + r"-.*egg", file) or re.match(name + r"-.*dist-info", file)
@@ -92,29 +101,33 @@ def package_exists_in_scope(sql_package_name: str, scope=None) -> bool:
 # Check that the package exists in scope.
 # For some reason this check works but there is a bug in pyODBC when asserting this is True.
 assert package_exists_in_scope("{name}", "{scope}") != False
-""".format(name=self._name, scope=self._scope._name)
+""".format(private_path_env=self._private_path_env, 
+        public_path_env=self._public_path_env, 
+        name=self._name, 
+        scope=self._scope._name)
 
     @property
     def base_script(self) -> str:
         return """    
 -- Check to make sure the package was installed
 BEGIN TRY
-    exec sp_execute_external_script
-    @language = N'Python',
+    EXEC sp_execute_external_script
+    @language = N'{language_name}',
     @script = ?
     print('Package successfully installed.')
 END TRY
 BEGIN CATCH
     print('Package installation failed.');
     THROW;
 END CATCH
-"""
+""".format(language_name = self._language_name)
 
 
 class DropLibraryBuilder(SQLBuilder):
 
-    def __init__(self, sql_package_name: str, scope: Scope):
+    def __init__(self, sql_package_name: str, scope: Scope, language_name: str):
         self._name = clean_library_name(sql_package_name)
+        self._language_name = language_name
         self._scope = scope
 
     @property
@@ -126,10 +139,9 @@ def base_script(self) -> str:
 """.format(
     name=self._name,
     auth=_get_authorization(self._scope),
-    dummy_spees=_get_dummy_spees()
+    dummy_spees=_get_dummy_spees(self._language_name)
 )
 
-
 def clean_library_name(pkgname: str):
     return pkgname.replace("-", "_").lower()
 
@@ -138,9 +150,9 @@ def _get_authorization(scope: Scope) -> str:
     return "AUTHORIZATION dbo" if scope == Scope.public_scope() else ""
 
 
-def _get_dummy_spees() -> str:
+def _get_dummy_spees(language_name: str) -> str:
     return """
-exec sp_execute_external_script
-@language = N'Python',
+EXEC sp_execute_external_script
+@language = N'{language_name}',
 @script = N''
-"""
+""".format(language_name = language_name)

Python/setup.py

@@ -12,11 +12,12 @@
 
 class PipDownloader:
 
-    def __init__(self, connection: ConnectionInfo, downloaddir: str, targetpackage: str):
+    def __init__(self, connection: ConnectionInfo, downloaddir: str, targetpackage: str, language_name: str):
         self._connection = connection
         self._downloaddir = downloaddir
         self._targetpackage = targetpackage
-        server_info = SQLPythonExecutor(connection).execute_function_in_sql(servermethods.get_server_info)
+        self._language_name = language_name
+        server_info = SQLPythonExecutor(connection, self._language_name).execute_function_in_sql(servermethods.get_server_info)
         globals().update(server_info)
 
     def download(self):

Python/sqlmlutils/packagemanagement/packagesqlbuilder.py

@@ -6,9 +6,6 @@
 
 from sqlmlutils.packagemanagement.scope import Scope
 
-_ENV_NAME_USER_PATH = "MRS_EXTLIB_USER_PATH"
-_ENV_NAME_SHARED_PATH = "MRS_EXTLIB_SHARED_PATH"
-
 def show_installed_packages():
     import pkg_resources
     return [(d.project_name, d.version) for d in pkg_resources.working_set]
@@ -30,40 +27,3 @@ def get_server_info():
         "abi_tag": pep425tags.get_abi_tag(), #'cp37m'
         "platform": sysconfig.get_platform().replace("-","_") #'win_amd64', 'linux_x86_64'
     }
-
-
-def check_package_install_success(sql_package_name: str) -> bool:
-    return package_exists_in_scope(sql_package_name)
-
-
-def package_files_in_scope(scope=Scope.private_scope()):
-    envdir = _ENV_NAME_SHARED_PATH if scope == Scope.public_scope() or os.environ.get(_ENV_NAME_USER_PATH, "") == "" \
-        else _ENV_NAME_USER_PATH
-    path = os.environ.get(envdir, "")
-    if os.path.isdir(path):
-        return os.listdir(path)
-    return []
-
-
-def package_exists_in_scope(sql_package_name: str, scope=None) -> bool:
-    if scope is None:
-        # default to user path for every user but DBOs
-        scope = Scope.public_scope() if (os.environ.get(_ENV_NAME_USER_PATH, "") == "") else Scope.private_scope()
-    package_files = package_files_in_scope(scope)
-    return any([_is_package_match(sql_package_name, package_file) for package_file in package_files])
-
-
-def _is_dist_info_file(name, file):
-    return re.match(name + r'-.*egg', file) or re.match(name + r'-.*dist-info', file)
-
-
-def _is_package_match(package_name, file):
-    package_name = package_name.lower()
-    file = file.lower()
-    return file == package_name or file == package_name + ".py" or \
-           _is_dist_info_file(package_name, file) or \
-           ("-" in package_name and
-            (package_name.split("-")[0] == file or _is_dist_info_file(package_name.replace("-", "_"), file)))
-
-
-

Python/sqlmlutils/packagemanagement/pipdownloader.py

@@ -19,9 +19,15 @@
 
 class SQLPackageManager:
 
-    def __init__(self, connection_info: ConnectionInfo):
+    def __init__(self, connection_info: ConnectionInfo, language_name: str = "Python"):
+        """Initialize a SQLPackageManager to manage packages on the SQL Server.
+
+        :param connection_info: The ConnectionInfo object that holds the connection string and other information.
+        :param language_name: The name of the language to be executed in sp_execute_external_script, if using EXTERNAL LANGUAGE. 
+        """
         self._connection_info = connection_info
-        self._pyexecutor = SQLPythonExecutor(connection_info)
+        self._pyexecutor = SQLPythonExecutor(connection_info, language_name=language_name)
+        self._language_name = language_name
 
     def install(self,
                 package: str,
@@ -104,29 +110,32 @@ def _get_default_scope(self):
         return Scope.public_scope() if is_sysadmin == 1 else Scope.private_scope()
 
     def _get_packages_by_user(self, owner='', scope: Scope=Scope.private_scope()):
-        has_user = (owner != '')
+        scope_num = 1 if scope == Scope.private_scope() else 0
+        
+        if scope_num == 0 and owner == '':
+            owner = "dbo"
 
         query = "DECLARE @principalId INT;  \
                 DECLARE @currentUser NVARCHAR(128);  \
                 SELECT @currentUser = "
 
-        if has_user:
+        if owner != '':
             query += "?;\n"
         else:
             query += "CURRENT_USER;\n"
 
-        scope_num = 1 if scope == Scope.private_scope() else 0
-
         query += "SELECT @principalId = USER_ID(@currentUser);  \
                        SELECT name, language, scope   \
                        FROM sys.external_libraries AS elib   \
                        WHERE elib.principal_id=@principalId   \
-                       AND elib.language='Python' AND elib.scope={scope_num}   \
-                       ORDER BY elib.name ASC;".format(scope_num=scope_num)
+                       AND elib.language='{language_name}' AND elib.scope={scope_num}   \
+                       ORDER BY elib.name ASC; \
+                       GO".format(language_name=self._language_name,
+                                  scope_num=scope_num)
         return self._pyexecutor.execute_sql_query(query, owner)
 
     def _drop_sql_package(self, sql_package_name: str, scope: Scope, out_file: str = None):
-        builder = DropLibraryBuilder(sql_package_name=sql_package_name, scope=scope)
+        builder = DropLibraryBuilder(sql_package_name=sql_package_name, scope=scope, language_name=self._language_name)
         execute_query(builder, self._connection_info, out_file)
 
     # TODO: Support not dependencies
@@ -146,7 +155,7 @@ def _install_from_pypi(self,
             target_package = target_package + "==" + version
 
         with tempfile.TemporaryDirectory() as temporary_directory:
-            pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package)
+            pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package, language_name = self._language_name)
             target_package_file = pipdownloader.download_single()
             self._install_from_file(target_package_file, scope, upgrade, out_file=out_file)
 
@@ -162,7 +171,7 @@ def _install_from_file(self, target_package_file: str, scope: Scope, upgrade: bo
 
         # Download requirements from PyPI
         with tempfile.TemporaryDirectory() as temporary_directory:
-            pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package_file)
+            pipdownloader = PipDownloader(self._connection_info, temporary_directory, target_package_file, language_name = self._language_name)
 
             # For now, we download all target package dependencies from PyPI.
             target_package_requirements, requirements_downloaded = pipdownloader.download()
@@ -189,8 +198,7 @@ def _install_many(self, target_package_file: str, dependency_files, scope: Scope
                 sqlexecutor._cnxn.rollback()
                 raise RuntimeError("Package installation failed, installed dependencies were rolled back.") from e
 
-    @staticmethod
-    def _install_single(sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False, out_file: str=None):
+    def _install_single(self, sqlexecutor: SQLQueryExecutor, package_file: str, scope: Scope, is_target=False, out_file: str=None):
         name = str(get_package_name_from_file(package_file))
         version = str(get_package_version_from_file(package_file))
         print("Installing {name} version: {version}".format(name=name, version=version))
@@ -200,9 +208,10 @@ def _install_single(sqlexecutor: SQLQueryExecutor, package_file: str, scope: Sco
             with zipfile.ZipFile(prezip, 'w') as zipf:
                 zipf.write(package_file, os.path.basename(package_file))
 
-            builder = CreateLibraryBuilder(pkg_name=name, pkg_filename=prezip, scope=scope)
+            builder = CreateLibraryBuilder(pkg_name=name, pkg_filename=prezip, scope=scope, language_name=self._language_name)
             sqlexecutor.execute(builder, out_file=out_file)
-            builder = CheckLibraryBuilder(pkg_name=name, scope=scope)
+            
+            builder = CheckLibraryBuilder(pkg_name=name, scope=scope, language_name=self._language_name)
             sqlexecutor.execute(builder, out_file=out_file)
 
     @staticmethod