fix: use MS SQL Server ODBC driver v18 by default

v17 package is no longer available in the Microsoft repository

.ci/docker-compose-file/.env

@@ -15,7 +15,7 @@ HSTREAMDB_ZK_TAG=3.8.1
 DATALAYERS_TAG=v2.1.7
 
 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server
-SQLSERVER_TAG=2019-CU19-ubuntu-20.04
+SQLSERVER_TAG=2022-CU15-ubuntu-22.04
 
 
 # Password for the 'elastic' user (at least 6 characters)

.ci/docker-compose-file/docker-compose-sqlserver.yaml

@@ -1,5 +1,3 @@
-version: '3.9'
-
 services:
   sql_server:
     container_name: sqlserver

.ci/docker-compose-file/docker-compose-toxiproxy.yaml

@@ -1,5 +1,3 @@
-version: '3.9'
-
 services:
   toxiproxy:
     container_name: toxiproxy

.ci/docker-compose-file/docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '3.9'
-
 services:
   erlang:
     hostname: erlang.emqx.net

.ci/docker-compose-file/odbc/odbcinst.ini

@@ -1,9 +1,9 @@
 [ms-sql]
-Description=Microsoft ODBC Driver 17 for SQL Server
-Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1
+Description=Microsoft ODBC Driver 18 for SQL Server
+Driver=/opt/microsoft/msodbcsql18/lib64/libmsodbcsql-18.4.so.1.1
 UsageCount=1
 
-[ODBC Driver 17 for SQL Server]
-Description=Microsoft ODBC Driver 17 for SQL Server
-Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1
+[ODBC Driver 18 for SQL Server]
+Description=Microsoft ODBC Driver 18 for SQL Server
+Driver=/opt/microsoft/msodbcsql18/lib64/libmsodbcsql-18.4.so.1.1
 UsageCount=1

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl

@@ -371,9 +371,7 @@ do_get_status(Conn) ->
 %% 'https://learn.microsoft.com/en-us/sql/connect/odbc/
 %%      dsn-connection-string-attribute?source=recommendations&view=sql-server-ver16#encrypt'
 conn_str([], Acc) ->
-    %% we should use this for msodbcsql 18+
-    %% lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]);
-    lists:join(";", Acc);
+    lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]);
 conn_str([{driver, Driver} | Opts], Acc) ->
     conn_str(Opts, ["Driver=" ++ str(Driver) | Acc]);
 conn_str([{server, Server} | Opts], Acc) ->

apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl

@@ -354,7 +354,7 @@ t_simple_query(Config) ->
     ok.
 
 -define(MISSING_TINYINT_ERROR,
-    "[Microsoft][ODBC Driver 17 for SQL Server][SQL Server]"
+    "[Microsoft][ODBC Driver 18 for SQL Server][SQL Server]"
     "Conversion failed when converting the varchar value 'undefined' to data type tinyint. SQLSTATE IS: 22018"
 ).
 
@@ -654,11 +654,7 @@ batch_size(Config) ->
     end.
 
 conn_str([], Acc) ->
-    %% TODO: for msodbc 18+, we need to add "Encrypt=YES;TrustServerCertificate=YES"
-    %% but havn't tested now
-    %% we should use this for msodbcsql 18+
-    %% lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]);
-    lists:join(";", Acc);
+    lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]);
 conn_str([{driver, Driver} | Opts], Acc) ->
     conn_str(Opts, ["Driver=" ++ str(Driver) | Acc]);
 conn_str([{host, Host} | Opts], Acc) ->

deploy/docker/Dockerfile.msodbc

@@ -1,26 +1,31 @@
+## Extended version of emqx/emqx-enterprise image with MS SQL Server ODBC driver
+
 ## This Dockerfile should not run in GitHub Action or any other automated process.
 ## It should be manually executed by the needs of the user.
 ##
-## Before manaually execute:
-## Please confirm the EMQX-Enterprise version you are using and modify the base layer image tag
-## ```bash
-## $ docker build -f=Dockerfile.msodbc -t emqx-enterprise-with-msodbc:5.0.3-alpha.2 .
-## ```
+## docker build -f deploy/docker/Dockerfile.msodbc \
+##   --build-arg BUILD_FROM=emqx/emqx-enterprise:5.8.0 \
+##   -t emqx/emqx-enterprise:5.8.0-msodbc .
+##
+## or like this
+##
+## DOCKER_BUILD_NOCACHE=true \
+## PKG_VSN=5.8.0 \
+## EMQX_DOCKERFILE=deploy/docker/Dockerfile.msodbc \
+## BUILD_FROM=emqx/emqx-enterprise:5.8.0 \
+## EMQX_IMAGE_TAG=emqx/emqx-enterprise:5.8.0-msodbc \
+## ./build emqx-enterprise docker
 
-ARG VERSION=5.5.0
-FROM emqx/emqx-enterprise:$VERSION
+ARG BUILD_FROM=emqx/emqx-enterprise:5.8.0
+FROM $BUILD_FROM
 
 USER root
 
-RUN apt-get update \
-    && apt-get install -y gnupg2 curl apt-utils \
-    && . /etc/os-release \
-    && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
-    && curl "https://packages.microsoft.com/config/debian/$VERSION_ID/prod.list" > /etc/apt/sources.list.d/mssql-release.list \
-    && apt-get update \
-    && ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev \
-    && sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+COPY scripts/install-msodbc-driver.sh ./
+
+RUN ./install-msodbc-driver.sh && \
+    rm -f install-msodbc-driver.sh && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 USER emqx

scripts/install-msodbc-driver.sh

@@ -1,22 +1,28 @@
 #!/usr/bin/env bash
 
-## this script install msodbcsql17 and unixodbc-dev on ci environment
-## specific to ubuntu 16.04, 18.04, 20.04, 22.04
+# Install the MS SQL Server ODBC driver on a Debian-based system.
+# https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server
+
 set -euo pipefail
 
-# install msodbcsql17
-VERSION=$(lsb_release -rs)
-if ! [[ "16.04 18.04 20.04 22.04" == *"$VERSION"* ]];
-then
-    echo "Ubuntu $VERSION is not currently supported.";
-    exit 1;
+if [ ! -f /etc/debian_version ]; then
+  echo "This script is only intended for Debian-based systems"
+  exit 1
 fi
 
-curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
-curl https://packages.microsoft.com/config/ubuntu/"$VERSION"/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
-## TODO: upgrade builder image
-apt-get update && \
-ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev mssql-tools && \
+apt-get -qq update && apt-get install -yqq curl gpg
+
+# shellcheck disable=SC1091
+. /etc/os-release
+
+# ubuntu
+curl -fsSL https://packages.microsoft.com/keys/microsoft.asc > /etc/apt/trusted.gpg.d/microsoft.asc
+# debian
+curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg
+curl -fsSL "https://packages.microsoft.com/config/${ID}/${VERSION_ID}/prod.list" > /etc/apt/sources.list.d/mssql-release.list
+
+apt-get -qq update
+ACCEPT_EULA=Y apt-get install -yqq msodbcsql18 unixodbc-dev
 ## and not needed to modify /etc/odbcinst.ini
 ## docker-compose will mount one in .ci/docker-compose-file/odbc
-sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini
+sed -i 's/ODBC Driver 18 for SQL Server/ms-sql/g' /etc/odbcinst.ini