Merge pull request #54 from Opetushallitus/OK-504__dbt-automatisointi

Ok 504  dbt automatisointi

.github/workflows/build-dbt-runner.yml

@@ -0,0 +1,20 @@
+name: Build DBT Runner Container
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'dbt/**'
+      - 'dbt-container/**'
+
+jobs:
+  build-and-deploy-container:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build container
+        run: dbt-container/build.sh
+
+

cdk/lambda/lampi/LampiFileChangedReceiver.ts

@@ -1,20 +1,51 @@
+/* eslint @typescript-eslint/no-var-requires: "off" */
 import { SendMessageCommand, SQSClient } from '@aws-sdk/client-sqs';
 import { APIGatewayProxyEventV2 } from 'aws-lambda';
 import { Context } from 'aws-lambda/handler';
 
 import { LampiEvent, LampiS3Event, lampiKeyExists } from './common';
 
+const { SSMClient, GetParameterCommand } = require('@aws-sdk/client-ssm');
+
 exports.handler = async (event: APIGatewayProxyEventV2, context: Context) => {
+  const awsRegion = process.env.AWS_REGION;
+  console.log(`AWS Region: ${awsRegion}`);
+
+  const lampiAuthTokenSecretName = process.env.lampiAuthTokenSecretName;
+  console.log(`lampiAuthTokenSecretName: ${lampiAuthTokenSecretName}`);
+
+  const parameterCommand = new GetParameterCommand({
+    Name: lampiAuthTokenSecretName,
+    WithDecryption: true,
+  });
+
+  const ssmClient = new SSMClient({ region: awsRegion });
+  const ssmResponse = await ssmClient.send(parameterCommand);
+
+  const lampiAuthToken = ssmResponse.Parameter.Value;
+  console.log(`lampiAuthToken: ${lampiAuthToken}`);
+
   console.log(JSON.stringify(event, null, 4));
+
   if (!event?.body) {
     console.error('Viestissä ei ollut bodya tai viesti oli tyhjä');
     return {
       statusCode: 500,
     };
   }
+
   const lampiEvent: LampiEvent = JSON.parse(event.body);
+
+  if (lampiEvent.token !== lampiAuthToken) {
+    console.error('Autentikaatio epäonnistui!');
+    return {
+      statusCode: 401,
+    };
+  }
+
   const lampiS3Event: LampiS3Event = lampiEvent.s3;
   const lampiKey = lampiS3Event.object.key;
+
   if (lampiKeyExists(lampiKey)) {
     console.log(
       `Uusi tunnistettu tiedosto saapunut Lampeen (${lampiKey}). Lähetetään tiedosto ladattavaksi.`
@@ -31,6 +62,7 @@ exports.handler = async (event: APIGatewayProxyEventV2, context: Context) => {
   } else {
     console.log(`Tuntematon tiedosto: ${lampiKey}`);
   }
+
   return {
     statusCode: 200,
   };

cdk/lib/lambda-stack.ts

@@ -434,6 +434,8 @@ export class LambdaStack extends cdk.Stack {
       }
     );
 
+    const lampiAuthTokenSecretName = `/${config.environment}/lambda/lampi-auth-token`;
+
     const lampiTiedostoMuuttunutLambda = new lambdaNodejs.NodejsFunction(
       this,
       lampiTiedostoMuuttunutLambdaName,
@@ -451,6 +453,7 @@ export class LambdaStack extends cdk.Stack {
         environment: {
           environment: config.environment,
           lampiSiirtotiedostoQueueUrl: lampiSiirtotiedostoQueue.queueUrl,
+          lampiAuthTokenSecretName: lampiAuthTokenSecretName,
         },
         bundling: {
           commandHooks: {
@@ -470,6 +473,23 @@ export class LambdaStack extends cdk.Stack {
       })
     );
 
+    /*
+    const lampiAuthTokenParam = ssm.StringParameter.fromStringParameterName(
+      this,
+      'LampiAuthTokenParam',
+      lampiAuthTokenSecretName,
+    );
+    */
+
+    const lampiAuthTokenParam = ssm.StringParameter.fromSecureStringParameterAttributes(
+      this,
+      'LampiAuthTokenParam',
+      {
+        parameterName: lampiAuthTokenSecretName,
+      }
+    );
+    lampiAuthTokenParam.grantRead(lampiTiedostoMuuttunutLambda);
+
     const lampiTiedostoMuuttunutLambdaUrl = lampiTiedostoMuuttunutLambda.addFunctionUrl({
       authType: lambda.FunctionUrlAuthType.NONE,
     });

dbt-container/Dockerfile

@@ -0,0 +1,12 @@
+FROM alpine:3.20
+RUN apk upgrade --no-cache
+RUN apk --no-cache add bash
+
+WORKDIR /root/
+COPY ./run.sh ./
+COPY ./install.sh ./
+RUN \
+  bash install.sh && \
+  rm install.sh
+
+ENTRYPOINT ["bash", "/root/run.sh"]

dbt-container/README.md

@@ -0,0 +1,4 @@
+## Kontin käynnistäminen
+```
+docker run --rm --name dbt-runner ovara-dbt-runner
+```

dbt-container/build.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cd "${0%/*}"
+docker build --progress=plain -t ovara-dbt-runner .
+cd -

dbt-container/install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+
+case "$(uname -m)" in
+  aarch64) ARCHITECTURE="arm64" ;;
+  x86_64) ARCHITECTURE="amd64" ;;
+  *) ARCHITECTURE=$(uname -m) ;;
+esac
+echo $ARCHITECTURE
+
+echo "Installing needed software"
+apk --no-cache add \
+  python3
+
+ln -sf /usr/bin/python3 /usr/bin/python
+
+ls -Al /root
+cat /root/run.sh

dbt-container/run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+echo "Running Ovara DBT script..."
+
+exit 0