airbyte-custom-chart

#airbyte #helm #oauth2-proxy
06-29-2024
Tldr

Airbyte 접근제어를 위해 oauth2-proxy 를 사용했고 chart 를 변경하여 default connector 를 제어했다. 접근제어는 이메일 또는 source/dest 커넥터를 차트에 추가하고 argo sync (auto sync 는 off 인 경우) 를 실행한다.

Why I did it

background

Airbyte

oauth2-proxy

with Oauth2

values 파일은 다음과 같이 구성하였습니다.

# airbyte root values yaml
...
oauth2-proxy:
  enabled: true
  image:
    repository: {xxxx}/bitnami/oauth2-proxy
    tag: latest

  nodeSelector:
    {key: value}

  service:
    type: NodePort
    port: 80

  configuration:
    clientID: xxx
    clientSecret: xxx
    redirectUrl: xxx

    content: |-
      skip_provider_button = true
      cookie_secure = false
      session_store_type = "redis"
      upstreams = [ "http://airbyte-airbyte-webapp-svc.data.svc.cluster.local:80" ]
      redis_connection_url = [ "redis://airbyte-redis-master.data.svc.cluster.local:6379/1" ]

    authenticatedEmailsFile:
      enabled: true
      content: |-
        jj.lee@xxx.com
        ...
        ...
        sh.lim@xxx.com
        yoori@xxx.com

default connector 제어

# airbyte-bootloader.manageCatalog
  manageCatalog:
    enabled: true
    sources:
      - "Coin API"
      - "CoinGecko Coins"
      - "CoinMarketCap"
      - "ElasticSearch"
    destinations:
      - "BigQuery"
      - "BigQuery (denormalized typed struct)"


# _helpers.tpl
{{- define "bootloader.connectionList" -}}
{{- if .Values.manageCatalog.enabled }}
    {{- $combinedSources := join "', '" .Values.manageCatalog.sources }}
    {{- $combinedDestinations := join "', '" .Values.manageCatalog.destinations }}
    {{- printf "('%s', '%s')" $combinedSources $combinedDestinations }}
{{- else}}
    {{- printf "('Coin API', 'CoinGecko Coins', 'CoinMarketCap', 'BigQuery', 'BigQuery (denormalized typed struct)')"}}
{{- end }}

modify airbyte-bootloader

# additional container
  containers:
    - name: airbyte-bootloader-clean
      image: {{ printf "%s:latest" .Values.image.clean.repository }}
      imagePullPolicy: "{{ .Values.image.pullPolicy }}"
      command:
        - "psql"
        - "-c"
        - "delete from actor_definition where name not in {{ include "bootloader.connectionList" . }};"
        - "dbname=db-airbyte user=$(DATABASE_USER) host=$(DATABASE_HOST)"
      env:
        {{- if eq .Values.global.deploymentMode "oss"  }}
        - name: DATABASE_HOST
          valueFrom:
            configMapKeyRef:
              name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }}
              key: DATABASE_HOST
        - name: PGPASSWORD
          valueFrom:
            secretKeyRef:
              name: {{ .Values.global.database.secretName | default (printf "%s-airbyte-secrets" .Release.Name ) }}
              key: {{ .Values.global.database.secretValue | default "DATABASE_PASSWORD" }}
        - name: DATABASE_USER
          valueFrom:
            secretKeyRef:
              name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }}
              key: DATABASE_USER
        {{- end }}

conlcusion