From 08b4c96ca694daf0329163c4faab98907812e1e3 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Mon, 6 Mar 2023 18:53:18 +0200 Subject: [PATCH 1/4] Add a mapping between Fybrik and airbyte-module versions (#2015) * Add a mapping between Fybrik and airbyte-module versions (#2009) --------- Signed-off-by: Revital Sur --- RELEASE.md | 11 ++++++----- site/airByteReleases.yaml | 1 + site/docs/get-started/quickstart.md | 2 +- site/docs/samples/chaining-sample.md | 16 ++++++++++++---- site/main.py | 10 +++++----- site/mkdocs.yml | 1 + 6 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 site/airByteReleases.yaml diff --git a/RELEASE.md b/RELEASE.md index df32f73ab..b08b387d0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,13 +2,14 @@ The process of creating a release is described in this document. Replace `X.Y.Z` with the version to be released. -## 1. Optionally update the ArrowFlight releases file +## 1. Optionally update the ArrowFlight and AirByte releases files The file `./site/arrowFlightReleases` contains a map between fybrik and the arrow-flight-module releases. -The quick start uses this file to map to a relevant arrow-flight release. +The file `./site/airByteReleases` contains a map between fybrik and the airbyte releases. +The quick start and samples use these files to map to a relevant arrow-flight/airbyte releases. * If there is a perfect march for the fybrik release is defined, its value will be used. * Otherwise, the major and minor fybrik release will be checked. -* If there is no matches, the `latest` arrow-flight release will be used. +* If there is no matches, the `latest` arrow-flight/airbyte release will be used. For example, if the file content is: @@ -17,8 +18,8 @@ v0.5: v0.5.0 v0.5.3: v0.5.3 v0.6: v0.6.0 ``` -The v0.5.3 fybrik release will use the v0.5.3 release of arrow-flight. -The v0.6.1 fybrik release will be mapped to the v0.6.0 of arrow-flight. +The v0.5.3 fybrik release will use the v0.5.3 release of arrow-flight/airbyte. +The v0.6.1 fybrik release will be mapped to the v0.6.0 of arrow-flight/airbyte. ## 2. Create a `releases/X.Y.Z` branch diff --git a/site/airByteReleases.yaml b/site/airByteReleases.yaml new file mode 100644 index 000000000..11e4863d2 --- /dev/null +++ b/site/airByteReleases.yaml @@ -0,0 +1 @@ +v1.3: v0.3.0 diff --git a/site/docs/get-started/quickstart.md b/site/docs/get-started/quickstart.md index 89f405772..0981dbf7e 100644 --- a/site/docs/get-started/quickstart.md +++ b/site/docs/get-started/quickstart.md @@ -1,5 +1,5 @@ # FybrikModule chaining sample This sample shows how to implement a use case where, based on the data source and governance policies, the Fybrik manager determines that it must deploy two FybrikModules to allow a workload access to a dataset. One FybrikModule handles reading the data and the second does the data transformation. Data is passed between the FybrikModules without writing to intermediate storage. @@ -15,13 +17,19 @@ To recreate this scenario, you will need a copy of the airbyte-module repository cd /tmp git clone https://github.com/fybrik/airbyte-module.git cd airbyte-module - git checkout v0.3.0 + {% if airByteRelease != 'latest' %} + git checkout {{ airByteRelease }} + {% endif %} export AIRBYTE_MODULE_DIR=${PWD} ``` 1. Install the Airbyte module: ```bash - kubectl apply -f https://github.com/fybrik/airbyte-module/releases/download/v0.3.0/module.yaml -n fybrik-system + {% if airByteRelease != 'latest' %} + kubectl apply -f https://github.com/fybrik/airbyte-module/releases/download/{{ airByteRelease }}/module.yaml -n fybrik-system + {% else %} + kubectl apply -f https://github.com/fybrik/airbyte-module/releases/{{ airByteRelease }}/download/module.yaml -n fybrik-system + {% endif %} ``` 1. Install the arrow-flight module for transformations: diff --git a/site/main.py b/site/main.py index 6ae43c53e..38981c9a2 100644 --- a/site/main.py +++ b/site/main.py @@ -42,10 +42,10 @@ def fybrik_version_comment_end(version, inverse): return "" @env.macro - def arrow_flight_module_version(version, arrow_flight_version): - if version in arrow_flight_version: - return arrow_flight_version[version] + def get_module_version(version, module_version): + if version in module_version: + return module_version[version] major_version = version[:4] - if major_version in arrow_flight_version: - return arrow_flight_version[major_version] + if major_version in module_version: + return module_version[major_version] return "latest" diff --git a/site/mkdocs.yml b/site/mkdocs.yml index 1a861558c..63a675f2a 100644 --- a/site/mkdocs.yml +++ b/site/mkdocs.yml @@ -113,3 +113,4 @@ plugins: include_yaml: - ./external.yaml - arrowFlight: "./arrowFlightReleases.yaml" + - airByte: "./airByteReleases.yaml" From 16770001ba1281b7aa723daf8affda8636d72bc7 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Mon, 6 Mar 2023 19:13:26 +0200 Subject: [PATCH 2/4] Add instructions for deployment of localstack on openshift in samples. (#2016) Signed-off-by: Revital Sur --- site/docs/samples/delete-sample.md | 25 +++++++++++++++++-------- site/docs/samples/notebook-read.md | 25 +++++++++++++++++-------- site/docs/samples/notebook-write.md | 25 +++++++++++++++++-------- 3 files changed, 51 insertions(+), 24 deletions(-) diff --git a/site/docs/samples/delete-sample.md b/site/docs/samples/delete-sample.md index e020bcb3d..4503037be 100644 --- a/site/docs/samples/delete-sample.md +++ b/site/docs/samples/delete-sample.md @@ -28,14 +28,23 @@ Make a note of the service endpoint, bucket name, and access credentials. You wi export SECRET_KEY="mysecretkey" ``` 2. Install localstack to the currently active namespace and wait for it to be ready: - ```bash - helm repo add localstack-charts https://localstack.github.io/helm-charts - helm install localstack localstack-charts/localstack \ - --set startServices="s3" \ - --set service.type=ClusterIP \ - --set livenessProbe.initialDelaySeconds=25 - kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s - ``` + === "Kubernetes" + helm repo add localstack-charts https://localstack.github.io/helm-charts + helm install localstack localstack-charts/localstack \ + --set startServices="s3" \ + --set service.type=ClusterIP \ + --set livenessProbe.initialDelaySeconds=25 + kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s + === "OpenShift" + helm repo add localstack-charts https://localstack.github.io/helm-charts + helm install localstack localstack-charts/localstack \ + --set startServices="s3" \ + --set service.type=ClusterIP \ + --set livenessProbe.initialDelaySeconds=25 \ + --set persistence.enabled=true \ + --set persistence.storageClass=ibmc-file-gold-gid \ + --set persistence.accessModes[0]=ReadWriteMany + kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s 3. Create a port-forward to communicate with localstack server: ```bash kubectl port-forward svc/localstack 4566:4566 & diff --git a/site/docs/samples/notebook-read.md b/site/docs/samples/notebook-read.md index 84e9fc170..97098aa9b 100644 --- a/site/docs/samples/notebook-read.md +++ b/site/docs/samples/notebook-read.md @@ -33,14 +33,23 @@ Make a note of the service endpoint, bucket name, and access credentials. You wi export SECRET_KEY="mysecretkey" ``` 2. Install localstack to the currently active namespace and wait for it to be ready: - ```bash - helm repo add localstack-charts https://localstack.github.io/helm-charts - helm install localstack localstack-charts/localstack \ - --set startServices="s3" \ - --set service.type=ClusterIP \ - --set livenessProbe.initialDelaySeconds=25 - kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s - ``` + === "Kubernetes" + helm repo add localstack-charts https://localstack.github.io/helm-charts + helm install localstack localstack-charts/localstack \ + --set startServices="s3" \ + --set service.type=ClusterIP \ + --set livenessProbe.initialDelaySeconds=25 + kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s + === "OpenShift" + helm repo add localstack-charts https://localstack.github.io/helm-charts + helm install localstack localstack-charts/localstack \ + --set startServices="s3" \ + --set service.type=ClusterIP \ + --set livenessProbe.initialDelaySeconds=25 \ + --set persistence.enabled=true \ + --set persistence.storageClass=ibmc-file-gold-gid \ + --set persistence.accessModes[0]=ReadWriteMany + kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s 3. Create a port-forward to communicate with localstack server: ```bash kubectl port-forward svc/localstack 4566:4566 & diff --git a/site/docs/samples/notebook-write.md b/site/docs/samples/notebook-write.md index 267c8d1d1..8d61f07d1 100644 --- a/site/docs/samples/notebook-write.md +++ b/site/docs/samples/notebook-write.md @@ -30,14 +30,23 @@ Make a note of the service endpoint and access credentials. You will need them l export SECRET_KEY="mysecretkey" ``` 2. Install localstack to the currently active namespace and wait for it to be ready: - ```bash - helm repo add localstack-charts https://localstack.github.io/helm-charts - helm install localstack localstack-charts/localstack \ - --set startServices="s3" \ - --set service.type=ClusterIP \ - --set livenessProbe.initialDelaySeconds=25 - kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s - ``` + === "Kubernetes" + helm repo add localstack-charts https://localstack.github.io/helm-charts + helm install localstack localstack-charts/localstack \ + --set startServices="s3" \ + --set service.type=ClusterIP \ + --set livenessProbe.initialDelaySeconds=25 + kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s + === "OpenShift" + helm repo add localstack-charts https://localstack.github.io/helm-charts + helm install localstack localstack-charts/localstack \ + --set startServices="s3" \ + --set service.type=ClusterIP \ + --set livenessProbe.initialDelaySeconds=25 \ + --set persistence.enabled=true \ + --set persistence.storageClass=ibmc-file-gold-gid \ + --set persistence.accessModes[0]=ReadWriteMany + kubectl wait --for=condition=ready --all pod -n fybrik-notebook-sample --timeout=120s create a port-forward to communicate with localstack server: ```bash kubectl port-forward svc/localstack 4566:4566 & From 056803946665744f78ec8e65a19c6b4edd0c01b3 Mon Sep 17 00:00:00 2001 From: Shlomit Koyfman Date: Thu, 9 Mar 2023 12:00:29 +0200 Subject: [PATCH 3/4] avoid resetting annotations (#2019) Signed-off-by: Shlomit Koyfman --- manager/controllers/app/plotter_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/manager/controllers/app/plotter_controller.go b/manager/controllers/app/plotter_controller.go index c95c83956..e74e203df 100644 --- a/manager/controllers/app/plotter_controller.go +++ b/manager/controllers/app/plotter_controller.go @@ -354,7 +354,6 @@ func (r *PlotterReconciler) reconcile(plotter *fapp.Plotter) (ctrl.Result, []err if plotter.Generation != plotter.Status.ObservedGeneration { log.Trace().Str(logging.ACTION, logging.UPDATE).Msg("Updating blueprint...") remoteBlueprint.Spec = blueprintSpec - remoteBlueprint.ObjectMeta.Annotations = map[string]string(nil) // reset annotations err := r.ClusterManager.UpdateBlueprint(cluster, remoteBlueprint) if err != nil { log.Error().Err(err).Msg("Could not update blueprint") From 1c60343c0f8be61c4cfcfa8e2d933b051e3052e0 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Mar 2023 11:04:36 +0000 Subject: [PATCH 4/4] Commit from GitHub Actions (Release) --- charts/fybrik-crd/Chart.yaml | 7 +++---- charts/fybrik-crd/charts/asset-crd/Chart.yaml | 8 ++------ charts/fybrik/Chart.yaml | 4 ++-- charts/vault/Chart.yaml | 8 ++++---- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/charts/fybrik-crd/Chart.yaml b/charts/fybrik-crd/Chart.yaml index 7c4cc3a38..11fe49f6e 100644 --- a/charts/fybrik-crd/Chart.yaml +++ b/charts/fybrik-crd/Chart.yaml @@ -8,12 +8,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.0 +version: 1.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.0.0 - +appVersion: 1.3.1 dependencies: - name: asset-crd - version: 0.0.0 + version: 1.3.1 condition: asset-crd.enabled diff --git a/charts/fybrik-crd/charts/asset-crd/Chart.yaml b/charts/fybrik-crd/charts/asset-crd/Chart.yaml index ca9cc26df..14ace6983 100644 --- a/charts/fybrik-crd/charts/asset-crd/Chart.yaml +++ b/charts/fybrik-crd/charts/asset-crd/Chart.yaml @@ -4,7 +4,6 @@ apiVersion: v2 name: asset-crd description: Asset CRD, used by the Katalog catalog - # A chart can be either an 'application' or a 'library' chart. # # Application charts are a collection of templates that can be packaged into versioned archives @@ -14,14 +13,11 @@ description: Asset CRD, used by the Katalog catalog # a dependency of application charts to inject those utilities and functions into the rendering # pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.0 - +version: 1.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 0.0.0 - +appVersion: 1.3.1 diff --git a/charts/fybrik/Chart.yaml b/charts/fybrik/Chart.yaml index 8bfa5d9e7..10b08d5d7 100644 --- a/charts/fybrik/Chart.yaml +++ b/charts/fybrik/Chart.yaml @@ -8,7 +8,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.0 +version: 1.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.0.0 +appVersion: 1.3.1 diff --git a/charts/vault/Chart.yaml b/charts/vault/Chart.yaml index 04c9e1ed9..05fbc5411 100644 --- a/charts/vault/Chart.yaml +++ b/charts/vault/Chart.yaml @@ -8,11 +8,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.0 +version: 1.3.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 1.12.2 dependencies: -- name: vault - version: "0.23.0" - repository: "https://helm.releases.hashicorp.com" + - name: vault + version: "0.23.0" + repository: "https://helm.releases.hashicorp.com"