eHealthAfrica · hkmshb · Mar 6, 2025 · Mar 10, 2025 · Mar 10, 2025 · Mar 11, 2025
diff --git a/.env b/.env
@@ -0,0 +1,11 @@
+#CKAN
+CKAN_VERSION=2.9.11
+
+# Redis
+REDIS_VERSION=6.0.7
+
+# Datapusher
+DATAPUSHER_VERSION=0.0.17
+
+# Solr
+SOLR_VERSION=9
diff --git a/Makefile b/Makefile
@@ -1,15 +1,55 @@
-VERSION := $(shell git describe --tags --exact-match 2>/dev/null || echo latest)
-DOCKERHUB_NAMESPACE ?= ehealthafrica
-IMAGE := ${DOCKERHUB_NAMESPACE}/ckan:${VERSION}-alpine
-
 build:
-	docker build -t ${IMAGE} rootfs
+	@./build.sh --deb
+
+build/dry-run:
+	@./build.sh --deb --dry-run
+
+build/no-cache:
+	@./build.sh --deb --no-cache
+
+
+# ---------------------------------------------------
+# datapusher
+# ---------------------------------------------------
+
+build/datapusher:
+	@./build.sh --datapusher
+
+build/datapusher/no-cache:
+	@./build.sh --datapusher --no-cache
+
+
+# ---------------------------------------------------
+# solr
+# ---------------------------------------------------
+
+build/solr:
+	@./build.sh --solr
+
+build/solr/no-cache:
+	@./build.sh --solr --no-cache
+
+
+# ---------------------------------------------------
+# docker commands
+# ---------------------------------------------------
+
+docker/build:
+	docker compose -f ./compose/docker-compose.yml build
+
+docker/down:
+	docker compose -f ./compose/docker-compose.yml down
+
+docker/up:
+	docker compose -f ./compose/docker-compose.yml up redis solr db -d
+	docker compose -f ./compose/docker-compose.yml up datapusher ckan
 
-build-no-cache:
-	docker build --no-cache -t ${IMAGE} rootfs
+docker/prune:
+	docker compose -f ./compose/docker-compose.yml down
+	docker volume rm compose_ckan_data
+	docker volume rm compose_solr_data
+	docker volume rm compose_pg_data
 
 lint-python:
 	python -m black --skip-string-normalization -l 98 .
 
-push: build
-	docker push ${IMAGE}
diff --git a/README.md b/README.md
@@ -4,6 +4,8 @@
 
 This repository contains the necessary files for building a base docker image for CKAN. The build can target either [Alpine Linux](https://alpinelinux.org/) or [Debian Linux](https//debian.org) and only includes required extensions to start a CKAN instance. This is based-off Keitaro's [docker-ckan](https://github.com/keitaroinc/docker-ckan).
 
+Additionally, the repository now contains necessary files for building Docker images for `Datapusher` and `Solr` and a sample Docker Compose setup with all services required to run a CKAN instance running on Docker. 
+
 > **NOTE**  
 > The `master` branch tracks and absorbs changes from the upstream Keitaro repository which then goes into the `develop` branch. The `develop` contains our modifications to the original work done by Keitaro mostly to include support for building a Debian Linux based CKAN image. The Debian build is modelled after the original Alpine build, to ensure the same build and runtime mechanisms apply across the targetted distros.
 >
@@ -20,6 +22,8 @@ Usage:
   ./build.sh [options]
 
 Options:
+  --datapusher            build image for datapusher
+  --solr                  build image for solr
   --deb         | -d      build Debain image.
   --dry-run               performs a dry-run to show configs.
   --help        | -h      show this message.
@@ -28,7 +32,9 @@ Options:
   --tag         | -t      the image tag.
 ```
 
-The script is configured to build an `Alpine` image by default (except when `--deb` flag is provided). The built image is named in the form `<namespace>/ckan` and tagged with the latest tag for the repository or "latest" if there is none (or uses value from `--tag <ckan-version>` option if provided). The resulting full image name becomes `<namespace>/ckan:<tag>-alpine` for Alpine or just `<namespace>/ckan:<tag>` for a Debian image. For instance: `ehealthafrica/ckan:2.7.8-alpine`.
+The script is configured to build an `Alpine` image by default (except when `--deb` flag is provided). The built image is named in the form `<namespace>/ckan` for CKAN and `<namespace>/ckan-<service>` for Solr and Datapuhser. The built image is tagged with the version set in `.env` for the service or "latest" if there is none. 
+
+The resulting full image name becomes `<namespace>/ckan(-service):<tag>-alpine` for Alpine or just `<namespace>/ckan(-service):<tag>` for a Debian image. For instance: `ehealthafrica/ckan:2.7.8-alpine` and `ehealthafrica/ckan-solr:9-alpine`.
 
 ## List
 
@@ -43,10 +49,10 @@ Check if the image shows up in the list of images:
 To start and test newly created image run:
 
 ```sh
- docker run <namespace>/ckan:<image-tag>
+ docker run --rm --name <image-name> <namespace>/ckan:<image-tag>
 ```
 
-Check if CKAN was succesfuly started on <http://localhost:5000>. The ckan site url is configured in ENV CKAN_SITE_URL.
+Check if CKAN was succesfuly started on <http://localhost:5000>. The ckan site url is configured in `ENV CKAN_SITE_URL`.
 
 ## Upload to DockerHub
 
@@ -60,19 +66,22 @@ docker push [options] <docker-hub>/ckan:<image-tag>
 
 ## Upgrade
 
-To upgrade the Docker files to use new CKAN version, in the Dockerfiles you should change:
+To upgrade the Docker files to use a new CKAN version, in the Dockerfiles you should change:
 
 > ENV GIT_BRANCH={ckan_release}
 
+and also update the `CKAN_VERSION` entry in the `.env` file.
+
 Check [CKAN repository](https://github.com/ckan/ckan/releases) for the latest releases. 
 If there are new libraries used by the new version requirements, those needs to be included too.
 
 ## Extensions
 
 Default extensions used in the Dockerfile  are kept in:
 
-> ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher
+> ENV CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher"
 
 ## Add new scripts
 
 You can add scripts to CKAN custom images and c
F438
opy them to the *docker-entrypoint.d* directory. Any *.sh or *.py file in that directory will be executed after the main initialization script (prerun.py) is executed.
+
diff --git a/build.sh b/build.sh
@@ -4,13 +4,11 @@ set -Eeuo pipefail
 
 deb=no
 dry_run=no
-build_ckan=yes
 nocache=no
-target_image=
+target_image=ckan
 target_image_fullname=
-namespace=${DOCKERHUB_NAMESPACE:=ehealthafrica}
-version=$(git describe --tags --exact-match 2>/dev/null || echo latest)
-
+namespace=${DOCKERHUB_NAMESPACE:-ehealthafrica}
+tag=$(git describe --tags --exact-match 2>/dev/null || echo ${CKAN_VERSION})
 
 function show_help {
   echo """
@@ -20,27 +18,32 @@ function show_help {
     ./build.sh [options]
 
   Options:
-    --build-datapusher      build image for datapusher (default: build ckan image)
-    --deb         | -d      build Debain image.
-    --dry-run               performs a dry-run to show configs.
-    --help        | -h      show this message.
-    --namespace             docker hub account name.
-    --no-cache              build image without using cache.
-    --tag         | -t      the image tag.
+    --datapusher        build image for datapusher
+    --solr              build image for solr
+    --deb         | -d  build Debain image.
+    --dry-run           performs a dry-run to show configs.
+    --help        | -h  show this message.
+    --namespace         docker hub account name.
+    --no-cache          build image without using cache.
+    --tag         | -t  the image tag.
   """
 }
 
 function _build_image_name {
   local deb=$1
-  local build_ckan=$2
+  local target_name=$2
   local namespace=$3
-  local tag=$4
 
-  target_image=ckan
+  tag=$4
   target_image_fullname="${namespace}/${target_image}:${tag}"
 
-  if [[ ${build_ckan} = "no" ]]; then
-    target_image=datapusher
+  if [[ ${target_name} = "datapusher" ]]; then
+    tag=$(git describe --tags --exact-match 2>/dev/null || echo ${DATAPUSHER_VERSION:-latest})
+    target_image_fullname="${namespace}/ckan-${target_image}:${tag}"
+  fi
+
+  if [[ ${target_name} = "solr" ]]; then
+    tag=$(git describe --tags --exact-match 2>/dev/null || echo ${SOLR_VERSION:-latest})
     target_image_fullname="${namespace}/ckan-${target_image}:${tag}"
   fi
 
@@ -63,17 +66,27 @@ function build {
   echo ">> Building CKAN Image '${image_fullname}' (using ${filename}) ..."
   echo ""
 
+  local image_dir=images/${image}
+  if [[ ${target_image} = "solr" ]]; then
+    image_dir=compose/${image}
+  fi
+
   if [[ ${nocache} = "yes" ]]; then
-    docker build -f rootfs/${image}/${filename} -t ${image_fullname} rootfs/${image}
+    docker build 
10000
-f ${image_dir}/${filename} -t ${image_fullname} ${image_dir}
   else
-    docker build --no-cache -f rootfs/${image}/${filename} -t ${image_fullname} rootfs/${image}
+    docker build --no-cache -f ${image_dir}/${filename} -t ${image_fullname} ${image_dir}
   fi
 }
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
-    --build-datapusher )
-      build_ckan=no
+    --datapusher )
+      target_image="datapusher"
+      shift
+    ;;
+
+    --solr )
+      target_image="solr"
       shift
     ;;
 
@@ -105,17 +118,17 @@ while [[ $# -gt 0 ]]; do
 
     -t | --tag )
       shift
-      version=$1
+      tag=$1
       shift
     ;;
   esac
 done
 
 # build image name which is stored in global var: result
-_build_image_name ${deb} ${build_ckan} ${namespace} ${version}
+_build_image_name ${deb} ${target_image} ${namespace} ${tag}
 
 if [[ $dry_run = "yes" ]]; then
-  echo "deb=${deb} build-ckan=${build_ckan} namespace=${namespace} tag=${version}"
+  echo "deb=${deb} target-image=${target_image} namespace=${namespace} tag=${tag}"
   echo ${target_image_fullname}
 else
   build ${deb} ${target_image} ${target_image_fullname} ${nocache}

diff --git a/compose/config/.ckan.env b/compose/config/.ckan.env
@@ -0,0 +1,49 @@
+# Runtime configuration of CKAN enabled through ckanext-envvars
+# Information about how it works: https://github.com/okfn/ckanext-envvars
+# Note that variables here take presedence over build/up time variables in .env
+
+#Compose variables
+# CKAN
+CKAN_MAX_UPLOAD_SIZE_MB=10
+CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD:-ckan}@db/ckan
+CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD:-ckan}@db/datastore
+CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD:-datastore}@db/datastore
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+CKAN_REDIS_URL=redis://redis:6379/1
+
+
+# Set to true to disable CKAN from starting and serve a maintenance page
+MAINTENANCE_MODE=false
+
+# General Settings
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5055
+CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
+# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
+CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
+CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
+# CKAN Plugins
+CKAN__PLUGINS="envvars image_view text_view recline_view datastore datapusher"
+# CKAN requires storage path to be set in order for filestore to be enabled
+CKAN__STORAGE_PATH=/srv/app/data
+CKAN__WEBASSETS__PATH=/srv/app/data/webassets
+# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
+CKAN_SYSADMIN_NAME=sysadmin
+CKAN_SYSADMIN_PASSWORD=password
+CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
+
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts
+
+PASSWORD_PROTECT=false
+
diff --git a/compose/config/.datapusher.env b/compose/config/.datapusher.env
@@ -0,0 +1,8 @@
+DATAPUSHER_MAX_CONTENT_LENGTH=10485760
+DATAPUSHER_CHUNK_SIZE=16384
+DATAPUSHER_CHUNK_INSERT_ROWS=250
+DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5005
+DATAPUSHER_SSL_VERIFY=False
diff --git a/compose/config/.postgres.env b/compose/config/.postgres.env
@@ -0,0 +1,5 @@
+# Database
+POSTGRES_PASSWORD=ckan
+POSTGRES_PORT=5432
+DATASTORE_READONLY_PASSWORD=datastore
+PGDATA=/var/lib/postgresql/data/db
diff --git a/compose/config/.solr.env b/compose/config/.solr.env
@@ -0,0 +1 @@
+CKAN_CORE_NAME=ckan
diff --git a/compose/config/backend-services.yml b/compose/config/backend-services.yml
@@ -0,0 +1,54 @@
+services:
+  db:
+    container_name: db
+    build:
+      context: postgresql
+    networks:
+      - backend
+    env_file:
+      - config/.postgres.env
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "ckan"]
+
+  solr:
+    container_name: solr
+    image: ehealthafrica/ckan-solr:${SOLR_VERSION}-alpine
+    build:
+      context: solr
+      args:
+        CKAN_BRANCH: ckan-${CKAN_VERSION}
+        SOLR_VERSION: ${SOLR_VERSION}
+    networks:
+      - backend
+    env_file:
+      - ${PWD}/.env
+      - config/.solr.env
+    ports:
+      - 8983:8983
+    volumes:
+      - solr_data:/var/solr
+    healthcheck:
+      test: ["CMD-SHELL", "/opt/solr/bin/solr","status"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+      start_period: 5s
+
+  redis:
+    container_name: redis
+    image: redis:${REDIS_VERSION}
+    networks:
+      - backend
+    healthcheck:
+      test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+
+volumes:
+  pg_data:
+  solr_data:
+