diff --git a/.dockerignore b/.dockerignore index e71275e2c..6eae04102 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,5 +16,4 @@ docs/ certificates/ server/ .github/ -./pathservice/taxonomy/ node_modules/ diff --git a/.env.github.example b/.env.github.example index 29992e869..5b2aec463 100644 --- a/.env.github.example +++ b/.env.github.example @@ -1,5 +1,3 @@ -IL_UI_ADMIN_USERNAME=admin # user/pass for dev mode -IL_UI_ADMIN_PASSWORD=password IL_UI_DEPLOYMENT=github # Start UI stack in github mode. OAUTH_GITHUB_ID= OAUTH_GITHUB_SECRET= @@ -12,14 +10,14 @@ NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO=github.com/instructlab-public/taxonomy-knowl NEXT_PUBLIC_AUTHENTICATION_ORG= NEXT_PUBLIC_TAXONOMY_REPO_OWNER= NEXT_PUBLIC_TAXONOMY_REPO= - -IL_GRANITE_API= -IL_GRANITE_MODEL_NAME= -IL_MERLINITE_API= -IL_MERLINITE_MODEL_NAME= - -IL_ENABLE_DEV_MODE=true #Enable this option if you want to enable UI features that helps in development, such as form Auto-Fill feature. - NEXT_PUBLIC_EXPERIMENTAL_FEATURES=false SLACK_WEBHOOK_URL= + +# (Optional) Enable this option if you want to enable UI features that helps in development, such as form Auto-Fill feature. +# Default: false +IL_ENABLE_DEV_MODE=false + +# (Optional) Enable document conversion. Any non-markdown file will be converted to markdown file +# Default: false +IL_ENABLE_DOC_CONVERSION=false diff --git a/.env.native.example b/.env.native.example index ec086ddd6..b415fdd70 100644 --- a/.env.native.example +++ b/.env.native.example @@ -1,16 +1,29 @@ +IL_UI_DEPLOYMENT=native IL_UI_ADMIN_USERNAME=admin IL_UI_ADMIN_PASSWORD=password NEXTAUTH_SECRET=your_super_secret_random_string NEXTAUTH_URL=http://localhost:3000 -IL_UI_DEPLOYMENT=native # Two deployment modes are available: github and native -IL_ENABLE_DEV_MODE=true #Enable this option if you want to enable UI features that helps in development, such as form Auto-Fill feature. +# (Optional) Enable this option if you want to enable UI features that helps in development, such as form Auto-Fill feature. +# Default: false +IL_ENABLE_DEV_MODE=false +# (Optional) Enable document conversion. Any non-markdown file will be converted to markdown file +# Default: false +IL_ENABLE_DOC_CONVERSION=false + +# (Optional) Document conversion requires docling service to convert the documents. +# Uncomment and fill in the http://host:port where docling service is running. +# By default it assuming docling service is running on local host listening on port 5001 +# IL_FILE_CONVERSION_SERVICE=http://localhost:5001 + +# (Required) Set to the parent directory where taxonomy repo is cloned. NEXT_PUBLIC_TAXONOMY_ROOT_DIR= +# (Optional)Enable experiment features like synthetic data generation, training, and chat evaluation. NEXT_PUBLIC_EXPERIMENTAL_FEATURES=false -# IL_FILE_CONVERSION_SERVICE=http://localhost:8000 # Uncomment and fill in the http://host:port if the docling conversion service is running. -# NEXT_PUBLIC_API_SERVER=http://localhost:8080 # Uncomment and point to the URL the api-server is running on. Native mode only and needs to be running on the same host as the UI. -# NEXT_PUBLIC_MODEL_SERVER_URL=http://x.x.x.x # Used for model chat evaluation vLLM instances. Currently, server side rendering is not supported so the client must have access to this address for model chat evaluation to function in the UI. Currently ports, 8000 & 8001 are hardcoded and why it is not an option to set. +# (Optional) Uncomment and point to the URL the api-server is running on. Native mode only and needs +# to be running on the same host as the UI. +# NEXT_PUBLIC_API_SERVER=http://localhost:8080 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1476f8753..f0053d486 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -54,15 +54,6 @@ updates: actions-deps: patterns: - "*" - - package-ecosystem: "gomod" - directory: "/pathservice" - schedule: - interval: "weekly" - reviewers: - - "instructlab/ui-maintainers" - target-branch: "main" - labels: - - "go dependencies" - package-ecosystem: "docker" directory: "/" schedule: @@ -127,15 +118,6 @@ updates: actions-deps: patterns: - "*" - - package-ecosystem: "gomod" - directory: "/pathservice" - schedule: - interval: "weekly" - reviewers: - - "instructlab/ui-maintainers" - target-branch: "release-1.0" - labels: - - "go dependencies" - package-ecosystem: "docker" directory: "/" schedule: diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 3c349d424..5fe14b923 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest steps: - name: "Harden Runner" - uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 + uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs diff --git a/.github/workflows/lint-ui.yml b/.github/workflows/lint-ui.yml index 244cb5ea3..c94a2e1d0 100644 --- a/.github/workflows/lint-ui.yml +++ b/.github/workflows/lint-ui.yml @@ -39,6 +39,7 @@ jobs: - name: Check for Formatting Changes run: | if [ -n "$(git status --porcelain)" ]; then + git status --porcelain echo "❌ Code formatting issues detected. Please run 'npm run pretty' and amend the commit." exit 1 else diff --git a/.github/workflows/pr-images.yml b/.github/workflows/pr-images.yml index 974715574..5760d8b93 100644 --- a/.github/workflows/pr-images.yml +++ b/.github/workflows/pr-images.yml @@ -10,8 +10,6 @@ env: GHCR_UI_IMAGE_NAME: "${{ github.repository }}/ui" QUAY_REGISTRY: quay.io QUAY_UI_IMAGE_NAME: instructlab-ui/ui - GHCR_PS_IMAGE_NAME: "${{ github.repository }}/pathservice" - QUAY_PS_IMAGE_NAME: instructlab-ui/pathservice jobs: build_and_publish_ui_qa_image: @@ -187,178 +185,3 @@ jobs: git add deploy/k8s/overlays/openshift/qa/kustomization.yaml git commit -m "[CI AUTOMATION]: Bumping QA UI image to tag: pr-${{ steps.get_pr_number.outputs.result }}" -s git push origin main - - build_and_publish_ps_qa_image: - name: Push QA pathservice container image to GHCR and QUAY - runs-on: ubuntu-latest - environment: registry-creds - permissions: - packages: write - contents: write - attestations: write - id-token: write - - steps: - - name: Check out the repo - uses: actions/checkout@v4 - with: - token: ${{ secrets.BOT_PAT }} - ref: 'main' - - - name: Skip if triggered by GitHub Actions bot - id: check_skip - run: |- - if [[ "$(git log -1 --pretty=format:'%s')" == *"[CI AUTOMATION]:"* ]]; then - echo "Workflow triggered by previous action commit. Skipping." - echo "SKIP_WORKFLOW=true" >> "$GITHUB_ENV" - else - echo "SKIP_WORKFLOW=false" >> "$GITHUB_ENV" - fi - - - name: Log in to the GHCR container image registry - if: env.SKIP_WORKFLOW == 'false' - uses: docker/login-action@v3 - with: - registry: ${{ env.GHCR_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to the Quay container image registry - if: env.SKIP_WORKFLOW == 'false' - uses: docker/login-action@v3 - with: - registry: ${{ env.QUAY_REGISTRY }} - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_TOKEN }} - - - name: Set up Docker Buildx - if: env.SKIP_WORKFLOW == 'false' - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers - if: env.SKIP_WORKFLOW == 'false' - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Get Pull Request Number from Commit - if: env.SKIP_WORKFLOW == 'false' - id: get_pr_number - uses: actions/github-script@v7 - with: - script: | - console.log("Repository owner:", context.repo.owner); - console.log("Repository name:", context.repo.repo); - console.log("Current commit SHA:", context.sha); - - const prs = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'closed', - sort: 'updated', - direction: 'desc' - }); - console.log("Number of closed PRs fetched:", prs.data.length); - - for (const pr of prs.data) { - console.log("Checking PR #", pr.number, "- Merged:"); - if (pr.merged_at != "") { - console.log("Found merged PR:", pr.number); - return pr.number; - } - } - - console.log("No merged PR found in the recent closed PRs."); - return ''; - - - name: Extract metadata (tags, labels) for pathservice image - if: env.SKIP_WORKFLOW == 'false' - id: ghcr_ps_meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_PS_IMAGE_NAME }} - - - name: Extract metadata (tags, labels) for pathservice image - if: env.SKIP_WORKFLOW == 'false' - id: quay_ps_meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_PS_IMAGE_NAME }} - - - name: Build and push QA PS image to ghcr.io - if: env.SKIP_WORKFLOW == 'false' - id: push-ps-ghcr - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: | - "${{ steps.ghcr_ps_meta.outputs.tags }}" - "${{ env.GHCR_REGISTRY }}/${{ env.GHCR_PS_IMAGE_NAME }}:pr-${{ steps.get_pr_number.outputs.result }}" - labels: ${{ steps.ghcr_ps_meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - file: pathservice/Containerfile - - - name: Generate QA PS GHCR artifact attestation - if: env.SKIP_WORKFLOW == 'false' - uses: actions/attest-build-provenance@v2 - with: - subject-name: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_PS_IMAGE_NAME}} - subject-digest: ${{ steps.push-ps-ghcr.outputs.digest }} - push-to-registry: true - - - name: Build and push QA PS image to quay.io - if: env.SKIP_WORKFLOW == 'false' - id: push-ps-quay - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: | - "${{ steps.quay_ps_meta.outputs.tags }}" - "${{ env.QUAY_REGISTRY }}/${{ env.QUAY_PS_IMAGE_NAME }}:pr-${{ steps.get_pr_number.outputs.result }}" - labels: ${{ steps.quay_ps_meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - file: pathservice/Containerfile - - - name: Generate QA PS Quay artifact attestation - if: env.SKIP_WORKFLOW == 'false' - uses: actions/attest-build-provenance@v2 - with: - subject-name: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_PS_IMAGE_NAME}} - subject-digest: ${{ steps.push-ps-quay.outputs.digest }} - push-to-registry: true - - - - name: Update coderefs before code changes - if: env.SKIP_WORKFLOW == 'false' - run: |- - git pull --ff-only - - - name: Update QA PS Quay image - if: env.SKIP_WORKFLOW == 'false' - id: update_qa_ps_manifest_image - env: - PR_TAG: "pr-${{ steps.get_pr_number.outputs.result }}" - run: |- - sudo wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/local/bin/yq - sudo chmod +x /usr/local/bin/yq - yq -i ' - (.images[] | select(.name == "quay.io/${{env.QUAY_PS_IMAGE_NAME}}") | .newTag) = env(PR_TAG) - ' deploy/k8s/overlays/openshift/qa/kustomization.yaml - - - name: Commit and push bump QA PS Image manifest - if: env.SKIP_WORKFLOW == 'false' - run: |- - git config user.name "platform-engineering-bot" - git config user.email "platform-engineering@redhat.com" - git add deploy/k8s/overlays/openshift/qa/kustomization.yaml - git commit -m "[CI AUTOMATION]: Bumping QA PS image to tag: pr-${{ steps.get_pr_number.outputs.result }}" -s - git push origin main diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index c8ffcfe14..37d61149f 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -6,10 +6,8 @@ on: env: GHCR_REGISTRY: ghcr.io GHCR_UI_IMAGE_NAME: ${{ github.repository }}/ui - GHCR_PS_IMAGE_NAME: ${{ github.repository }}/pathservice QUAY_REGISTRY: quay.io QUAY_UI_IMAGE_NAME: instructlab-ui/ui - QUAY_PS_IMAGE_NAME: instructlab-ui/pathservice jobs: build_and_publish_ui_prod_image: @@ -131,123 +129,3 @@ jobs: git add deploy/k8s/overlays/openshift/prod/kustomization.yaml git commit -m "[CI AUTOMATION]: Bumping Prod UI image to tag: ${{ github.event.release.tag_name }}" -s git push origin main - - build_and_publish_ps_prod_image: - name: Push UI container image to GHCR and QUAY - runs-on: ubuntu-latest - environment: registry-creds - permissions: - packages: write - contents: write - attestations: write - id-token: write - - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - - name: Log in to the GHCR container image registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GHCR_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to the Quay container image registry - uses: docker/login-action@v3 - with: - registry: ${{ env.QUAY_REGISTRY }} - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Extract metadata (tags, labels) for PS image - id: ghcr_ps_meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_PS_IMAGE_NAME }} - - - name: Extract metadata (tags, labels) for PS image - id: quay_ps_meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_PS_IMAGE_NAME }} - - - name: Build and push ps image to ghcr.io - id: push-ps-ghcr - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: ${{ steps.ghcr_ps_meta.outputs.tags }} - labels: ${{ steps.ghcr_ps_meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - file: pathservice/Containerfile - - - name: Generate GHCR PS Image attestation - uses: actions/attest-build-provenance@v2 - with: - subject-name: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_PS_IMAGE_NAME}} - subject-digest: ${{ steps.push-ps-ghcr.outputs.digest }} - push-to-registry: true - - - name: Build and push PS image to Quay.io - id: push-ps-quay - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: ${{ steps.quay_ps_meta.outputs.tags }} - labels: ${{ steps.quay_ps_meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - file: pathservice/Containerfile - - - name: Generate Quay PS Image attestation - uses: actions/attest-build-provenance@v2 - with: - subject-name: ${{ env.QUAY_REGISTRY }}/${{ env.QUAY_PS_IMAGE_NAME}} - subject-digest: ${{ steps.push-ps-quay.outputs.digest }} - push-to-registry: true - - - name: Checkout main on the repo - uses: actions/checkout@v4 - with: - token: ${{ secrets.BOT_PAT }} - ref: main - - - name: Update coderefs before code changes - run: |- - git pull --ff-only - - - name: Update Prod Quay PS image - id: update_prod_ps_manifest_image - env: - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: |- - sudo wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/local/bin/yq - sudo chmod +x /usr/local/bin/yq - yq -i ' - (.images[] | select(.name == "quay.io/instructlab-ui/pathservice") | .newTag) = env(RELEASE_TAG) - ' deploy/k8s/overlays/openshift/prod/kustomization.yaml - - - name: Commit and push bump to Prod PS image manifest - run: |- - git config user.name "platform-engineering-bot" - git config user.email "platform-engineering@redhat.com" - git add deploy/k8s/overlays/openshift/prod/kustomization.yaml - git commit -m "[CI AUTOMATION]: Bumping Prod PS image to tag: ${{ github.event.release.tag_name }}" -s - git push origin main diff --git a/Makefiles/containers-base/Makefile b/Makefiles/containers-base/Makefile index 20439718a..998659609 100644 --- a/Makefiles/containers-base/Makefile +++ b/Makefiles/containers-base/Makefile @@ -8,10 +8,6 @@ TARGET_IMAGE_ARCH?=amd64 CONTAINER_ENGINE?=docker #################### VALIDATION FUNCTIONS #################### -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - validate-container-engine: ifeq ($(CONTAINER_ENGINE),docker) @if ! command -v docker >/dev/null 2>&1; then \ @@ -38,12 +34,6 @@ ui-image: validate-container-engine src/Containerfile ## Build container image f $(CMD_PREFIX) $(CONTAINER_ENGINE) tag quay.io/instructlab-ui/ui:$(TAG) quay.io/instructlab-ui/ui:main $(CMD_PREFIX) $(CONTAINER_ENGINE) tag quay.io/instructlab-ui/ui:$(TAG) ghcr.io/instructlab/ui/ui:main -ps-image: validate-container-engine pathservice/Containerfile ## Build container image for the InstructLab PathService - $(ECHO_PREFIX) printf " %-12s pathservice/Containerfile\n" "[$(CONTAINER_ENGINE)]" - $(CMD_PREFIX) $(CONTAINER_ENGINE) build -f pathservice/Containerfile -t quay.io/instructlab-ui/pathservice:$(TAG) . - $(CMD_PREFIX) $(CONTAINER_ENGINE) tag quay.io/instructlab-ui/pathservice:$(TAG) quay.io/instructlab-ui/pathservice:main - $(CMD_PREFIX) $(CONTAINER_ENGINE) tag quay.io/instructlab-ui/pathservice:$(TAG) ghcr.io/instructlab/ui/pathservice:main - healthcheck-sidecar-image: validate-container-engine healthcheck-sidecar/Containerfile ## Build container image for the InstructLab Healthcheck-Sidecar $(ECHO_PREFIX) printf " %-12s healthcheck-sidecar/Containerfile\n" "[$(CONTAINER_ENGINE)]" $(CMD_PREFIX) $(CONTAINER_ENGINE) build -f healthcheck-sidecar/Containerfile -t quay.io/instructlab-ui/healthcheck-sidecar:$(TAG) healthcheck-sidecar diff --git a/Makefiles/deployment/kind/Makefile b/Makefiles/deployment/kind/Makefile index f225b8c50..97a8cb22c 100644 --- a/Makefiles/deployment/kind/Makefile +++ b/Makefiles/deployment/kind/Makefile @@ -2,10 +2,6 @@ # ║ KIND Targets ║ # ╚══════════════════════════════════════════════════════════╝ -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - #################### VARIABLE DEFINITIONS #################### ILAB_KUBE_CONTEXT?=kind-instructlab-ui @@ -37,10 +33,6 @@ load-images: validate-container-engine ## Load images onto Kind cluster $(CONTAINER_ENGINE) save -o $(REPO_ROOT)/healthcheck-sidecar-image.tar quay.io/instructlab-ui/healthcheck-sidecar:main ; \ kind load --name $(ILAB_KUBE_CLUSTER_NAME) image-archive $(REPO_ROOT)/healthcheck-sidecar-image.tar ; \ rm -f $(REPO_ROOT)/healthcheck-sidecar-image.tar ; \ - echo "Loading image: quay.io/instructlab-ui/pathservice:main ..." ; \ - $(CONTAINER_ENGINE) save -o $(REPO_ROOT)/pathservice.tar quay.io/instructlab-ui/pathservice:main ; \ - kind load --name $(ILAB_KUBE_CLUSTER_NAME) image-archive $(REPO_ROOT)/pathservice.tar ; \ - rm -f $(REPO_ROOT)/pathservice.tar ; \ echo "Loading image: postgres:15-alpine ..." ; \ $(CONTAINER_ENGINE) save -o $(REPO_ROOT)/postgresql-15-alpine.tar postgres:15-alpine ; \ kind load --name $(ILAB_KUBE_CLUSTER_NAME) image-archive $(REPO_ROOT)/postgresql-15-alpine.tar ; \ @@ -48,7 +40,6 @@ load-images: validate-container-engine ## Load images onto Kind cluster elif [ "$(CONTAINER_ENGINE)" == "docker" ]; then \ kind load --name $(ILAB_KUBE_CLUSTER_NAME) docker-image quay.io/instructlab-ui/ui:main ; \ kind load --name $(ILAB_KUBE_CLUSTER_NAME) docker-image quay.io/instructlab-ui/healthcheck-sidecar:main ; \ - kind load --name $(ILAB_KUBE_CLUSTER_NAME) docker-image quay.io/instructlab-ui/pathservice:main ; \ kind load --name $(ILAB_KUBE_CLUSTER_NAME) docker-image postgres:15-alpine ; \ fi; @@ -87,7 +78,6 @@ deploy: wait-for-readiness ## Deploy a InstructLab UI development stack onto a k .PHONY: redeploy redeploy: ui-image load-images ## Redeploy the InstructLab UI stack onto a kubernetes cluster $(CMD_PREFIX) kubectl --context=$(ILAB_KUBE_CONTEXT) -n $(ILAB_KUBE_NAMESPACE) rollout restart deploy/ui - $(CMD_PREFIX) kubectl --context=$(ILAB_KUBE_CONTEXT) -n $(ILAB_KUBE_NAMESPACE) rollout restart deploy/pathservice .PHONY: undeploy undeploy: ## Undeploy the InstructLab UI stack from a kubernetes cluster diff --git a/Makefiles/deployment/openshift/prod/Makefile b/Makefiles/deployment/openshift/prod/Makefile index 2edaeaaf7..1e941aedd 100644 --- a/Makefiles/deployment/openshift/prod/Makefile +++ b/Makefiles/deployment/openshift/prod/Makefile @@ -2,10 +2,6 @@ # ║ Production Openshift Deployment Targets ║ # ╚══════════════════════════════════════════════════════════╝ -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - #################### VARIABLE DEFINITIONS #################### SEALED_SECRETS_CONTROLLER_NAMESPACE=kube-system @@ -47,7 +43,6 @@ deploy-prod-openshift: ## Deploy production stack of the InstructLab UI on OpenS .PHONY: redeploy-prod-openshift redeploy-prod-openshift: ## Redeploy production stack of the InstructLab UI on OpenShift $(CMD_PREFIX) $(OC) -n $(ILAB_KUBE_NAMESPACE) rollout restart deploy/ui - $(CMD_PREFIX) $(OC) -n $(ILAB_KUBE_NAMESPACE) rollout restart deploy/pathservice .PHONY: undeploy-prod-openshift undeploy-prod-openshift: ## Undeploy production stack of the InstructLab UI on OpenShift diff --git a/Makefiles/deployment/openshift/qa/Makefile b/Makefiles/deployment/openshift/qa/Makefile index 45aefb9fb..44102a350 100644 --- a/Makefiles/deployment/openshift/qa/Makefile +++ b/Makefiles/deployment/openshift/qa/Makefile @@ -1,9 +1,6 @@ # ╔══════════════════════════════════════════════════════════╗ # ║ QA Openshift Deployment Targets ║ # ╚══════════════════════════════════════════════════════════╝ -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) #################### DEPLOYMENT FUNCTIONS #################### @@ -22,7 +19,6 @@ deploy-qa-openshift: ## Deploy QA stack of the InstructLab UI on OpenShift .PHONY: redeploy-qa-openshift redeploy-qa-openshift: ## Redeploy QA stack of the InstructLab UI on OpenShift $(CMD_PREFIX) $(OC) -n $(ILAB_KUBE_NAMESPACE) rollout restart deploy/ui - $(CMD_PREFIX) $(OC) -n $(ILAB_KUBE_NAMESPACE) rollout restart deploy/pathservice .PHONY: undeploy-qa-openshift undeploy-qa-openshift: ## Undeploy QA stack of the InstructLab UI on OpenShift diff --git a/Makefiles/devcontainer/Makefile b/Makefiles/devcontainer/Makefile index 8acb22076..983f719b9 100644 --- a/Makefiles/devcontainer/Makefile +++ b/Makefiles/devcontainer/Makefile @@ -2,10 +2,6 @@ # ║ Devcontainer Targets ║ # ╚══════════════════════════════════════════════════════════╝ -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - #################### DEPENDENCY FUNCTIONS #################### DEVCONTAINER_BINARY_EXISTS ?= $(shell command -v devcontainer) diff --git a/Makefiles/linting/Makefile b/Makefiles/linting/Makefile index 7dcd450a9..cc5623f59 100644 --- a/Makefiles/linting/Makefile +++ b/Makefiles/linting/Makefile @@ -2,10 +2,6 @@ # ║ Linting Targets ║ # ╚══════════════════════════════════════════════════════════╝ -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - ###################### LINTING FUNCTIONS ##################### ##@ Lint commands - Commands for linting various file formats diff --git a/Makefiles/local/Makefile b/Makefiles/local/Makefile index ee4b18f19..311356d52 100644 --- a/Makefiles/local/Makefile +++ b/Makefiles/local/Makefile @@ -2,10 +2,6 @@ # ║ Local Development Targets ║ # ╚══════════════════════════════════════════════════════════╝ -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - #################### DEPENDENCY FUNCTIONS #################### ##@ Local Deployment - Helper commands for deploying UI stack locally on your machine @@ -21,17 +17,15 @@ check-python3: #################### DEPLOYMENT FUNCTIONS #################### .PHONY: start-dev-local -start-dev-local: ## Start the npm and pathservice local instances - $(CMD_PREFIX) echo "Starting ui and pathservice..." - $(CMD_PREFIX) cd $(REPO_ROOT)/pathservice; go run main.go & echo $$! > ../pathservice.pid - $(CMD_PREFIX) npm run dev & echo $$! > ui.pid +start-dev-local: ## Start the npm local instances + $(CMD_PREFIX) echo "Starting ui ..." + $(CMD_PREFIX) npm run dev & $(CMD_PREFIX) echo "Development environment started." .PHONY: stop-dev-local -stop-dev-local: ## Stop the npm and pathservice local instances - $(CMD_PREFIX) echo "Stopping ui and pathservice..." - $(CMD_PREFIX) if [ -f ui.pid ]; then kill -2 `cat ui.pid` && rm ui.pid || echo "Failed to stop ui"; fi - $(CMD_PREFIX) if [ -f pathservice.pid ]; then kill -2 `cat pathservice.pid` && rm pathservice.pid || echo "Failed to stop pathservice"; fi +stop-dev-local: ## Stop the npm local instances + $(CMD_PREFIX) echo "Stopping ui ..." + $(CMD_PREFIX) pkill npm $(CMD_PREFIX) echo "Development environment stopped." .PHONY: start-healthcheck-sidecar-local diff --git a/Makefiles/podman-compose/Makefile b/Makefiles/podman-compose/Makefile index 6a2343d2a..cafce5048 100644 --- a/Makefiles/podman-compose/Makefile +++ b/Makefiles/podman-compose/Makefile @@ -3,10 +3,6 @@ # ║ Container Runtime Compose Deployment Targets ║ # ╚══════════════════════════════════════════════════════════╝ -.PHONY: help -help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - #################### DEPLOYMENT FUNCTIONS #################### ##@ Podman Deployment - Helper commands for deploying UI stack in Podman diff --git a/README.md b/README.md index eaf368e7a..e836e7107 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Current scope of the project is to work on following personas: The technical overview and developer docs for getting started can be found [here](docs/development.md). +The UI can be installed and run locally by following the instruction [here](installers/podman/README.md). + ## Contributing If you have suggestions for how InstructLab UI can be improved, please open an [issue](https://github.com/instructlab/ui/issues) and tag it with `user suggested` label. We'd love all and any contributions. diff --git a/api-server/handlers.go b/api-server/handlers.go index 8c8fa2bc0..e3231b6aa 100644 --- a/api-server/handlers.go +++ b/api-server/handlers.go @@ -660,22 +660,20 @@ func (srv *ILabServer) runVllmContainerHandler( srv.log.Infof("No existing job found for model '%s'. Starting a new job.", servedModelName) cmdArgs := []string{ - "run", "--rm", - fmt.Sprintf("--device=nvidia.com/gpu=%d", gpuIndex), - fmt.Sprintf("-e=NVIDIA_VISIBLE_DEVICES=%d", gpuIndex), - "-v", "/usr/local/cuda-12.4/lib64:/usr/local/cuda-12.4/lib64", + "run", "--rm", "-it", + "--device", fmt.Sprintf("nvidia.com/gpu=%d", gpuIndex), + "--security-opt", "label=disable", + "--net", "host", + "--shm-size", "10G", + "--pids-limit", "-1", "-v", fmt.Sprintf("%s:%s", hostVolume, containerVolume), - "-p", fmt.Sprintf("%s:%s", port, port), - "--ipc=host", - "vllm/vllm-openai:latest", - "--host", "0.0.0.0", - "--port", port, - "--model", modelPath, - "--load-format", "safetensors", - "--config-format", "hf", - "--trust-remote-code", - "--device", "cuda", + "--entrypoint", "/opt/app-root/bin/vllm", + "registry.redhat.io/rhelai1/instructlab-nvidia-rhel9:1.4-1738905416", + "serve", modelPath, "--served-model-name", servedModelName, + "--load-format", "safetensors", + "--host", "127.0.0.1", + "--port", port, } // Log the command for debugging @@ -685,7 +683,7 @@ func (srv *ILabServer) runVllmContainerHandler( // Create a unique job ID and a log file jobID := fmt.Sprintf("v-%d", time.Now().UnixNano()) logFilePath := filepath.Join("logs", fmt.Sprintf("%s.log", jobID)) - srv.log.Infof("Starting vllm-openai container with job_id: %s, logs: %s", jobID, logFilePath) + srv.log.Infof("Starting vllm container with job_id: %s, logs: %s", jobID, logFilePath) cmd := exec.Command("podman", cmdArgs...) diff --git a/api-server/qna-eval/requirements.txt b/api-server/qna-eval/requirements.txt index 4cf94a672..8b9f69428 100644 --- a/api-server/qna-eval/requirements.txt +++ b/api-server/qna-eval/requirements.txt @@ -39,7 +39,7 @@ google-api-core==2.24.0 google-auth==2.37.0 googleapis-common-protos==1.66.0 grpcio==1.69.0 -h11==0.14.0 +h11==0.16.0 httpcore==1.0.7 httptools==0.6.4 httpx==0.28.1 @@ -130,7 +130,7 @@ python-dateutil==2.9.0.post0 python-dotenv==1.0.1 PyYAML==6.0.2 pyzmq==26.2.0 -ray==2.40.0 +ray==2.43.0 referencing==0.35.1 regex==2024.11.6 requests==2.32.3 @@ -150,7 +150,7 @@ textual==1.0.0 tiktoken==0.7.0 tinycss2==1.4.0 tokenizers==0.21.0 -torch==2.5.1 +torch==2.7.0 torchvision==0.20.1 tornado==6.4.2 tqdm==4.67.1 @@ -163,14 +163,14 @@ urllib3==2.3.0 uvicorn==0.34.0 uvloop==0.21.0 virtualenv==20.28.1 -vllm==0.7.2 +vllm==0.8.4 watchfiles==1.0.3 wcwidth==0.2.13 webencodings==0.5.1 websockets==14.1 wrapt==1.17.0 xformers==0.0.28.post3 -xgrammar==0.1.9 +xgrammar==0.1.18 yarg==0.1.9 yarl==1.18.3 zipp==3.21.0 diff --git a/api-server/rhelai-install/rhelai-install.sh b/api-server/rhelai-install/rhelai-install.sh index 5bedeb600..308e5a9fa 100755 --- a/api-server/rhelai-install/rhelai-install.sh +++ b/api-server/rhelai-install/rhelai-install.sh @@ -22,11 +22,11 @@ if [ -d "/tmp/api-server" ]; then fi mkdir -p /tmp/api-server -cd /tmp/ui/api-server -wget https://instructlab-ui.s3.us-east-1.amazonaws.com/apiserver/apiserver-linux-amd64.tar.gz -tar -xzf apiserver-linux-amd64.tar.gz -mv apiserver-linux-amd64/ilab-apiserver /usr/local/sbin -rm -rf apiserver-linux-amd64 apiserver-linux-amd64.tar.gz +cd /tmp/api-server +curl -sLO https://instructlab-ui.s3.us-east-1.amazonaws.com/apiserver/apiserver-linux-amd64.tar.gz +tar -xzf apiserver-linux-amd64.tar.gz +mv apiserver-linux-amd64/ilab-apiserver "{$HOME}/.local/bin" +rm -rf apiserver-linux-amd64 apiserver-linux-amd64.tar.gz /tmp/api-server CUDA_FLAG="" diff --git a/api-server/vllm-serve.go b/api-server/vllm-serve.go index 48436d6ad..cc4f89045 100644 --- a/api-server/vllm-serve.go +++ b/api-server/vllm-serve.go @@ -29,7 +29,7 @@ func (srv *ILabServer) ListVllmContainers() ([]VllmContainer, error) { format := "{{.ID}}|{{.Image}}|{{.Command}}|{{.CreatedAt}}|{{.Status}}|{{.Ports}}|{{.Names}}" cmd := exec.Command("podman", "ps", - "--filter", "ancestor=vllm/vllm-openai:latest", + "--filter", "ancestor=registry.redhat.io/rhelai1/instructlab-nvidia-rhel9:1.4-1738905416", "--format", format, ) @@ -102,7 +102,8 @@ func (srv *ILabServer) ExtractVllmArgs(containerID string) (string, string, erro containerID, err, inspectErr.String()) } - // The command is a JSON array, e.g. ["--host","0.0.0.0","--port","8000","--model","/path","--served-model-name","pre-train"] + // The command is a JSON array, e.g.: + // ["serve","/var/home/cloud-user/.cache/instructlab/models/granite-8b-starter-v1","--served-model-name","pre-train","--load-format","safetensors","--host","127.0.0.1","--port","8000"] var cmdArgs []string if err := json.Unmarshal(inspectOut.Bytes(), &cmdArgs); err != nil { return "", "", fmt.Errorf("error unmarshalling command args for container %s: %v", @@ -138,6 +139,13 @@ func (srv *ILabServer) parseVllmArgs(args []string) (string, string, error) { } } } + + // If modelPath wasn't set via the flag, check for a positional model path. + // If the command starts with "serve" and a second argument exists, use that as the model path. + if modelPath == "" && len(args) > 1 && args[0] == "serve" { + modelPath = args[1] + } + if servedModelName == "" || modelPath == "" { return "", "", errors.New("required arguments --served-model-name or --model not found") } diff --git a/deploy/k8s/base/doclingserve/deployment.yaml b/deploy/k8s/base/doclingserve/deployment.yaml index ce54cee40..56e8a4a1a 100644 --- a/deploy/k8s/base/doclingserve/deployment.yaml +++ b/deploy/k8s/base/doclingserve/deployment.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: doclingserve - image: ghcr.io/ds4sd/docling-serve-cpu:main + image: quay.io/ds4sd/docling-serve:latest imagePullPolicy: Always ports: - name: http diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 8f2c6cb56..e3dd06d34 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -4,7 +4,6 @@ namespace: instructlab resources: - namespace.yaml - ui - - pathservice - doclingserve labels: - includeSelectors: true diff --git a/deploy/k8s/base/pathservice/deployment.yaml b/deploy/k8s/base/pathservice/deployment.yaml deleted file mode 100644 index 38f83104b..000000000 --- a/deploy/k8s/base/pathservice/deployment.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pathservice -spec: - replicas: 1 - strategy: - type: RollingUpdate - template: - spec: - containers: - - name: pathservice - image: quay.io/instructlab-ui/pathservice:PATCHED_FROM_OVERLAYS - imagePullPolicy: Always - resources: - requests: - cpu: 100m - memory: 200Mi - ports: - - name: http - protocol: TCP - containerPort: 4000 - restartPolicy: Always diff --git a/deploy/k8s/base/pathservice/kustomization.yaml b/deploy/k8s/base/pathservice/kustomization.yaml deleted file mode 100644 index 9b8eb74e4..000000000 --- a/deploy/k8s/base/pathservice/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - deployment.yaml - - service.yaml -labels: - - includeSelectors: true - pairs: - app.kubernetes.io/component: pathservice - app.kubernetes.io/instance: pathservice - app.kubernetes.io/name: pathservice diff --git a/deploy/k8s/base/pathservice/service.yaml b/deploy/k8s/base/pathservice/service.yaml deleted file mode 100644 index d9d4d4633..000000000 --- a/deploy/k8s/base/pathservice/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: pathservice -spec: - clusterIP: None - selector: - app.kubernetes.io/component: pathservice - app.kubernetes.io/instance: pathservice - app.kubernetes.io/name: pathservice - ports: - - port: 4000 - targetPort: 4000 diff --git a/deploy/k8s/overlays/kind/kustomization.yaml b/deploy/k8s/overlays/kind/kustomization.yaml index 138344c7e..4f7c2b2b1 100644 --- a/deploy/k8s/overlays/kind/kustomization.yaml +++ b/deploy/k8s/overlays/kind/kustomization.yaml @@ -25,12 +25,3 @@ patches: - op: replace path: /spec/template/spec/containers/0/image value: quay.io/instructlab-ui/ui:main - - # Override the pathservice image for Kind deployment - - target: - kind: Deployment - name: pathservice - patch: |- - - op: replace - path: /spec/template/spec/containers/0/image - value: quay.io/instructlab-ui/pathservice:main # Override this image if you want to use a different pathservice image diff --git a/deploy/k8s/overlays/openshift/prod/kustomization.yaml b/deploy/k8s/overlays/openshift/prod/kustomization.yaml index 5ce6e0d27..2f282e9ea 100644 --- a/deploy/k8s/overlays/openshift/prod/kustomization.yaml +++ b/deploy/k8s/overlays/openshift/prod/kustomization.yaml @@ -31,31 +31,29 @@ patches: - op: replace path: /spec/template/spec/containers/0/envFrom/0/secretRef/name value: prod.env - # - op: add - # path: /spec/template/spec/containers/0/readinessProbe - # value: - # exec: - # command: - # - sh - # - -c - # - "/opt/app-root/src/src/healthcheck-probe.sh" - # initialDelaySeconds: 5 - # periodSeconds: 10 - # - op: add - # path: /spec/template/spec/containers/- - # value: - # name: model-endpoint-healthcheck-sidecar - # image: quay.io/instructlab-ui/healthcheck-sidecar - # # imagePullPolicy: Always # until image lands in quay cannot use pullPolicy: Always - # ports: - # - containerPort: 8080 - # envFrom: - # - secretRef: - # name: prod.env + # - op: add + # path: /spec/template/spec/containers/0/readinessProbe + # value: + # exec: + # command: + # - sh + # - -c + # - "/opt/app-root/src/src/healthcheck-probe.sh" + # initialDelaySeconds: 5 + # periodSeconds: 10 + # - op: add + # path: /spec/template/spec/containers/- + # value: + # name: model-endpoint-healthcheck-sidecar + # image: quay.io/instructlab-ui/healthcheck-sidecar + # # imagePullPolicy: Always # until image lands in quay cannot use pullPolicy: Always + # ports: + # - containerPort: 8080 + # envFrom: + # - secretRef: + # name: prod.env images: - name: quay.io/instructlab-ui/ui - newTag: v1.0.0-beta.3 - - name: quay.io/instructlab-ui/pathservice - newTag: v1.0.0-beta.3 - # - name: quay.io/instructlab-ui/healthcheck-sidecar - # newTag: main # not currently available in our quay org + newTag: v1.1.1 +# - name: quay.io/instructlab-ui/healthcheck-sidecar +# newTag: main # not currently available in our quay org diff --git a/deploy/k8s/overlays/openshift/qa/kustomization.yaml b/deploy/k8s/overlays/openshift/qa/kustomization.yaml index 16fcc7ff8..9f9b2fa26 100644 --- a/deploy/k8s/overlays/openshift/qa/kustomization.yaml +++ b/deploy/k8s/overlays/openshift/qa/kustomization.yaml @@ -30,8 +30,6 @@ patches: patch: "- op: replace\n path: /spec/template/spec/containers/0/envFrom/0/secretRef/name\n value: qa.env\n- op: add \n path: /spec/template/spec/containers/0/readinessProbe\n value:\n exec:\n command:\n - sh\n - -c\n - \"/opt/app-root/src/src/healthcheck-probe.sh\"\n initialDelaySeconds: 5\n periodSeconds: 10\n- op: add\n path: /spec/template/spec/containers/-\n value:\n name: model-endpoint-healthcheck-sidecar\n image: quay.io/instructlab-ui/healthcheck-sidecar\n imagePullPolicy: Always # until image lands in quay cannot use pullPolicy: Always\n ports:\n - containerPort: 8080\n envFrom:\n - secretRef:\n name: qa.env" images: - name: quay.io/instructlab-ui/ui - newTag: pr-579 - - name: quay.io/instructlab-ui/pathservice - newTag: pr-580 + newTag: pr-752 - name: quay.io/instructlab-ui/healthcheck-sidecar newTag: pr-375 diff --git a/deploy/podman/github/README.md b/deploy/podman/github/README.md index acd0993f0..ceef1c40a 100644 --- a/deploy/podman/github/README.md +++ b/deploy/podman/github/README.md @@ -32,6 +32,7 @@ kubectl create secret generic ui-env \ --from-literal=IL_MERLINITE_API="" \ --from-literal=IL_MERLINITE_MODEL_NAME="" \ --from-literal=IL_ENABLE_DEV_MODE=false \ + --from-literal=IL_ENABLE_DOC_CONVERSION=false \ --dry-run=client -o yaml > secret.yaml ``` @@ -87,14 +88,6 @@ If you are playing with Github mode deployment for making upstream contributions > 1. Uncomment the `securityContext` in the `instructlab-ui.yaml` file and set the value of `runAsGroup` to the value of the host user's group id. > `id` command should give you the `gid` of the host user. > -> 2. Make sure cpu and cpusets cgroup controllers are enabled for the user. To check if the cgroup controllers are enabled, run the following command: -> ```cat "/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/cgroup.controllers"``` -> -> If the output of the above command does not contain `cpu` and `cpuset`, then you need to enable these cgroup controllers for the user. To enable these cgroup controllers, create the following file `/etc/systemd/system/user@.service.d/delegate.conf` with the following content: -> ->```[Service] -> Delegate=memory pids cpu cpuset``` -> Save the file and run `sudo systemctl daemon-reload` followed by `sudo systemctl restart user@$(id -u).service` to apply the changes. ## Accessing the UI diff --git a/deploy/podman/github/instructlab-ui.yaml b/deploy/podman/github/instructlab-ui.yaml index 5cacdacd3..36ba94428 100644 --- a/deploy/podman/github/instructlab-ui.yaml +++ b/deploy/podman/github/instructlab-ui.yaml @@ -1,27 +1,5 @@ apiVersion: apps/v1 kind: Deployment -metadata: - name: pathservice -spec: - replicas: 1 - selector: - matchLabels: - app: pathservice - template: - metadata: - labels: - app: pathservice - spec: - containers: - - name: pathservice - image: ghcr.io/instructlab/ui/pathservice:main - ports: - - containerPort: 4000 - hostPort: 4000 - imagePullPolicy: Always ---- -apiVersion: apps/v1 -kind: Deployment metadata: name: doclingserve spec: @@ -36,7 +14,7 @@ spec: spec: containers: - name: doclingserve - image: ghcr.io/ds4sd/docling-serve-cpu:main + image: quay.io/ds4sd/docling-serve:latest ports: - containerPort: 5001 hostPort: 5001 @@ -58,7 +36,7 @@ spec: spec: containers: - name: ui - image: ghcr.io/instructlab/ui/ui:main + image: quay.io/instructlab-ui/ui:latest env: - name: IL_UI_DEPLOYMENT valueFrom: @@ -140,6 +118,11 @@ spec: secretKeyRef: name: ui-env key: IL_ENABLE_DEV_MODE + - name: IL_ENABLE_DOC_CONVERSION + valueFrom: + secretKeyRef: + name: ui-env + key: IL_ENABLE_DOC_CONVERSION ports: - containerPort: 3000 hostPort: 3000 diff --git a/deploy/podman/github/secret.yaml.example b/deploy/podman/github/secret.yaml.example index 583d3622f..05b0aa96f 100644 --- a/deploy/podman/github/secret.yaml.example +++ b/deploy/podman/github/secret.yaml.example @@ -16,6 +16,7 @@ data: IL_MERLINITE_API: "" IL_MERLINITE_MODEL_NAME: "" IL_ENABLE_DEV_MODE: "" + IL_ENABLE_DOC_CONVERSION: "" kind: Secret metadata: creationTimestamp: null diff --git a/deploy/podman/native/README.md b/deploy/podman/native/README.md index 10a7b3045..fa3df1c73 100644 --- a/deploy/podman/native/README.md +++ b/deploy/podman/native/README.md @@ -1,64 +1,13 @@ # InstructLab UI Native mode deployment in Podman -Please follow the below instructions to deploy UI stack with Native mode enabled in Podman. +Please follow the below instructions to deploy UI stack with Native mode enabled in Podman. These instructions are mostly manual instructions. If you would like to use installer to install the UI stack, please follow the instruction provided [here](../../../installers/podman/README.md). -## Deploy using the installer (Recommended) - -Make a temporary directory and download the installer - -```shell -mkdir instructlab-ui -cd instructlab-ui - -curl -o ilab-ui-native-installer.sh -fsSL https://raw.githubusercontent.com/instructlab/ui/refs/heads/main/installers/podman/ilab-ui-native-installer.sh -``` - -Give execution permission to the install - -```shell -chmod a+x ilab-ui-native-installer.sh -``` - -Execute the installer and follow the instructions prompted on the terminal. - -If your deployment machine has InstructLab (ilab CLI) setup, either on the host or in python virtual environment, use the following command - -```shell -./ilab-ui-native-installer.sh install --username --password - -e.g ./ilab-ui-native-installer.sh install --username admin --password passw0rd -``` - -If your deployment machine don't have InstructLab CLI setup, please clone the taxonomy repo and fire the following command. - -```shell -./ilab-ui-native-installer.sh install --username --password --taxonomy-dir - -e.g ./ilab-ui-native-installer.sh install --username admin --password passw0rd --taxonomy-dir /Users/johndoe/instructlab/taxonomy -``` - ->[!NOTE] -> In the absence of InstructLab CLI, UI won't be able to support the synthetic data generation and fine tuning, but skill and knowledge contribution should work as expected. - -If you are deploying the UI stack on a remote machine, please provide the auth url in the input - -```shell -./ilab-ui-native-installer.sh install --username --password --taxonomy-dir --auth-url http://:3000 -``` - -Please use `--help` to see more options supported by the installer. - -## Deploy manually - -If you would like to install the UI stack manually, it's a two step process - -- Generate the secret file with the required input -- Deploy the UI stack manifest file using podman. +## Generate Secret A secret is required to provide required input to the UI stack in a secure way. -There are two options to generate the secret's file, either using `kubectl` or filling in values in the `secret.yaml` provided. +Two options exist to generate the secret, either using `kubectl` or filling in values in the `secret.yaml` provided. ### Generate secrets using kubectl @@ -76,6 +25,7 @@ kubectl create secret generic ui-env \ --from-literal=NEXT_PUBLIC_TAXONOMY_ROOT_DIR="" \ --from-literal=NEXT_PUBLIC_EXPERIMENTAL_FEATURES="false" \ --from-literal=IL_ENABLE_DEV_MODE=false \ + --from-literal=IL_ENABLE_DOC_CONVERSION=false \ --dry-run=client -o yaml > secret.yaml ``` @@ -119,14 +69,6 @@ podman kube play instructlab-ui.yaml > 1. Uncomment the `securityContext` in the `instructlab-ui.yaml` file and set the value of `runAsGroup` to the value of the host user's group id. > `id` command should give you the `gid` of the host user. > -> 2. Make sure cpu and cpusets cgroup controllers are enabled for the user. To check if the cgroup controllers are enabled, run the following command: -> ```cat "/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/cgroup.controllers"``` -> -> If the output of the above command does not contain `cpu` and `cpuset`, then you need to enable these cgroup controllers for the user. To enable these cgroup controllers, create the following file `/etc/systemd/system/user@.service.d/delegate.conf` with the following content: -> ->```[Service] -> Delegate=memory pids cpu cpuset``` -> Save the file and run `sudo systemctl daemon-reload` followed by `sudo systemctl restart user@$(id -u).service` to apply the changes. ## Accessing the UI @@ -134,12 +76,6 @@ The Instructlab UI should now be accessible from `http://localhost:3000` or `htt ## Cleaning up -If you used installer to install the UI stack, fire the following command - -```shell -./ilab-ui-native-installer.sh uninstall -``` - To clean up the deployment, use `podman kube down` to delete the deployment. ```bash diff --git a/deploy/podman/native/instructlab-ui.yaml b/deploy/podman/native/instructlab-ui.yaml index d60177d6b..5402098bd 100644 --- a/deploy/podman/native/instructlab-ui.yaml +++ b/deploy/podman/native/instructlab-ui.yaml @@ -1,27 +1,5 @@ apiVersion: apps/v1 kind: Deployment -metadata: - name: pathservice -spec: - replicas: 1 - selector: - matchLabels: - app: pathservice - template: - metadata: - labels: - app: pathservice - spec: - containers: - - name: pathservice - image: ghcr.io/instructlab/ui/pathservice:main - ports: - - containerPort: 4000 - hostPort: 4000 - imagePullPolicy: Always ---- -apiVersion: apps/v1 -kind: Deployment metadata: name: doclingserve spec: @@ -36,7 +14,7 @@ spec: spec: containers: - name: doclingserve - image: ghcr.io/ds4sd/docling-serve:main + image: quay.io/ds4sd/docling-serve: ports: - containerPort: 5001 hostPort: 5001 @@ -58,7 +36,7 @@ spec: spec: containers: - name: ui - image: ghcr.io/instructlab/ui/ui:main + image: quay.io/instructlab-ui/ui: securityContext: runAsGroup: 1000 volumeMounts: @@ -105,6 +83,11 @@ spec: secretKeyRef: name: ui-env key: IL_ENABLE_DEV_MODE + - name: IL_ENABLE_DOC_CONVERSION + valueFrom: + secretKeyRef: + name: ui-env + key: IL_ENABLE_DOC_CONVERSION - name: NEXT_PUBLIC_API_SERVER valueFrom: secretKeyRef: diff --git a/deploy/podman/native/secret.yaml.example b/deploy/podman/native/secret.yaml.example index afabcfeb8..11f2df614 100644 --- a/deploy/podman/native/secret.yaml.example +++ b/deploy/podman/native/secret.yaml.example @@ -1,14 +1,15 @@ apiVersion: v1 data: - IL_UI_ADMIN_PASSWORD: - IL_UI_ADMIN_USERNAME: - IL_UI_DEPLOYMENT: - IL_ENABLE_DEV_MODE: - NEXT_PUBLIC_EXPERIMENTAL_FEATURES: - NEXT_PUBLIC_TAXONOMY_ROOT_DIR: - NEXTAUTH_URL: - NEXTAUTH_SECRET: - NEXT_PUBLIC_API_SERVER: + IL_UI_ADMIN_PASSWORD: "" + IL_UI_ADMIN_USERNAME: "" + IL_UI_DEPLOYMENT: "" + IL_ENABLE_DEV_MODE: "" + IL_ENABLE_DOC_CONVERSION: "" + NEXT_PUBLIC_EXPERIMENTAL_FEATURES: "" + NEXT_PUBLIC_TAXONOMY_ROOT_DIR: "" + NEXTAUTH_URL: "" + NEXTAUTH_SECRET: "" + NEXT_PUBLIC_API_SERVER: "" kind: Secret metadata: diff --git a/docs/development.md b/docs/development.md index 2baece568..f97505f82 100644 --- a/docs/development.md +++ b/docs/development.md @@ -17,10 +17,7 @@ cp .env.native.example .env make start-dev-local ``` -This will start the UI and the dependent pathservice locally on the machine. - -> [!NOTE] -> It might ask for permission to allow to listen on port 4000. +This will start the UI on the machine listening on default port 3000 To stop the local dev environment run the following: @@ -110,26 +107,15 @@ Example [.env](../.env.example) file. ## Local Dev Chat Environment -### 1) Using the ilab command line tool - -For the chat functionality to work you need a ilab model chat instance. To run this locally: - -`cd server` +### Using the ilab command line tool -[https://github.com/instructlab/instructlab?tab=readme-ov-file#-getting-started](https://github.com/instructlab/instructlab?tab=readme-ov-file#-getting-started) +For the chat functionality to work you need a ilab model chat instance. To run this locally see [Ilab Getting Started](https://github.com/instructlab/instructlab?tab=readme-ov-file#-getting-started). After you use the `ilab serve` command you should have, by default, a chat server instance running on port 8000. -### 2) Using Podman - -#### Current issues - -- The docker image that runs the server does not utilise Mac Metal GPU and therefore is very slow when answering prompts -- The docker image is very large as it contains the model itself. Potential to have the model incorporated via a docker volume to reduce the size of the actual image. - -`docker run -p 8000:8000 aevo987654/instructlab_chat_8000:v2` +### 2) Running in a Container -This should run a server on port 8000 +See the upstream CLI documentation for running ilab in a [container](https://github.com/instructlab/instructlab/tree/main/containers). ### Configuring the chat environment to use a local ilab model chat instance @@ -271,16 +257,16 @@ If you'd like to run a specific single test, use the following command with the ** NOTE: requires the `devcontainer` binary -A devcontainer is provided in case you don't want to or can't install these dependencies and tools into you local enviroment. +A devcontainer is provided in case you don't want to or can't install these dependencies and tools into you local environment. Additionally, make commands have been provided to make it very easy to spin the environment up or down. To get setup, simple use the `make cycle-dev-container` target, which will check for existing versions of the devcontainer image, delete their pods and the image to ensure you have a clean start, build it from scratch and start the container. -Alternatively you can use the `make build-dev-container`, `make start-dev-container` to buildand run the container respectively. +Alternatively you can use the `make build-dev-container`, `make start-dev-container` to build and run the container respectively. After simply `make enter-dev-container` to exec into it. It is compatible with both `docker` and `podman` which you can set with the `CONTAINER_ENGINE` environment variable. The dev container will mount your local `.env` file into the workspace as well, so you can develop without having to -reconstruct your settings. Currently the `devcontainer` does not support intelligent port reassigment, it is pinned +reconstruct your settings. Currently the `devcontainer` does not support intelligent port reassignment, it is pinned to port `3000`. ## Updating the Sealed Secrets for the ArgoCD Application @@ -302,7 +288,7 @@ BE CERTAIN to delete the un-encrypted secret file, we do not want to leak these to its correct location within this repo. The goal, however, is to keep these secrets updated with the contents of the `.env` file. -You can do this using the command below from the root of the repo, however be sure to subsitute your environment (`prod` or `qa`) +You can do this using the command below from the root of the repo, however be sure to substitute your environment (`prod` or `qa`) where it asks you to: ```bash diff --git a/installers/podman/README.md b/installers/podman/README.md new file mode 100644 index 000000000..645f19b23 --- /dev/null +++ b/installers/podman/README.md @@ -0,0 +1,126 @@ +# Local Install of InstructLab UI + +Instructions on how to install the InstructLab UI on your local development machine such as a MacBook or Linux Machine. + +## Installation Prerequisites + +To run the UI locally `ilab-ui-native-installer.sh` needs to be downloaded: + +```bash +mkdir ui-local +cd ui-local +curl -o ilab-ui-native-installer.sh -fsSL https://raw.githubusercontent.com/instructlab/ui/refs/heads/main/installers/podman/ilab-ui-native-installer.sh +chmod a+x ilab-ui-native-installer.sh +``` + +The UI has two pre-requisites: Podman and InstructLab. + +To install these on a Mac run: + +```bash +brew install podman +podman machine init +podman machine start +``` + +On a Linux (Fedora) run: + +```bash +dnf install podman +podman machine init +podman machine start +``` + +For Instructlab installation and initialization run: + +```bash +python3.11 -m venv venv +source ./venv/bin/activate +pip install instructlab +ilab config init --non-interactive +``` + +Developers of Instructlab can use the virtual environments they already have on their systems. + +## Starting the UI + +Once the ilab is set up, run `ilab-ui-native-installer.sh` with the username and password of your preference to start the UI up. + +```bash +./ilab-ui-native-installer.sh install --username admin --password passw0rd! +``` + +If you want to install the main branch of the UI project, use the following command + +```bash +./ilab-ui-native-installer.sh install --username admin --password passw0rd! --deploy main +``` + +UI Installer does the following: + +- Check if InstructLab is set up on the host. +- Extract the taxonomy repository directory path from the InstructLab configuration file +- Download and install the ilab-apiserver binary. It allows the UI to communicate actions to InstructLab and retrieve the results of the commands. +- Generates the secret.yaml file to fill in all the user-provided information and the discovered information securely. +- Download the InstructLab deployment YAML and update the YAML with the required parameters. +- Deploy the secrets and the deployment YAML on the Podman, to set up the UI. + +Once the installation is successfully completed, you can log in to the UI at `http://localhost:3000` with the username and password provided to the installer. + +A taxonomy is needed to start the UI. Without providing `--taxonomy-dir` to the script the default taxonomy used is the upstream taxonomy in `~/.local/share/instructlab/taxonomy`. + +The custom empty taxonomy can be used. The taxonomy must be an initialized git repository. + +```bash +mkdir /home/user/ui-local/taxonomy +cd /home/user/ui-local/taxonomy +git config --global init.defaultBranch main +``` + +```bash +./ilab-ui-native-installer.sh install --username admin --password passw0rd! --taxonomy-dir /home/user/taxonomy +Checking if ilab (InstructLab CLI) is installed natively... + +ilab is natively installed on the host. + +NOTE: If you are using python virtual environment for InstructLab setup, you can use --python-venv-dir option to skip the discovery. + +Seems like ilab is configured on the host. Discover the taxonomy path... + +Discovering ilab configured taxonomy on the host... + +ilab configured taxonomy repo is detected under the directory - /home/user/.local/share/instructlab/taxonomy + +and it's not same as provided by the user - /home/user/ui-local/taxonomy + +It's recommended to use the taxonomy repo configured for the ilab CLI. + +In case you do not choose ilab configured taxonomy, data generation and fine tune features will not work. + +But skill and knowledge contribution features should work as expected. + +Please choose the taxonomy repo you would like to use with InstructLab UI: +1. ilab Taxonomy - /home/user/.local/share/instructlab/taxonomy +2. Provided Taxonomy - /home/user/ui-local/taxonomy +Please choose the taxonomy repo you would like to use with InstructLab UI? (1/2): 2 +... +``` + +## Stopping the UI + +To stop the UI, the `uninstall` command can be used with the `ilab-ui-native-installer.sh` script. + +```bash +./ilab-ui-native-installer.sh uninstall +Are you sure you want to uninstall the InstructLab UI stack? (yes/no): yes +``` + +## Troubleshooting + +If pod startup times out with: + +```text +Error: playing YAML file: encountered while bringing up pod ui-pod: initializing source docker://quay.io/instructlab-ui/ui:latest: pinging container registry quay.io: Get "https://quay.io/v2/": dial tcp: lookup quay.io: no such host +``` + +If Docker compatibility mode is enabled, try disabling it. diff --git a/installers/podman/ilab-ui-native-installer.sh b/installers/podman/ilab-ui-native-installer.sh index af9938eed..c365e81ba 100755 --- a/installers/podman/ilab-ui-native-installer.sh +++ b/installers/podman/ilab-ui-native-installer.sh @@ -18,6 +18,7 @@ declare DEV_MODE="false" declare EXPERIMENTAL_FEATURES="" declare PYENV_DIR="" declare API_SERVER_URL="" +declare DEPLOY="released" BINARY_INSTALL_DIR=./ IS_ILAB_INSTALLED="true" @@ -54,6 +55,9 @@ usage_install() { echo -e " Installer auto enable these features if InstructLab is setup on host." echo -e "${green} --python-venv-dir ${reset} (Optional)Path to the InstructLab Python virtual environment directory." echo -e " e.g /var/home/cloud-user/instructlab/venv \n" + echo -e "${green} --deploy TEXT${reset} (Optional)Version of UI to install." + echo -e " \"main\" for deploying latest UI" + echo -e " \"released\" for deploying the latest released version of ui. (Default: released)" exit 1 } @@ -68,7 +72,14 @@ usage_uninstall() { # Check if Podman is installed check_podman() { if ! command -v podman &>/dev/null; then - echo -e "${red}Podman is not installed!.${reset}\n" + echo -e "${red}Podman is not installed!. Podman is mandatory requirement for UI.${reset}\n" + exit 1 + fi +} + +check_git() { + if ! command -v git &>/dev/null; then + echo -e "${red}Git is not installed!. Git is mandatory requirement for UI${reset}\n" exit 1 fi } @@ -93,7 +104,7 @@ find_hostip() { # Check if the ports required by UI stacks are free check_ports() { - ports=(3000 4000 5001 8080) + ports=(3000 5001 8080) for port in "${ports[@]}"; do if lsof -i :"$port" &>/dev/null || netstat -an 2>/dev/null | grep -q ":$port "; then echo -e "${red}Warning: Port $port is required by the InstructLab UI and it's currently in use.${reset}" @@ -106,7 +117,7 @@ check_ports() { # Check if UI stack is already running check_ui_stack() { # Check if UI containers are already running - containers=("ui-pod-ui" "doclingserve-pod-doclingserve" "pathservice-pod-pathservice") + containers=("ui-pod-ui" "doclingserve-pod-doclingserve") # Check each container for container in "${containers[@]}"; do @@ -147,11 +158,11 @@ verify_user_taxonomy() { echo -e "${red}and it's not same as provided by the user - $USER_TAXONOMY_DIR${reset}\n" echo -e "${blue}It's recommended to use the taxonomy repo configured for the ilab CLI.${reset}\n" echo -e "${blue}In case you do not choose ilab configured taxonomy, data generation and fine tune features will not work.${reset}\n" - echo -e "${blue}But skill and knowledge contribution features should work as expected.${reset}\n" + echo -e "${blue}But skills and knowledge contribution features should work as expected.${reset}\n" echo -e "${red}Please choose the taxonomy repo you would like to use with InstructLab UI:${reset}" echo -e "${red}1. ilab Taxonomy - $DISCOVERED_TAXONOMY_DIR${reset}" - echo -e "${red}2. Provided Taxonomy - $USER_TAXONOMY_DIR${reset}" - read -r -p "Please choose the taxonomy repo you would like to use with InstructLab UI? (1/2): " CHOICE + echo -e "${red}2. User provided Taxonomy - $USER_TAXONOMY_DIR${reset}" + read -r -p "Use following taxonomy repo with InstructLab UI (1/2): " CHOICE if [[ "$CHOICE" == "1" ]]; then SELECTED_TAXONOMY_DIR=$DISCOVERED_TAXONOMY_DIR else @@ -166,6 +177,30 @@ verify_user_taxonomy() { SELECTED_TAXONOMY_DIR=$USER_TAXONOMY_DIR } +# Check if the selected taxonomy is empty. If not, make sure it's git initialized repository. +check_taxonomy_readiness() { + local TAXONOMY_DIR="$1" + # Check if the directory exists + if [ ! -d "$TAXONOMY_DIR" ]; then + echo -e "${red}Selected taxonomy repository directory does not exist. Taxonomy repository is a mandatory requirement for UI installation.${reset}\n" + exit 1 + fi + + # Check if the directory is empty + if [ -z "$(ls -A "$TAXONOMY_DIR" 2>/dev/null)" ]; then + echo -e "${red}Selected taxonomy repository directory is empty. Seems like taxonomy repository is not cloned?.${reset}\n" + exit 1 + fi + + # Check if the directory is a Git repository + if git -C "$TAXONOMY_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo -e "${green}Selected taxonomy repository directory is properly setup and ready for use.${reset}\n" + else + echo -e "${red}Selected taxonomy repository directory is not a git repository. Seems like taxonomy repository is not cloned?.${reset}\n" + exit 1 + fi +} + # Discover python virtual environments on host discover_pyenv() { echo -e "${green}Discovering python virtual environment present on the host...${reset}\n" @@ -215,7 +250,8 @@ discover_taxonomy_path() { taxonomy_path=$(ilab config show 2>/dev/null | grep "taxonomy_path" | head -n 1 | cut -d ':' -f 2 | sed 's/^ *//;s/ *$//' | tr -d '\r\n') else # Always use discovered python virtual environment, - venv_ilab="$DISCOVERED_PYENV_DIR/bin/ilab" + venv_ilab_dir=$([ -z "$DISCOVERED_PYENV_DIR" ] && echo "$PYENV_DIR" || echo "$DISCOVERED_PYENV_DIR") + venv_ilab="$venv_ilab_dir/bin/ilab" taxonomy_path=$($venv_ilab config show 2>/dev/null | grep "taxonomy_path" | head -n 1 | cut -d ':' -f 2 | sed 's/^ *//;s/ *$//' | tr -d '\r\n') fi else @@ -240,6 +276,30 @@ discover_ilab() { } +normalize_taxonomy_path() { + if [[ "$USER_TAXONOMY_DIR" != /* ]]; then + if ! command -v realpath &>/dev/null; then + USER_TAXONOMY_DIR=$(realpath "$USER_TAXONOMY_DIR" 2>/dev/null) + else + USER_TAXONOMY_DIR="$(cd "$(dirname "$USER_TAXONOMY_DIR")" && pwd)/$(basename "$USER_TAXONOMY_DIR")" + fi + echo -e "${blue} Taxonomy path normalized to absolute path $USER_TAXONOMY_DIR${reset}\n" + fi +} + +normalize_pyvenv_dir_path() { + echo -e "normalize_pyvenv_dir_path called- $PYENV_DIR" + + if [[ "$PYENV_DIR" != /* ]]; then + if ! command -v realpath &>/dev/null; then + PYENV_DIR=$(realpath "$PYENV_DIR" 2>/dev/null) + else + PYENV_DIR="$(cd "$(dirname "$PYENV_DIR")" && pwd)/$(basename "$PYENV_DIR")" + fi + echo -e "${blue} Python virtual environment path normalized to absolute path $PYENV_DIR${reset}\n" + fi +} + # Parse input arguments if [[ $# -lt 1 ]]; then usage @@ -261,6 +321,7 @@ if [[ "$COMMAND" == "install" ]]; then ;; --taxonomy-dir) USER_TAXONOMY_DIR=$(echo "$2" | sed 's/^ *//;s/ *$//') + normalize_taxonomy_path shift 2 ;; --auth-url) @@ -281,6 +342,11 @@ if [[ "$COMMAND" == "install" ]]; then ;; --python-venv-dir) PYENV_DIR=$(echo "$2" | sed 's/^ *//;s/ *$//') + normalize_pyvenv_dir_path + shift 2 + ;; + --deploy) + DEPLOY="$2" shift 2 ;; --help) @@ -297,6 +363,7 @@ if [[ "$COMMAND" == "install" ]]; then fi check_podman + check_git check_ports check_ui_stack @@ -305,7 +372,11 @@ if [[ "$COMMAND" == "install" ]]; then verify_user_pyenv else discover_ilab - echo -e "\n${blue}NOTE: If you are using python virtual environment for InstructLab setup, you can use --python-venv-dir option to skip the discovery.${reset}\n" + if [[ "$DISCOVERED_PYENV_DIR" == "" ]]; then + IS_ILAB_INSTALLED="false" + else + echo -e "\n${blue}NOTE: If you are using python virtual environment for InstructLab setup, you can use --python-venv-dir option to skip the discovery.${reset}\n" + fi fi # ilab is not set up and the user didn't provide that taxonomy info as well. Exit with warning info. @@ -313,8 +384,8 @@ if [[ "$COMMAND" == "install" ]]; then if [[ "$USER_TAXONOMY_DIR" == "" ]]; then echo -e "${red}Given that ilab is not set up on the host, taxonomy repo is not discovered.${reset}\n" echo -e "${red}To proceed with the installation please do one of the following:${reset}" - echo -e "${red}1. Clone the taxonomy repo and provide the path to the directory using '--taxonomy-dir' option.${reset}" - echo -e "${red}2. Set up the ilab. If using virtual environment to set up the ilab, use --python-venv-dir option to virtual environment directory.${reset}" + echo -e "${red}1. Clone the taxonomy repo and provide the absolute path to the directory using '--taxonomy-dir' option.${reset}" + echo -e "${red}2. Set up the ilab. If using virtual environment to set up the ilab, use --python-venv-dir option to provide virtual environment directory.${reset}" exit 1 fi fi @@ -343,6 +414,9 @@ if [[ "$COMMAND" == "install" ]]; then fi fi + echo -e "${green}Check readiness for taxonomy repository directory : $SELECTED_TAXONOMY_DIR${reset}\n" + check_taxonomy_readiness "$SELECTED_TAXONOMY_DIR" + echo -e "${green}Starting InstructLab UI installation...${reset}\n" echo -e "${green}InstructLab UI will be set up with taxonomy present in $SELECTED_TAXONOMY_DIR.${reset}\n" @@ -493,6 +567,21 @@ if [[ "$COMMAND" == "install" ]]; then echo -e "${green}Deploying the secrets in Podman...${reset}\n" podman kube play secret.yaml + # Replace image tags in the manifest file + if [[ "$DEPLOY" == "main" ]]; then + if [[ "$OS" == "darwin" ]]; then + sed -i "" "s||main|g" instructlab-ui.yaml + else + sed -i "s||main|g" instructlab-ui.yaml + fi + else + if [[ "$OS" == "darwin" ]]; then + sed -i "" "s||latest|g" instructlab-ui.yaml + else + sed -i "s||latest|g" instructlab-ui.yaml + fi + fi + echo -e "\n${green}Deploying the UI stack containers in Podman...${reset}\n" podman kube play instructlab-ui.yaml diff --git a/package-lock.json b/package-lock.json index 64569b8c9..c6aaf0513 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,26 +10,26 @@ "license": "Apache-2.0", "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.7.2", - "@next/env": "^15.1.6", - "@patternfly/chatbot": "^2.2.0-prerelease.19", - "@patternfly/react-core": "^6.1.0", - "@patternfly/react-icons": "^6.0.0", - "@patternfly/react-styles": "^6.0.0", - "@patternfly/react-table": "^6.1.0", + "@next/env": "^15.3.1", + "@patternfly/chatbot": "^2.2.1", + "@patternfly/react-core": "^6.2.2", + "@patternfly/react-icons": "^6.2.0", + "@patternfly/react-styles": "^6.2.0", + "@patternfly/react-table": "^6.2.2", "@patternfly/virtual-assistant": "^2.0.2", - "axios": "^1.7.9", + "axios": "^1.9.0", "date-fns": "^4.1.0", - "dompurify": "^3.2.4", + "dompurify": "^3.2.5", "fs": "^0.0.1-security", - "isomorphic-git": "^1.29.0", + "isomorphic-git": "^1.30.1", "js-yaml": "^4.1.0", - "next": "^15.1.6", + "next": "^15.3.1", "next-auth": "^4.24.11", "node-fetch": "^3.3.2", "react": "18.3.1", "react-dom": "18.3.1", - "sass": "^1.84.0", - "uuid": "^11.0.5", + "sass": "^1.87.0", + "uuid": "^11.1.0", "winston": "^3.17.0" }, "devDependencies": { @@ -50,7 +50,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.24.7", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.10.tgz", + "integrity": "sha512-2WJMeRQPHKSPemqk/awGrAiuFfzBmOIPXKizAsVhWH9YJqLZ0H+HS4c8loHGgW6utJ3E/ejXQUsiGaQy2NZ9Fw==", "license": "MIT", "dependencies": { "regenerator-runtime": "^0.14.0" @@ -75,6 +77,16 @@ "kuler": "^2.0.0" } }, + "node_modules/@emnapi/runtime": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.1.tgz", + "integrity": "sha512-LMshMVP0ZhACNjQNYXiU1iZJ6QCcv0lUdPDPugqGvCGXt5xtRVBPdtA0qU12pEXZzpWAhWlZYptfdAFq10DOVQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@emotion/is-prop-valid": { "version": "0.7.3", "license": "MIT", @@ -282,7 +294,9 @@ "license": "BSD-3-Clause" }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.5", + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.1.tgz", + "integrity": "sha512-pn44xgBtgpEbZsu+lWf2KNb6OAf70X68k+yk69Ic2Xz11zHR/w24/U49XT7AeRwJ0Px+mhALhU5LPci1Aymk7A==", "cpu": [ "arm64" ], @@ -298,11 +312,35 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.4" + "@img/sharp-libvips-darwin-arm64": "1.1.0" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.1.tgz", + "integrity": "sha512-VfuYgG2r8BpYiOUN+BfYeFo69nP/MIwAtSJ7/Zpxc5QF3KS22z8Pvg3FkrSFJBPNQ7mmcUcYQFBmEQp7eu1F8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.1.0" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.4", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.1.0.tgz", + "integrity": "sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA==", "cpu": [ "arm64" ], @@ -315,6 +353,323 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.1.0.tgz", + "integrity": "sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.1.0.tgz", + "integrity": "sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.1.0.tgz", + "integrity": "sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.1.0.tgz", + "integrity": "sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.1.0.tgz", + "integrity": "sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.1.0.tgz", + "integrity": "sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.1.0.tgz", + "integrity": "sha512-jYZdG+whg0MDK+q2COKbYidaqW/WTz0cc1E+tMAusiDygrM4ypmSCjOJPmFTvHHJ8j/6cAGyeDWZOsK06tP33w==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.1.0.tgz", + "integrity": "sha512-wK7SBdwrAiycjXdkPnGCPLjYb9lD4l6Ze2gSdAGVZrEL05AOUJESWU2lhlC+Ffn5/G+VKuSm6zzbQSzFX/P65A==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.1.tgz", + "integrity": "sha512-anKiszvACti2sGy9CirTlNyk7BjjZPiML1jt2ZkTdcvpLU1YH6CXwRAZCA2UmRXnhiIftXQ7+Oh62Ji25W72jA==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.1.0" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.1.tgz", + "integrity": "sha512-kX2c+vbvaXC6vly1RDf/IWNXxrlxLNpBVWkdpRq5Ka7OOKj6nr66etKy2IENf6FtOgklkg9ZdGpEu9kwdlcwOQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.1.0" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.1.tgz", + "integrity": "sha512-7s0KX2tI9mZI2buRipKIw2X1ufdTeaRgwmRabt5bi9chYfhur+/C1OXg3TKg/eag1W+6CCWLVmSauV1owmRPxA==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.1.0" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.1.tgz", + "integrity": "sha512-wExv7SH9nmoBW3Wr2gvQopX1k8q2g5V5Iag8Zk6AVENsjwd+3adjwxtp3Dcu2QhOXr8W9NusBU6XcQUohBZ5MA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.1.0" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.1.tgz", + "integrity": "sha512-DfvyxzHxw4WGdPiTF0SOHnm11Xv4aQexvqhRDAoD00MzHekAj9a/jADXeXYCDFH/DzYruwHbXU7uz+H+nWmSOQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.1.0" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.1.tgz", + "integrity": "sha512-pax/kTR407vNb9qaSIiWVnQplPcGU8LRIJpDT5o8PdAx5aAA7AS3X9PS8Isw1/WfqgQorPotjrZL3Pqh6C5EBg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.1.0" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.1.tgz", + "integrity": "sha512-YDybQnYrLQfEpzGOQe7OKcyLUCML4YOXl428gOOzBgN6Gw0rv8dpsJ7PqTHxBnXnwXr8S1mYFSLSa727tpz0xg==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.4.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.1.tgz", + "integrity": "sha512-WKf/NAZITnonBf3U1LfdjoMgNO5JYRSlhovhRhMxXVdvWYveM4kM3L8m35onYIdh75cOMCo1BexgVQcCDzyoWw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.1.tgz", + "integrity": "sha512-hw1iIAHpNE8q3uMIRCgGOeDoz9KtFNarFLQclLxr/LK1VBkj8nby18RjFvr6aP7USRYAjTZW6yisnBWMX571Tw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "dev": true, @@ -356,6 +711,27 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/@lukeed/csprng": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", + "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@lukeed/uuid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@lukeed/uuid/-/uuid-2.0.1.tgz", + "integrity": "sha512-qC72D4+CDdjGqJvkFMMEAtancHUQ7/d/tAiHf64z8MopFDmcrtbcJuerDtFceuAfQJ2pDSfCKCtbqoGBNnwg0w==", + "license": "MIT", + "dependencies": { + "@lukeed/csprng": "^1.1.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@monaco-editor/loader": { "version": "1.4.0", "license": "MIT", @@ -379,9 +755,9 @@ } }, "node_modules/@next/env": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/env/-/env-15.1.6.tgz", - "integrity": "sha512-d9AFQVPEYNr+aqokIiPLNK/MTyt3DWa/dpKveiAaVccUadFbhFEvY6FXYX2LJO2Hv7PHnLBu2oWwB4uBuHjr/w==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.3.1.tgz", + "integrity": "sha512-cwK27QdzrMblHSn9DZRV+DQscHXRuJv6MydlJRpFSqJWZrTYMLzKDeyueJNN9MGd8NNiUKzDQADAf+dMLXX7YQ==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -393,9 +769,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.1.6.tgz", - "integrity": "sha512-u7lg4Mpl9qWpKgy6NzEkz/w0/keEHtOybmIl0ykgItBxEM5mYotS5PmqTpo+Rhg8FiOiWgwr8USxmKQkqLBCrw==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.3.1.tgz", + "integrity": "sha512-hjDw4f4/nla+6wysBL07z52Gs55Gttp5Bsk5/8AncQLJoisvTBP0pRIBK/B16/KqQyH+uN4Ww8KkcAqJODYH3w==", "cpu": [ "arm64" ], @@ -409,9 +785,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.1.6.tgz", - "integrity": "sha512-x1jGpbHbZoZ69nRuogGL2MYPLqohlhnT9OCU6E6QFewwup+z+M6r8oU47BTeJcWsF2sdBahp5cKiAcDbwwK/lg==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.3.1.tgz", + "integrity": "sha512-q+aw+cJ2ooVYdCEqZVk+T4Ni10jF6Fo5DfpEV51OupMaV5XL6pf3GCzrk6kSSZBsMKZtVC1Zm/xaNBFpA6bJ2g==", "cpu": [ "x64" ], @@ -425,9 +801,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.1.6.tgz", - "integrity": "sha512-jar9sFw0XewXsBzPf9runGzoivajeWJUc/JkfbLTC4it9EhU8v7tCRLH7l5Y1ReTMN6zKJO0kKAGqDk8YSO2bg==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.3.1.tgz", + "integrity": "sha512-wBQ+jGUI3N0QZyWmmvRHjXjTWFy8o+zPFLSOyAyGFI94oJi+kK/LIZFJXeykvgXUk1NLDAEFDZw/NVINhdk9FQ==", "cpu": [ "arm64" ], @@ -441,9 +817,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.1.6.tgz", - "integrity": "sha512-+n3u//bfsrIaZch4cgOJ3tXCTbSxz0s6brJtU3SzLOvkJlPQMJ+eHVRi6qM2kKKKLuMY+tcau8XD9CJ1OjeSQQ==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.3.1.tgz", + "integrity": "sha512-IIxXEXRti/AulO9lWRHiCpUUR8AR/ZYLPALgiIg/9ENzMzLn3l0NSxVdva7R/VDcuSEBo0eGVCe3evSIHNz0Hg==", "cpu": [ "arm64" ], @@ -457,9 +833,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.1.6.tgz", - "integrity": "sha512-SpuDEXixM3PycniL4iVCLyUyvcl6Lt0mtv3am08sucskpG0tYkW1KlRhTgj4LI5ehyxriVVcfdoxuuP8csi3kQ==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.3.1.tgz", + "integrity": "sha512-bfI4AMhySJbyXQIKH5rmLJ5/BP7bPwuxauTvVEiJ/ADoddaA9fgyNNCcsbu9SlqfHDoZmfI6g2EjzLwbsVTr5A==", "cpu": [ "x64" ], @@ -473,9 +849,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.1.6.tgz", - "integrity": "sha512-L4druWmdFSZIIRhF+G60API5sFB7suTbDRhYWSjiw0RbE+15igQvE2g2+S973pMGvwN3guw7cJUjA/TmbPWTHQ==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.3.1.tgz", + "integrity": "sha512-FeAbR7FYMWR+Z+M5iSGytVryKHiAsc0x3Nc3J+FD5NVbD5Mqz7fTSy8CYliXinn7T26nDMbpExRUI/4ekTvoiA==", "cpu": [ "x64" ], @@ -489,9 +865,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.1.6.tgz", - "integrity": "sha512-s8w6EeqNmi6gdvM19tqKKWbCyOBvXFbndkGHl+c9YrzsLARRdCHsD9S1fMj8gsXm9v8vhC8s3N8rjuC/XrtkEg==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.3.1.tgz", + "integrity": "sha512-yP7FueWjphQEPpJQ2oKmshk/ppOt+0/bB8JC8svPUZNy0Pi3KbPx2Llkzv1p8CoQa+D2wknINlJpHf3vtChVBw==", "cpu": [ "arm64" ], @@ -505,9 +881,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.1.6.tgz", - "integrity": "sha512-6xomMuu54FAFxttYr5PJbEfu96godcxBTRk1OhAvJq0/EnmFU/Ybiax30Snis4vdWZ9LGpf7Roy5fSs7v/5ROQ==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.3.1.tgz", + "integrity": "sha512-3PMvF2zRJAifcRNni9uMk/gulWfWS+qVI/pagd+4yLF5bcXPZPPH2xlYRYOsUjmCJOXSTAC2PjRzbhsRzR2fDQ==", "cpu": [ "x64" ], @@ -623,18 +999,25 @@ } }, "node_modules/@patternfly/chatbot": { - "version": "2.2.0-prerelease.19", - "resolved": "https://registry.npmjs.org/@patternfly/chatbot/-/chatbot-2.2.0-prerelease.19.tgz", - "integrity": "sha512-jlEmuQE+ZRvZPeYpIcrFlQEcAMKnC9y3O9AxjYrfxucFiTcn6VitMcv/TYioCfFph+Q1BPi30vhwBn2SaMzXVw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@patternfly/chatbot/-/chatbot-2.2.1.tgz", + "integrity": "sha512-lVIUADO1l1O6+vQk1MDh3gbodwX8CxnyANlWFlY08FJ11XOt0p1Bec4AUnO0H8Cjk0cTHF6aV93MDF9NpjFCZQ==", + "license": "MIT", "dependencies": { "@patternfly/react-code-editor": "^6.1.0", "@patternfly/react-core": "^6.1.0", "@patternfly/react-icons": "^6.1.0", + "@patternfly/react-table": "^6.1.0", + "@segment/analytics-next": "^1.76.0", "clsx": "^2.1.0", "framer-motion": "^11.3.28", "path-browserify": "^1.0.1", + "posthog-js": "^1.194.4", "react-markdown": "^9.0.1", "react-syntax-highlighter": "^15.5.0", + "rehype-external-links": "^3.0.0", + "rehype-sanitize": "^6.0.0", + "rehype-unwrap-images": "^1.0.0", "remark-gfm": "^4.0.0" }, "peerDependencies": { @@ -659,13 +1042,15 @@ } }, "node_modules/@patternfly/react-core": { - "version": "6.1.0", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/@patternfly/react-core/-/react-core-6.2.2.tgz", + "integrity": "sha512-JUrZ57JQ4bkmed1kxaciXb0ZpIVYyCHc2HjtzoKQ5UNRlx204zR2isATSHjdw2GFcWvwpkC5/fU2BR+oT3opbg==", "license": "MIT", "dependencies": { - "@patternfly/react-icons": "^6.1.0", - "@patternfly/react-styles": "^6.1.0", - "@patternfly/react-tokens": "^6.1.0", - "focus-trap": "7.6.2", + "@patternfly/react-icons": "^6.2.2", + "@patternfly/react-styles": "^6.2.2", + "@patternfly/react-tokens": "^6.2.2", + "focus-trap": "7.6.4", "react-dropzone": "^14.3.5", "tslib": "^2.8.1" }, @@ -675,7 +1060,9 @@ } }, "node_modules/@patternfly/react-icons": { - "version": "6.1.0", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/@patternfly/react-icons/-/react-icons-6.2.2.tgz", + "integrity": "sha512-XkBwzuV/uiolX+T6QgB3RIqphM1m+vAZjAe3McYtyY22j1rsOdlWDE4RtRrJ1q7EoIZwyZHj0h8T9vMfUsLn4Q==", "license": "MIT", "peerDependencies": { "react": "^17 || ^18", @@ -683,17 +1070,21 @@ } }, "node_modules/@patternfly/react-styles": { - "version": "6.1.0", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/@patternfly/react-styles/-/react-styles-6.2.2.tgz", + "integrity": "sha512-rncRDq66H8VnLyb9DrHHlZtPddlpNL9+W0XuQC0L7F6p78hOwSZmoGTW2Vq8/wJplDj8h/61qRpfRF9VEYPW0g==", "license": "MIT" }, "node_modules/@patternfly/react-table": { - "version": "6.1.0", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/@patternfly/react-table/-/react-table-6.2.2.tgz", + "integrity": "sha512-7CxVKhnpA+f8dLJ0hVvzZOe4Djx/nE+w70ipeAHf4Yi5JwfDWmbK97YvjYPfamp/bsXTLtPcK2n4AoY5DQX6Pg==", "license": "MIT", "dependencies": { - "@patternfly/react-core": "^6.1.0", - "@patternfly/react-icons": "^6.1.0", - "@patternfly/react-styles": "^6.1.0", - "@patternfly/react-tokens": "^6.1.0", + "@patternfly/react-core": "^6.2.2", + "@patternfly/react-icons": "^6.2.2", + "@patternfly/react-styles": "^6.2.2", + "@patternfly/react-tokens": "^6.2.2", "lodash": "^4.17.21", "tslib": "^2.8.1" }, @@ -703,7 +1094,9 @@ } }, "node_modules/@patternfly/react-tokens": { - "version": "6.1.0", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/@patternfly/react-tokens/-/react-tokens-6.2.2.tgz", + "integrity": "sha512-2GRWDPBTrcTlGNFc5NPJjrjEVU90RpgcGX/CIe2MplLgM32tpVIkeUtqIoJPLRk5GrbhyFuHJYRU+O93gU4o3Q==", "license": "MIT" }, "node_modules/@patternfly/virtual-assistant": { @@ -750,11 +1143,122 @@ "node": ">=18" } }, + "node_modules/@rrweb/types": { + "version": "2.0.0-alpha.17", + "resolved": "https://registry.npmjs.org/@rrweb/types/-/types-2.0.0-alpha.17.tgz", + "integrity": "sha512-AfDTVUuCyCaIG0lTSqYtrZqJX39ZEYzs4fYKnexhQ+id+kbZIpIJtaut5cto6dWZbB3SEe4fW0o90Po3LvTmfg==", + "license": "MIT", + "peer": true, + "dependencies": { + "rrweb-snapshot": "^2.0.0-alpha.17" + } + }, "node_modules/@rushstack/eslint-patch": { "version": "1.10.3", "dev": true, "license": "MIT" }, + "node_modules/@segment/analytics-core": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@segment/analytics-core/-/analytics-core-1.8.1.tgz", + "integrity": "sha512-EYcdBdhfi1pOYRX+Sf5orpzzYYFmDHTEu6+w0hjXpW5bWkWct+Nv6UJg1hF4sGDKEQjpZIinLTpQ4eioFM4KeQ==", + "license": "MIT", + "dependencies": { + "@lukeed/uuid": "^2.0.0", + "@segment/analytics-generic-utils": "1.2.0", + "dset": "^3.1.4", + "tslib": "^2.4.1" + } + }, + "node_modules/@segment/analytics-generic-utils": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@segment/analytics-generic-utils/-/analytics-generic-utils-1.2.0.tgz", + "integrity": "sha512-DfnW6mW3YQOLlDQQdR89k4EqfHb0g/3XvBXkovH1FstUN93eL1kfW9CsDcVQyH3bAC5ZsFyjA/o/1Q2j0QeoWw==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.1" + } + }, + "node_modules/@segment/analytics-next": { + "version": "1.77.0", + "resolved": "https://registry.npmjs.org/@segment/analytics-next/-/analytics-next-1.77.0.tgz", + "integrity": "sha512-xZiSz8iOPoiYt0K0w3fMG5khaPzeLuALfQYUvle6rfNnhjs8N0WiV1Vx/EwgtJPPw3saol5n4oW8xhcGWeQAIA==", + "license": "MIT", + "dependencies": { + "@lukeed/uuid": "^2.0.0", + "@segment/analytics-core": "1.8.1", + "@segment/analytics-generic-utils": "1.2.0", + "@segment/analytics.js-video-plugins": "^0.2.1", + "@segment/facade": "^3.4.9", + "dset": "^3.1.4", + "js-cookie": "3.0.1", + "node-fetch": "^2.6.7", + "tslib": "^2.4.1", + "unfetch": "^4.1.0" + } + }, + "node_modules/@segment/analytics-next/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/@segment/analytics.js-video-plugins": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@segment/analytics.js-video-plugins/-/analytics.js-video-plugins-0.2.1.tgz", + "integrity": "sha512-lZwCyEXT4aaHBLNK433okEKdxGAuyrVmop4BpQqQSJuRz0DglPZgd9B/XjiiWs1UyOankg2aNYMN3VcS8t4eSQ==", + "license": "ISC", + "dependencies": { + "unfetch": "^3.1.1" + } + }, + "node_modules/@segment/analytics.js-video-plugins/node_modules/unfetch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/unfetch/-/unfetch-3.1.2.tgz", + "integrity": "sha512-L0qrK7ZeAudGiKYw6nzFjnJ2D5WHblUBwmHIqtPS6oKUd+Hcpk7/hKsSmcHsTlpd1TbTNsiRBUKRq3bHLNIqIw==", + "license": "MIT" + }, + "node_modules/@segment/facade": { + "version": "3.4.10", + "resolved": "https://registry.npmjs.org/@segment/facade/-/facade-3.4.10.tgz", + "integrity": "sha512-xVQBbB/lNvk/u8+ey0kC/+g8pT3l0gCT8O2y9Z+StMMn3KAFAQ9w8xfgef67tJybktOKKU7pQGRPolRM1i1pdA==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "@segment/isodate-traverse": "^1.1.1", + "inherits": "^2.0.4", + "new-date": "^1.0.3", + "obj-case": "0.2.1" + } + }, + "node_modules/@segment/isodate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@segment/isodate/-/isodate-1.0.3.tgz", + "integrity": "sha512-BtanDuvJqnACFkeeYje7pWULVv8RgZaqKHWwGFnL/g/TH/CcZjkIVTfGDp/MAxmilYHUkrX70SqwnYSTNEaN7A==", + "license": "SEE LICENSE IN LICENSE" + }, + "node_modules/@segment/isodate-traverse": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@segment/isodate-traverse/-/isodate-traverse-1.1.1.tgz", + "integrity": "sha512-+G6e1SgAUkcq0EDMi+SRLfT48TNlLPF3QnSgFGVs0V9F3o3fq/woQ2rHFlW20W0yy5NnCUH0QGU3Am2rZy/E3w==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "@segment/isodate": "^1.0.3" + } + }, "node_modules/@swc/counter": { "version": "0.1.3", "license": "Apache-2.0" @@ -1315,7 +1819,9 @@ } }, "node_modules/axios": { - "version": "1.7.9", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.9.0.tgz", + "integrity": "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", @@ -1549,6 +2055,17 @@ "node": ">= 0.6" } }, + "node_modules/core-js": { + "version": "3.41.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.41.0.tgz", + "integrity": "sha512-SJ4/EHwS36QMJd6h/Rg+GyR4A5xE0FSI3eZ+iBVpfqf1x0eTSg1smWLHrA+2jQThZSh97fmSgFSU8B61nxosxA==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/crc-32": { "version": "1.2.2", "license": "Apache-2.0", @@ -1786,6 +2303,8 @@ }, "node_modules/detect-libc": { "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", "license": "Apache-2.0", "optional": true, "engines": { @@ -1830,14 +2349,23 @@ } }, "node_modules/dompurify": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.4.tgz", - "integrity": "sha512-ysFSFEDVduQpyhzAob/kkuJjf5zWkZD8/A9ywSp1byueyuCfHamrCBa14/Oc2iiB0e51B+NpxSl5gmzn+Ms/mg==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.5.tgz", + "integrity": "sha512-mLPd29uoRe9HpvwP2TxClGQBzGXeEC/we/q+bFlmPPmj2p2Ugl3r6ATu/UU1v77DXNcehiBg9zsr1dREyA/dJQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, + "node_modules/dset": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "dev": true, @@ -2826,6 +3354,12 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fflate": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz", + "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "6.0.1", "dev": true, @@ -2896,7 +3430,9 @@ "license": "MIT" }, "node_modules/focus-trap": { - "version": "7.6.2", + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.6.4.tgz", + "integrity": "sha512-xx560wGBk7seZ6y933idtjJQc1l+ck+pI3sKvhKozdBV1dRZoKhkW5xoCaFv9tQiX5RH1xfSxjuNu6g+lmN/gw==", "license": "MIT", "dependencies": { "tabbable": "^6.2.0" @@ -3248,6 +3784,46 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-has-property": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-has-property/-/hast-util-has-property-3.0.0.tgz", + "integrity": "sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-interactive": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-interactive/-/hast-util-interactive-3.0.0.tgz", + "integrity": "sha512-9VFa3kP6AT40BNYcPmn3jpsG+1KPDF0rUFCrFVQDUsuUXZ3YLODm8UGV0tmYzFpcOIQXTAOi2ccS3ywlj2dQTA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-has-property": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-element": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz", + "integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-parse-selector": { "version": "2.2.5", "license": "MIT", @@ -3256,6 +3832,21 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-sanitize": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/hast-util-sanitize/-/hast-util-sanitize-5.0.2.tgz", + "integrity": "sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "unist-util-position": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-to-jsx-runtime": { "version": "2.3.2", "license": "MIT", @@ -3439,6 +4030,18 @@ "node": ">= 0.4" } }, + "node_modules/is-absolute-url": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-4.0.1.tgz", + "integrity": "sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-alphabetical": { "version": "2.0.1", "license": "MIT", @@ -3860,7 +4463,9 @@ "license": "ISC" }, "node_modules/isomorphic-git": { - "version": "1.29.0", + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/isomorphic-git/-/isomorphic-git-1.30.1.tgz", + "integrity": "sha512-eWBlPIPDOctGY/bTUc/whs6EZ8YvnG1H2kOjTCJ/AkvBWUzODXcfulhpiA8Y4Px9e+bRYYkifE5fSE8FcRk8Ew==", "license": "MIT", "dependencies": { "async-lock": "^1.4.1", @@ -3880,7 +4485,7 @@ "isogit": "cli.cjs" }, "engines": { - "node": ">=12" + "node": ">=14.17" } }, "node_modules/iterator.prototype": { @@ -3919,6 +4524,15 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-cookie": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.1.tgz", + "integrity": "sha512-+0rgsUXZu4ncpPxRL+lNEptWMOWl9etvPHc/koSRp6MPwpRYAhmk0dUG00J4bxVV3r9uUzfo24wW0knS07SKSw==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "license": "MIT" @@ -5087,13 +5701,22 @@ "dev": true, "license": "MIT" }, + "node_modules/new-date": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/new-date/-/new-date-1.0.3.tgz", + "integrity": "sha512-0fsVvQPbo2I18DT2zVHpezmeeNYV2JaJSrseiHLc17GNOxJzUdx5mvSigPu8LtIfZSij5i1wXnXFspEs2CD6hA==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "@segment/isodate": "1.0.3" + } + }, "node_modules/next": { - "version": "15.1.6", - "resolved": "https://registry.npmjs.org/next/-/next-15.1.6.tgz", - "integrity": "sha512-Hch4wzbaX0vKQtalpXvUiw5sYivBy4cm5rzUKrBnUB/y436LGrvOUqYvlSeNVCWFO/770gDlltR9gqZH62ct4Q==", + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/next/-/next-15.3.1.tgz", + "integrity": "sha512-8+dDV0xNLOgHlyBxP1GwHGVaNXsmp+2NhZEYrXr24GWLHtt27YrBPbPuHvzlhi7kZNYjeJNR93IF5zfFu5UL0g==", "license": "MIT", "dependencies": { - "@next/env": "15.1.6", + "@next/env": "15.3.1", "@swc/counter": "0.1.3", "@swc/helpers": "0.5.15", "busboy": "1.6.0", @@ -5108,15 +5731,15 @@ "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "15.1.6", - "@next/swc-darwin-x64": "15.1.6", - "@next/swc-linux-arm64-gnu": "15.1.6", - "@next/swc-linux-arm64-musl": "15.1.6", - "@next/swc-linux-x64-gnu": "15.1.6", - "@next/swc-linux-x64-musl": "15.1.6", - "@next/swc-win32-arm64-msvc": "15.1.6", - "@next/swc-win32-x64-msvc": "15.1.6", - "sharp": "^0.33.5" + "@next/swc-darwin-arm64": "15.3.1", + "@next/swc-darwin-x64": "15.3.1", + "@next/swc-linux-arm64-gnu": "15.3.1", + "@next/swc-linux-arm64-musl": "15.3.1", + "@next/swc-linux-x64-gnu": "15.3.1", + "@next/swc-linux-x64-musl": "15.3.1", + "@next/swc-win32-arm64-msvc": "15.3.1", + "@next/swc-win32-x64-msvc": "15.3.1", + "sharp": "^0.34.1" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -5246,6 +5869,12 @@ "version": "0.9.15", "license": "MIT" }, + "node_modules/obj-case": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/obj-case/-/obj-case-0.2.1.tgz", + "integrity": "sha512-PquYBBTy+Y6Ob/O2574XHhDtHJlV1cJHMCgW+rDRc9J5hhmRelJB3k5dTK/3cVmFVtzvAKuENeuLpoyTzMzkOg==", + "license": "MIT" + }, "node_modules/object-assign": { "version": "4.1.1", "license": "MIT", @@ -5633,6 +6262,50 @@ "node": ">= 0.4" } }, + "node_modules/postcss": { + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", + "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "nanoid": "^3.3.8", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/posthog-js": { + "version": "1.225.1", + "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.225.1.tgz", + "integrity": "sha512-JO12aGlKbznD91osyNPgsQXP4jwLYdDtGrEL6idjGBb4ETFpud6tfe2Jdaow3672c3fNjFFNyu2ybXC3793VOA==", + "license": "MIT", + "dependencies": { + "core-js": "^3.38.1", + "fflate": "^0.4.8", + "preact": "^10.19.3", + "web-vitals": "^4.2.0" + }, + "peerDependencies": { + "@rrweb/types": "2.0.0-alpha.17" + } + }, "node_modules/preact": { "version": "10.22.1", "license": "MIT", @@ -6002,6 +6675,54 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/rehype-external-links": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/rehype-external-links/-/rehype-external-links-3.0.0.tgz", + "integrity": "sha512-yp+e5N9V3C6bwBeAC4n796kc86M4gJCdlVhiMTxIrJG5UHDMh+PJANf9heqORJbt1nrCbDwIlAZKjANIaVBbvw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-is-element": "^3.0.0", + "is-absolute-url": "^4.0.0", + "space-separated-tokens": "^2.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-sanitize": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/rehype-sanitize/-/rehype-sanitize-6.0.0.tgz", + "integrity": "sha512-CsnhKNsyI8Tub6L4sm5ZFsme4puGfc6pYylvXo1AeqaGbjOYyzNv3qZPwvs0oMJ39eryyeOdmxwUIo94IpEhqg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-sanitize": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-unwrap-images": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/rehype-unwrap-images/-/rehype-unwrap-images-1.0.0.tgz", + "integrity": "sha512-wzW5Mk9IlVF2UwXC5NtIZsx1aHYbV8+bLWjJnlZaaamz5QU52RppWtq1uEZJqGo8d9Y4RuDqidB6r9RFpKugIg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-interactive": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-gfm": { "version": "4.0.0", "license": "MIT", @@ -6154,6 +6875,16 @@ "node": "*" } }, + "node_modules/rrweb-snapshot": { + "version": "2.0.0-alpha.18", + "resolved": "https://registry.npmjs.org/rrweb-snapshot/-/rrweb-snapshot-2.0.0-alpha.18.tgz", + "integrity": "sha512-hBHZL/NfgQX6wO1D9mpwqFu1NJPpim+moIcKhFEjVTZVRUfCln+LOugRc4teVTCISYHN8Cw5e2iNTWCSm+SkoA==", + "license": "MIT", + "peer": true, + "dependencies": { + "postcss": "^8.4.38" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "dev": true, @@ -6235,9 +6966,9 @@ } }, "node_modules/sass": { - "version": "1.84.0", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.84.0.tgz", - "integrity": "sha512-XDAbhEPJRxi7H0SxrnOpiXFQoUJHwkR2u3Zc4el+fK/Tt5Hpzw5kkQ59qVDfvdaUq6gCrEZIbySFBM2T9DNKHg==", + "version": "1.87.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.87.0.tgz", + "integrity": "sha512-d0NoFH4v6SjEK7BoX810Jsrhj7IQSYHAHLi/iSpgqKc7LaIDshFRlSg5LOymf9FqQhxEHs2W5ZQXlvy0KD45Uw==", "license": "MIT", "dependencies": { "chokidar": "^4.0.0", @@ -6315,14 +7046,16 @@ "license": "MIT" }, "node_modules/sharp": { - "version": "0.33.5", + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.1.tgz", + "integrity": "sha512-1j0w61+eVxu7DawFJtnfYcvSv6qPFvfTaqzTQ2BLknVhHTwGS8sc63ZBF4rzkWMBVKybo4S5OBtDdZahh2A1xg==", "hasInstallScript": true, "license": "Apache-2.0", "optional": true, "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", - "semver": "^7.6.3" + "semver": "^7.7.1" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -6331,29 +7064,32 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.5", - "@img/sharp-darwin-x64": "0.33.5", - "@img/sharp-libvips-darwin-arm64": "1.0.4", - "@img/sharp-libvips-darwin-x64": "1.0.4", - "@img/sharp-libvips-linux-arm": "1.0.5", - "@img/sharp-libvips-linux-arm64": "1.0.4", - "@img/sharp-libvips-linux-s390x": "1.0.4", - "@img/sharp-libvips-linux-x64": "1.0.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", - "@img/sharp-libvips-linuxmusl-x64": "1.0.4", - "@img/sharp-linux-arm": "0.33.5", - "@img/sharp-linux-arm64": "0.33.5", - "@img/sharp-linux-s390x": "0.33.5", - "@img/sharp-linux-x64": "0.33.5", - "@img/sharp-linuxmusl-arm64": "0.33.5", - "@img/sharp-linuxmusl-x64": "0.33.5", - "@img/sharp-wasm32": "0.33.5", - "@img/sharp-win32-ia32": "0.33.5", - "@img/sharp-win32-x64": "0.33.5" + "@img/sharp-darwin-arm64": "0.34.1", + "@img/sharp-darwin-x64": "0.34.1", + "@img/sharp-libvips-darwin-arm64": "1.1.0", + "@img/sharp-libvips-darwin-x64": "1.1.0", + "@img/sharp-libvips-linux-arm": "1.1.0", + "@img/sharp-libvips-linux-arm64": "1.1.0", + "@img/sharp-libvips-linux-ppc64": "1.1.0", + "@img/sharp-libvips-linux-s390x": "1.1.0", + "@img/sharp-libvips-linux-x64": "1.1.0", + "@img/sharp-libvips-linuxmusl-arm64": "1.1.0", + "@img/sharp-libvips-linuxmusl-x64": "1.1.0", + "@img/sharp-linux-arm": "0.34.1", + "@img/sharp-linux-arm64": "0.34.1", + "@img/sharp-linux-s390x": "0.34.1", + "@img/sharp-linux-x64": "0.34.1", + "@img/sharp-linuxmusl-arm64": "0.34.1", + "@img/sharp-linuxmusl-x64": "0.34.1", + "@img/sharp-wasm32": "0.34.1", + "@img/sharp-win32-ia32": "0.34.1", + "@img/sharp-win32-x64": "0.34.1" } }, "node_modules/sharp/node_modules/color": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", "license": "MIT", "optional": true, "dependencies": { @@ -6366,6 +7102,8 @@ }, "node_modules/sharp/node_modules/color-convert": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "license": "MIT", "optional": true, "dependencies": { @@ -6377,11 +7115,15 @@ }, "node_modules/sharp/node_modules/color-name": { "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "license": "MIT", "optional": true }, "node_modules/sharp/node_modules/semver": { - "version": "7.6.3", + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", "license": "ISC", "optional": true, "bin": { @@ -6495,7 +7237,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.0", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" @@ -6786,6 +7530,8 @@ }, "node_modules/tabbable": { "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz", + "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==", "license": "MIT" }, "node_modules/tapable": { @@ -6836,6 +7582,12 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/trim-lines": { "version": "3.0.1", "license": "MIT", @@ -7018,6 +7770,12 @@ "dev": true, "license": "MIT" }, + "node_modules/unfetch": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/unfetch/-/unfetch-4.2.0.tgz", + "integrity": "sha512-F9p7yYCn6cIW9El1zi0HI6vqpeIvBsr3dSuRO6Xuppb1u5rXpCPmMvLSyECLhybr9isec8Ohl0hPekMVrEinDA==", + "license": "MIT" + }, "node_modules/unified": { "version": "11.0.5", "license": "MIT", @@ -7106,7 +7864,9 @@ "license": "MIT" }, "node_modules/uuid": { - "version": "11.0.5", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -7147,6 +7907,28 @@ "node": ">= 8" } }, + "node_modules/web-vitals": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-4.2.4.tgz", + "integrity": "sha512-r4DIlprAGwJ7YM11VZp4R884m0Vmgr6EAKe3P+kO0PPj3Unqyvv59rczf6UiGcb9Z8QxZVcqKNwv/g0WNdWwsw==", + "license": "Apache-2.0" + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "dev": true, diff --git a/package.json b/package.json index 3556b2ef4..c4f8b0590 100644 --- a/package.json +++ b/package.json @@ -18,26 +18,26 @@ }, "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.7.2", - "@next/env": "^15.1.6", - "@patternfly/chatbot": "^2.2.0-prerelease.19", - "@patternfly/react-core": "^6.1.0", - "@patternfly/react-icons": "^6.0.0", - "@patternfly/react-styles": "^6.0.0", - "axios": "^1.7.9", - "@patternfly/react-table": "^6.1.0", + "@next/env": "^15.3.1", + "@patternfly/chatbot": "^2.2.1", + "@patternfly/react-core": "^6.2.2", + "@patternfly/react-icons": "^6.2.0", + "@patternfly/react-styles": "^6.2.0", + "@patternfly/react-table": "^6.2.2", "@patternfly/virtual-assistant": "^2.0.2", + "axios": "^1.9.0", "date-fns": "^4.1.0", - "dompurify": "^3.2.4", + "dompurify": "^3.2.5", "fs": "^0.0.1-security", - "isomorphic-git": "^1.29.0", + "isomorphic-git": "^1.30.1", "js-yaml": "^4.1.0", - "next": "^15.1.6", + "next": "^15.3.1", "next-auth": "^4.24.11", "node-fetch": "^3.3.2", "react": "18.3.1", "react-dom": "18.3.1", - "sass": "^1.84.0", - "uuid": "^11.0.5", + "sass": "^1.87.0", + "uuid": "^11.1.0", "winston": "^3.17.0" }, "devDependencies": { diff --git a/pathservice/Containerfile b/pathservice/Containerfile deleted file mode 100644 index ee7c653a8..000000000 --- a/pathservice/Containerfile +++ /dev/null @@ -1,23 +0,0 @@ -ARG BUILDER_IMAGE="registry.access.redhat.com/ubi9/go-toolset:1.22.5-1730550521" -ARG BASE_IMAGE="registry.access.redhat.com/ubi9-micro:9.5-1731934928" - -FROM ${BUILDER_IMAGE} AS builder - -WORKDIR /opt/app-root/src - -COPY ./pathservice ./ - -# Build the Go app -RUN go build -o bin/pathservice - -# Start a new stage from scratch -FROM ${BASE_IMAGE} - -# Copy the binary from the previous stage -COPY --from=builder /opt/app-root/src/bin/pathservice /pathservice - -# Expose the port 4000 to the outside world -EXPOSE 4000 - -# Command to run the executable -CMD ["/pathservice"] diff --git a/pathservice/cmd/pathservice.go b/pathservice/cmd/pathservice.go deleted file mode 100644 index a2a4c4ce9..000000000 --- a/pathservice/cmd/pathservice.go +++ /dev/null @@ -1,306 +0,0 @@ -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "log" - "net/http" - "os" - "os/signal" - "path/filepath" - "sync" - "syscall" - "time" - - "github.com/spf13/cobra" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - git "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing" - githttp "github.com/go-git/go-git/v5/plumbing/transport/http" -) - -const ( - repoURL = "https://github.com/instructlab/taxonomy" - repoDir = "/tmp/taxonomy" - checkInterval = 1 * time.Minute // Interval for checking updates - serviceLogLevel = "IL_UI_DEPLOYMENT" - SKILLS = "compositional_skills/" // Currently, we are only supporting compositional skills - KNOWLEDGE = "knowledge/" -) - -type PathService struct { - ctx context.Context - logger *zap.SugaredLogger - wg *sync.WaitGroup - httpServer *http.Server -} - -func NewPathService(ctx context.Context, logger *zap.SugaredLogger) *PathService { - return &PathService{ - ctx: ctx, - logger: logger, - } - -} - -func (ps *PathService) cloneRepo() error { - // check if the repo directory exists - if _, err := os.Stat(repoDir); err == nil { - ps.logger.Errorf("Repository already exists at %s, skip cloning", repoDir) - return nil - } - // Clone options - cloneOptions := &git.CloneOptions{ - URL: repoURL, - Auth: &githttp.BasicAuth{ - Username: "", - Password: "", - }, - Progress: os.Stdout, - } - - cloneOptions.InsecureSkipTLS = true - - _, err := git.PlainClone(repoDir, false, cloneOptions) - return err -} - -func (ps *PathService) deleteRepo() error { - return os.RemoveAll(repoDir) -} - -func (ps *PathService) getRemoteHeadHash() (plumbing.Hash, error) { - rem := git.NewRemote(nil, &config.RemoteConfig{ - Name: "origin", - URLs: []string{repoURL}, - }) - refs, err := rem.List(&git.ListOptions{}) - if err != nil { - return plumbing.Hash{}, err - } - - for _, ref := range refs { - if ref.Name().IsBranch() && ref.Name().Short() == "main" { - return ref.Hash(), nil - } - } - return plumbing.Hash{}, fmt.Errorf("main branch not found") -} - -func (ps *PathService) getLocalHeadHash() (plumbing.Hash, error) { - repo, err := git.PlainOpen(repoDir) - if err != nil { - return plumbing.Hash{}, err - } - - ref, err := repo.Head() - if err != nil { - return plumbing.Hash{}, err - } - return ref.Hash(), nil -} - -func (ps *PathService) checkForUpdates(ctx context.Context, wg *sync.WaitGroup, logger *zap.SugaredLogger) { - - wg.Add(1) - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - startTime := time.Now() - for { - select { - case <-ctx.Done(): - ps.logger.Infof("Shutting down the repo syncer...") - wg.Done() - return - case t := <-ticker.C: - if time.Since(startTime) < checkInterval { - continue - } - startTime = t - logger.Debugf("Syncing with upstream taxonomy repository...") - remoteHash, err := ps.getRemoteHeadHash() - if err != nil { - logger.Errorf("Failed to get remote head hash: %v", err) - continue - } - - localHash, err := ps.getLocalHeadHash() - if err != nil { - logger.Errorf("Failed to get local head hash: %v", err) - continue - } - - if remoteHash != localHash { - logger.Infof("New changes detected, updating repository...") - err = ps.deleteRepo() - if err != nil { - logger.Errorf("Failed to delete repository: %v", err) - continue - } - - err = ps.cloneRepo() - if err != nil { - logger.Errorf("Failed to clone repository: %v", err) - continue - } - - logger.Infof("Repository updated successfully.") - } else { - logger.Debugf("No new changes detected.") - } - } - - } - -} - -func (ps *PathService) skillPathHandler(w http.ResponseWriter, r *http.Request){ - ps.pathHandler(w, r, SKILLS) -} - -func (ps *PathService) knowledgePathHandler(w http.ResponseWriter, r *http.Request){ - ps.pathHandler(w, r, KNOWLEDGE) -} - -func (ps *PathService) pathHandler(w http.ResponseWriter, r *http.Request, rootDir string) { - dirName := r.URL.Query().Get("dir_name") - - dirName = rootDir + dirName - var subDirs []string - dirPath := filepath.Join(repoDir, dirName) - entries, err := os.ReadDir(dirPath) - if err != nil { - http.Error(w, "Directory path doesn't exist", http.StatusInternalServerError) - return - } - - for _, entry := range entries { - if entry.IsDir() { - subDirs = append(subDirs, entry.Name()) - } - } - response, err := json.Marshal(subDirs) - if err != nil { - http.Error(w, "Error creating response", http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Write(response) -} - -func (ps *PathService) Start() { - ctx, cancel := signal.NotifyContext(ps.ctx, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGINT) - defer cancel() - wg := &sync.WaitGroup{} - ps.wg = wg - - // Clone the repository - err := ps.cloneRepo() - if err != nil { - ps.logger.Errorf("Failed to clone the repository: %v", err) - } - - // Start periodic update check in a separate goroutine - go ps.checkForUpdates(ctx, wg, ps.logger) - - // Setup HTTP server - httpMux := http.NewServeMux() - httpMux.HandleFunc("/tree/skills", ps.skillPathHandler) - httpMux.HandleFunc("/tree/knowledge", ps.knowledgePathHandler) - httpServer := &http.Server{ - Addr: ":4000", - Handler: httpMux, - ErrorLog: log.Default(), - ReadTimeout: 30 * time.Second, - // Crank up WriteTimeout a bit more than usually - // necessary just so we can do long CPU profiles - // and not hit net/http/pprof's "profile - // duration exceeds server's WriteTimeout". - WriteTimeout: 5 * time.Minute, - } - ps.httpServer = httpServer - - wg.Add(1) - defer wg.Done() - ps.logger.Infof("Server listening on port %s", httpServer.Addr) - err = httpServer.ListenAndServe() - if err != nil { - if err != http.ErrServerClosed { - ps.logger.Fatalf("Failed to start http service %v", err) - } - } - <-ctx.Done() -} - -func (ps *PathService) Stop() { - if ps.httpServer != nil { - ps.wg.Add(1) - defer ps.wg.Done() - shutdownHttpCtx, _ := context.WithTimeout(ps.ctx, 1*time.Second) - err := ps.httpServer.Shutdown(shutdownHttpCtx) - if err != nil { - ps.logger.Errorf("Failed to shutdown http server: %v", err) - return - } - ps.logger.Infof("Http server stopped successfully") - } -} - -func (ps *PathService) WaitForGracefulShutdown() { - ps.wg.Wait() - ps.logger.Infof("Path service stopped successfully") -} - -func Execute() { - debug := os.Getenv(serviceLogLevel) - var logger *zap.Logger - var err error - if debug != "" { - logCfg := zap.NewDevelopmentConfig() - logger, err = logCfg.Build() - logger.Info("Debug logging enabled") - } else { - logCfg := zap.NewProductionConfig() - logCfg.DisableStacktrace = true - logCfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - logger, err = logCfg.Build() - } - if err != nil { - logger.Fatal(err.Error()) - } - - var rootCmd = &cobra.Command{ - Use: "pathservice", - Short: "Path service for taxonomy tree", - Run: func(cmd *cobra.Command, args []string) { - pathService := NewPathService(cmd.Context(), logger.Sugar()) - - sigchan := make(chan os.Signal, 1) - signal.Notify( - sigchan, - syscall.SIGINT, - syscall.SIGTERM, - syscall.SIGQUIT, - ) - go func(pathService *PathService) { - <-sigchan - pathService.Stop() - }(pathService) - - pathService.Start() - pathService.WaitForGracefulShutdown() - }, - } - - rootCmd.PersistentFlags().StringP("version", "v", "1.0.0", "Version of the taxonomy path service") - - if err := rootCmd.Execute(); err != nil { - logger.Error(err.Error()) - os.Exit(1) - } -} diff --git a/pathservice/go.mod b/pathservice/go.mod deleted file mode 100644 index fa7cd97e1..000000000 --- a/pathservice/go.mod +++ /dev/null @@ -1,38 +0,0 @@ -module github.com/ui/pathservice - -go 1.21.4 - -require github.com/spf13/cobra v1.8.1 - -require ( - dario.cat/mergo v1.0.0 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v1.1.5 // indirect - github.com/cloudflare/circl v1.3.7 // indirect - github.com/cyphar/filepath-securejoin v0.3.6 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.6.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/pjbgf/sha1cd v0.3.2 // indirect - github.com/skeema/knownhosts v1.3.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect -) - -require ( - github.com/emirpasic/gods v1.18.1 // indirect - github.com/go-git/go-git/v5 v5.13.2 - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/xanzy/ssh-agent v0.3.3 // indirect - go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.34.0 // indirect - golang.org/x/sys v0.29.0 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect -) diff --git a/pathservice/go.sum b/pathservice/go.sum deleted file mode 100644 index 04008f099..000000000 --- a/pathservice/go.sum +++ /dev/null @@ -1,122 +0,0 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v1.1.5 h1:eoAQfK2dwL+tFSFpr7TbOaPNUbPiJj4fLYwwGE1FQO4= -github.com/ProtonMail/go-crypto v1.1.5/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM= -github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elazarl/goproxy v1.4.0 h1:4GyuSbFa+s26+3rmYNSuUVsx+HgPrV1bk1jXI0l9wjM= -github.com/elazarl/goproxy v1.4.0/go.mod h1:X/5W/t+gzDyLfHW4DrMdpjqYjpXsURlBt9lpBDxZZZQ= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= -github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.13.2 h1:7O7xvsK7K+rZPKW6AQR1YyNhfywkv7B8/FsP3ki6Zv0= -github.com/go-git/go-git/v5 v5.13.2/go.mod h1:hWdW5P4YZRjmpGHwRH2v3zkWcNl6HeXaXQEMGb3NJ9A= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= -github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= -github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= -github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pathservice/main.go b/pathservice/main.go deleted file mode 100644 index 76b82790f..000000000 --- a/pathservice/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/ui/pathservice/cmd" - -func main() { - cmd.Execute() -} diff --git a/renovate.json b/renovate.json index 3743d343d..117e9cfbc 100644 --- a/renovate.json +++ b/renovate.json @@ -12,8 +12,7 @@ "deploy/k8s/overlays/openshift/prod/kustomization.yaml" ], "matchStrings": [ - "value: quay.io/instructlab-ui/ui:(?.*)", - "value: quay.io/instructlab-ui/pathservice:(?.*)" + "value: quay.io/instructlab-ui/ui:(?.*)" ], "datasourceTemplate": "github-releases", "depNameTemplate": "instructlab/ui", diff --git a/server/Containerfile b/server/Containerfile deleted file mode 100644 index 18da2bc7d..000000000 --- a/server/Containerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.11 - -# Set working directory -WORKDIR /app - -RUN pip install --upgrade pip -RUN pip install --no-cache-dir instructlab==0.16.1 - -# Copy project files to the working directory -COPY config.yaml . - -# Download the merlinite model -RUN ilab download - -# Copy project files to the working directory -COPY . . - -EXPOSE 8000 - -# Run the chat server with the specified model family and model file -CMD ["ilab", "serve", "--model-family", "merlinite", "--model-path", "models/merlinite-7b-lab-Q4_K_M.gguf"] \ No newline at end of file diff --git a/server/config.yaml b/server/config.yaml deleted file mode 100644 index ee4f316b7..000000000 --- a/server/config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -chat: - context: default - greedy_mode: false - logs_dir: data/chatlogs - max_tokens: null - model: models/merlinite-7b-lab-Q4_K_M.gguf - session: null - vi_mode: false - visible_overflow: true -general: - log_level: INFO -generate: - chunk_word_count: 1000 - model: models/merlinite-7b-lab-Q4_K_M.gguf - num_cpus: 10 - num_instructions: 100 - output_dir: generated - prompt_file: prompt.txt - seed_file: seed_tasks.json - taxonomy_base: origin/main - taxonomy_path: taxonomy -serve: - gpu_layers: -1 - host_port: 0.0.0.0:8000 - max_ctx_size: 4096 - model_path: models/merlinite-7b-lab-Q4_K_M.gguf diff --git a/src/app/api/auth/[...nextauth]/route.ts b/src/app/api/auth/[...nextauth]/route.ts index 161f6db1b..0d08ed34c 100644 --- a/src/app/api/auth/[...nextauth]/route.ts +++ b/src/app/api/auth/[...nextauth]/route.ts @@ -102,10 +102,9 @@ const authOptions: NextAuthOptions = { 'X-GitHub-Api-Version': '2022-11-28' }, validateStatus: (status) => { - return [204, 302, 404].includes(status); + return [204, 302, 404, 401].includes(status); } }); - if (response.status === 204) { console.log(`User ${githubProfile.login} successfully authenticated with GitHub organization - ${ORG}`); logger.info(`User ${githubProfile.login} successfully authenticated with GitHub organization - ${ORG}`); @@ -114,6 +113,10 @@ const authOptions: NextAuthOptions = { console.log(`User ${githubProfile.login} is not a member of the ${ORG} organization`); logger.warn(`User ${githubProfile.login} is not a member of the ${ORG} organization`); return `/login?error=NotOrgMember&user=${githubProfile.login}`; // Redirect to custom error page + } else if (response.status === 401) { + console.log(`The GitHub token is invalid.`); + logger.warn(`The GitHub token is invalid.`); + return `/login?error=InvalidToken`; } else { console.log(`Unexpected error while authenticating user ${githubProfile.login} with ${ORG} github organization.`); logger.error(`Unexpected error while authenticating user ${githubProfile.login} with ${ORG} github organization.`); diff --git a/src/app/api/convert-http/route.ts b/src/app/api/convert-http/route.ts new file mode 100644 index 000000000..9f80e5361 --- /dev/null +++ b/src/app/api/convert-http/route.ts @@ -0,0 +1,68 @@ +// src/app/api/convert-http/route.ts +'use server'; + +import { NextResponse } from 'next/server'; + +interface ConvertHttpRequestBody { + options?: { + from_formats?: string[]; + to_formats?: string[]; + image_export_mode?: string; + table_mode?: string; + abort_on_error?: boolean; + return_as_file?: boolean; + do_table_structure?: boolean; + include_images?: boolean; + }; + http_sources: { url: string }[]; +} + +// convert a doc from a URL (provided via http_sources) to Markdown. +export async function POST(request: Request) { + const baseUrl = process.env.IL_FILE_CONVERSION_SERVICE || 'http://doclingserve:5001'; + + try { + const healthRes = await fetch(`${baseUrl}/health`); + if (!healthRes.ok) { + console.error('The file conversion service is offline or returned non-OK status:', healthRes.status, healthRes.statusText); + return NextResponse.json({ error: 'Conversion service is offline, only markdown files accepted.' }, { status: 503 }); + } + + const healthData = await healthRes.json(); + if (!healthData.status || healthData.status !== 'ok') { + console.error('Conversion service health check response not "ok":', healthData); + return NextResponse.json({ error: 'Conversion service is offline, only markdown files accepted.' }, { status: 503 }); + } + } catch (error: unknown) { + console.error('Error conversion service health check:', error); + return NextResponse.json({ error: 'Conversion service is offline, only markdown files accepted.' }, { status: 503 }); + } + + try { + const body: ConvertHttpRequestBody = await request.json(); + + const res = await fetch(`${baseUrl}/v1alpha/convert/source`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); + + if (!res.ok) { + console.error('Conversion service responded with error', res.status, res.statusText); + return NextResponse.json({ error: `Conversion service call failed. ${res.statusText}` }, { status: 500 }); + } + + const data = await res.json(); + + // Return the markdown wrapped in JSON for the client to parse + return NextResponse.json({ content: data }, { status: 200 }); + } catch (error: unknown) { + if (error instanceof Error) { + console.error('Error during URL conversion route call:', error); + return NextResponse.json({ error: 'URL conversion failed.', message: error.message }, { status: 500 }); + } else { + console.error('Unknown error during conversion route call:', error); + return NextResponse.json({ error: 'Conversion failed due to an unknown error.' }, { status: 500 }); + } + } +} diff --git a/src/app/api/native/convert/route.ts b/src/app/api/convert/route.ts similarity index 78% rename from src/app/api/native/convert/route.ts rename to src/app/api/convert/route.ts index 88dc43da8..046152c27 100644 --- a/src/app/api/native/convert/route.ts +++ b/src/app/api/convert/route.ts @@ -1,4 +1,4 @@ -// src/app/api/native/convert/route.ts +// src/app/api/convert/route.ts import { NextResponse } from 'next/server'; `use server`; @@ -16,14 +16,14 @@ interface ConvertRequestBody { // This route calls the external REST service to convert any doc => markdown export async function POST(request: Request) { - try { - // 1. Parse JSON body from client - const body: ConvertRequestBody = await request.json(); + // 1. Parse JSON body from client + const body: ConvertRequestBody = await request.json(); - // 2. Read the IL_FILE_CONVERSION_SERVICE from .env - const baseUrl = process.env.IL_FILE_CONVERSION_SERVICE || 'http://doclingserve:5001'; + // 2. Read the IL_FILE_CONVERSION_SERVICE from .env + const baseUrl = process.env.IL_FILE_CONVERSION_SERVICE || 'http://doclingserve:5001'; - // 3. Check the health of the conversion service before proceeding + // 3. Check the health of the conversion service before proceeding + try { const healthRes = await fetch(`${baseUrl}/health`); if (!healthRes.ok) { console.error('The file conversion service is offline or returned non-OK status:', healthRes.status, healthRes.statusText); @@ -36,8 +36,13 @@ export async function POST(request: Request) { console.error('Doc->md conversion service health check response not "ok":', healthData); return NextResponse.json({ error: 'Conversion service is offline, only markdown files accepted.' }, { status: 503 }); } + } catch (error: unknown) { + console.error('Error conversion service health check:', error); + return NextResponse.json({ error: 'Conversion service is offline, only markdown files accepted.' }, { status: 503 }); + } - // 4. Service is healthy, proceed with md conversion + // 4. Service is healthy, proceed with md conversion + try { const res = await fetch(`${baseUrl}/v1alpha/convert/source`, { method: 'POST', headers: { diff --git a/src/app/api/envConfig/route.ts b/src/app/api/envConfig/route.ts index 101e88a0a..fa78e6cbd 100644 --- a/src/app/api/envConfig/route.ts +++ b/src/app/api/envConfig/route.ts @@ -16,6 +16,7 @@ export async function GET() { UPSTREAM_REPO_NAME: process.env.NEXT_PUBLIC_TAXONOMY_REPO || '', DEPLOYMENT_TYPE: process.env.IL_UI_DEPLOYMENT || '', ENABLE_DEV_MODE: process.env.IL_ENABLE_DEV_MODE || 'false', + ENABLE_DOC_CONVERSION: process.env.IL_ENABLE_DOC_CONVERSION || 'false', EXPERIMENTAL_FEATURES: process.env.NEXT_PUBLIC_EXPERIMENTAL_FEATURES || '', TAXONOMY_ROOT_DIR: process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || '', TAXONOMY_KNOWLEDGE_DOCUMENT_REPO: diff --git a/src/app/api/github/download/route.ts b/src/app/api/github/download/route.ts new file mode 100644 index 000000000..1578c778e --- /dev/null +++ b/src/app/api/github/download/route.ts @@ -0,0 +1,54 @@ +// src/app/api/github/download/route.ts +'use server'; +import { NextRequest, NextResponse } from 'next/server'; +import { getToken } from 'next-auth/jwt'; +import { GITHUB_API_URL } from '@/types/const'; +import { getGitHubUsername } from '@/utils/github'; + +const UPSTREAM_REPO_NAME = process.env.NEXT_PUBLIC_TAXONOMY_REPO!; + +export async function POST(req: NextRequest) { + const token = await getToken({ req, secret: process.env.NEXTAUTH_SECRET! }); + + if (!token || !token.accessToken) { + console.error('Unauthorized: Missing or invalid access token'); + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); + } + + const githubToken = token.accessToken as string; + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${githubToken}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28' + }; + + const githubUsername = await getGitHubUsername(headers); + try { + const { branchName } = await req.json(); + + if (!branchName || typeof branchName !== 'string') { + return NextResponse.json({ error: 'contribution branch does not exist on remote taxonomy.' }, { status: 400 }); + } + + const tarballUrl = `${GITHUB_API_URL}/repos/${githubUsername}/${UPSTREAM_REPO_NAME}/tarball/${branchName}`; + const tarballRes = await fetch(tarballUrl, { + headers: headers + }); + + if (!tarballRes.ok) { + return NextResponse.json({ error: 'Failed to download taxonomy for the contribution.' }, { status: 500 }); + } + + return new NextResponse(tarballRes.body, { + headers: { + 'Content-Type': 'application/gzip', + 'Content-Disposition': `attachment`, + 'Cache-Control': 'no-store' + } + }); + } catch (error) { + console.error('failed to download taxonomy for the contribution:', error); + return NextResponse.json({ error: error }, { status: 500 }); + } +} diff --git a/src/app/api/github/knowledge-files/route.ts b/src/app/api/github/knowledge-files/route.ts index 136a23f60..d73e2001b 100644 --- a/src/app/api/github/knowledge-files/route.ts +++ b/src/app/api/github/knowledge-files/route.ts @@ -1,10 +1,17 @@ // src/app/api/github/knowledge-files/route.ts import { NextRequest, NextResponse } from 'next/server'; import { getToken } from 'next-auth/jwt'; - -const GITHUB_API_URL = 'https://api.github.com'; -const TAXONOMY_DOCUMENTS_REPO = process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs'; -const BASE_BRANCH = 'main'; +import { + BASE_BRANCH, + checkIfRepoExists, + forkRepo, + getBranchSha, + getGitHubUsernameAndEmail, + readCommit, + GITHUB_API_URL, + TAXONOMY_DOCUMENTS_REPO +} from '@/app/api/github/utils'; +import path from 'path'; // Interface for the response interface KnowledgeFile { @@ -32,7 +39,7 @@ export async function POST(req: NextRequest) { try { const body = await req.json(); - const { files } = body; + const { branchName, currentCommitSHA, newFiles, updatedExistingFiles } = body; // Fetch GitHub username and email const { githubUsername, userEmail } = await getGitHubUsernameAndEmail(headers); @@ -59,24 +66,87 @@ export async function POST(req: NextRequest) { const baseBranchSha = await getBranchSha(headers, githubUsername, repoName, BASE_BRANCH); console.log(`Base branch SHA: ${baseBranchSha}`); + // New directory for the files + const newSubDirectory = new Date().toISOString().replace(/[-:.]/g, '').replace('T', 'T').slice(0, -1); + + // Read the preview commit sha + let contributionName = branchName; + let existingSubDirectory = ''; + const finalFiles = new Map(); + + const commitMsg = await readCommit(headers, githubUsername, repoName, currentCommitSHA); + if (commitMsg.length != 0) { + const name = extractContributionName(commitMsg); + if (name.length === 0) { + console.warn('Contribution name not found. Looks like the SHA is not correct.'); + console.log('Continue uploading the newly provided document.'); + } else { + console.log(`Contribution name found for the sha ${currentCommitSHA}:`, contributionName); + contributionName = name; + } + + const directory = extractSubDirectoryName(commitMsg); + if (directory.length === 0) { + console.warn('No subdirectory exist. Either the commit sha is invalid, or the docs are manually deleted.'); + console.log('Continue uploading the newly provided document.'); + } else { + console.log(`Document sub directory exist for the sha ${currentCommitSHA}:`, directory); + existingSubDirectory = directory; + } + } else { + console.log('Uploading the documents for the first time for the contribution'); + } + + let existingFiles = []; + if (existingSubDirectory.length != 0) { + // Read all the files from the existing directory. + existingFiles = await fetchAllFilesFromDir(headers, githubUsername, repoName, path.join(contributionName, existingSubDirectory)); + existingFiles.map((existingFile: { fileName: string; filePath: string; fileSha: string; fileContent: string }) => { + if (updatedExistingFiles.some((file: { fileName: string; fileContent: string }) => file.fileName === existingFile.fileName)) { + console.log('Re-uploading existing file : ', existingFile.filePath); + finalFiles.set(existingFile.fileName, { + fileName: path.join(contributionName, newSubDirectory, existingFile.fileName), + fileContent: existingFile.fileContent + }); + } else { + console.log(`${existingFile.fileName} is either deleted or replaced with newer version`); + } + }); + } + // Create files in the main branch with unique filenames e.g. foo-20240618T203521842.md - const timestamp = new Date().toISOString().replace(/[-:.]/g, '').replace('T', 'T').slice(0, -1); - const filesWithTimestamp = files.map((file: { fileName: string; fileContent: string }) => { - const [name, extension] = file.fileName.split(/\.(?=[^.]+$)/); - return { - fileName: `${name}-${timestamp}.${extension}`, + newFiles.map((file: { fileName: string; fileContent: string; action: string }) => { + console.log(`Uploading new file ${file.fileName} from knowledge contribution ${branchName}`); + finalFiles.set(file.fileName, { + fileName: path.join(contributionName, newSubDirectory, file.fileName), fileContent: file.fileContent - }; + }); }); - const commitSha = await createFilesCommit(headers, githubUsername, repoName, BASE_BRANCH, filesWithTimestamp, userEmail, baseBranchSha); + const commitSha = await createFilesCommit( + headers, + githubUsername, + repoName, + BASE_BRANCH, + contributionName, + newSubDirectory, + finalFiles, + userEmail, + baseBranchSha + ); console.log(`Created files commit SHA: ${commitSha}`); + if (existingSubDirectory.length != 0 && existingFiles.length != 0) { + // Deleting the existing files + await deleteExistingFiles(headers, githubUsername, repoName, BASE_BRANCH, contributionName, existingSubDirectory); + console.log('Existing files are cleaned up'); + } + return NextResponse.json( { repoUrl: `https://github.com/${githubUsername}/${repoName}`, commitSha, - documentNames: filesWithTimestamp.map((file: { fileName: string }) => file.fileName), + documentNames: Array.from(finalFiles.values()).map((file: { fileName: string }) => file.fileName), //TODO: prUrl: `https://github.com/${githubUsername}/${repoName}` }, { status: 201 } @@ -87,88 +157,22 @@ export async function POST(req: NextRequest) { } } -async function getGitHubUsernameAndEmail(headers: HeadersInit): Promise<{ githubUsername: string; userEmail: string }> { - const response = await fetch(`${GITHUB_API_URL}/user`, { headers }); - - if (!response.ok) { - const errorText = await response.text(); - console.error('Failed to fetch GitHub username and email:', response.status, errorText); - throw new Error('Failed to fetch GitHub username and email'); - } - - const data = await response.json(); - return { githubUsername: data.login, userEmail: data.email }; -} - -async function checkIfRepoExists(headers: HeadersInit, owner: string, repo: string): Promise { - const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}`, { headers }); - const exists = response.ok; - if (!exists) { - const errorText = await response.text(); - console.error('Repository does not exist:', response.status, errorText); - } - return exists; -} - -async function forkRepo(headers: HeadersInit, owner: string, repo: string, forkOwner: string) { - const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/forks`, { - method: 'POST', - headers - }); - - if (!response.ok) { - const errorText = await response.text(); - console.error('Failed to fork repository:', response.status, errorText); - throw new Error('Failed to fork repository'); - } - - // Wait for the fork to be created - let forkCreated = false; - for (let i = 0; i < 10; i++) { - const forkExists = await checkIfRepoExists(headers, forkOwner, repo); - if (forkExists) { - forkCreated = true; - break; - } - await new Promise((resolve) => setTimeout(resolve, 3000)); - } - - if (!forkCreated) { - throw new Error('Failed to confirm fork creation'); - } -} - -async function getBranchSha(headers: HeadersInit, owner: string, repo: string, branch: string): Promise { - console.log(`Fetching branch SHA for ${branch}...`); - const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/ref/heads/${branch}`, { headers }); - - if (!response.ok) { - const errorText = await response.text(); - console.error('Failed to get branch SHA:', response.status, errorText); - if (response.status === 409 && errorText.includes('Git Repository is empty')) { - throw new Error('Git Repository is empty.'); - } - throw new Error('Failed to get branch SHA'); - } - - const data = await response.json(); - console.log('Branch SHA:', data.object.sha); - return data.object.sha; -} - async function createFilesCommit( headers: HeadersInit, owner: string, repo: string, branchName: string, - files: { fileName: string; fileContent: string }[], + contributionName: string, + subDirectory: string, + files: Map, userEmail: string, baseSha: string ): Promise { console.log('Creating files commit...'); // Create blobs for each file + const filesArray = Array.from(files.values()); const blobs = await Promise.all( - files.map((file) => + filesArray.map((file) => fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs`, { method: 'POST', headers, @@ -179,7 +183,6 @@ async function createFilesCommit( }).then((response) => response.json()) ) ); - console.log('Blobs created:', blobs); // Create tree const createTreeResponse = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees`, { @@ -187,7 +190,7 @@ async function createFilesCommit( headers, body: JSON.stringify({ base_tree: baseSha, - tree: files.map((file, index) => ({ + tree: filesArray.map((file, index) => ({ path: file.fileName, mode: '100644', type: 'blob', @@ -213,7 +216,7 @@ async function createFilesCommit( method: 'POST', headers, body: JSON.stringify({ - message: `Add files: ${files.map((file) => file.fileName).join(', ')}\n\nSigned-off-by: ${userEmail}`, + message: `Contribution Name:${contributionName}\n\nDocument uploaded: ${filesArray.map((file) => file.fileName).join(', ')}\n\nSub-Directory:${subDirectory}\n\nSigned-off-by: ${owner} <${userEmail}>`, tree: treeData.sha, parents: [baseSha] }) @@ -245,8 +248,94 @@ async function createFilesCommit( return commitData.sha; } +async function deleteExistingFiles( + headers: HeadersInit, + owner: string, + repo: string, + branchName: string, + contributionName: string, + subDirectory: string +) { + console.log('Deleting existing files for contribution : ', contributionName); + const baseBranchSha = await getBranchSha(headers, owner, repo, branchName); + console.log(`${contributionName}: base branch sha :`, baseBranchSha); + + try { + let response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${baseBranchSha}?recursive=1`, { + headers + }); + + if (!response.ok) { + console.error('Failed to fetch the git tree', response.statusText); + } + const data = await response.json(); + + const dirPath = path.join(contributionName, subDirectory); + const updatedTree = data.tree.map((item: { path: string; sha: string }) => { + if (item.path.startsWith(dirPath)) { + return { ...item, sha: null }; // Mark for deletion + } + return item; + }); + + response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees`, { + method: 'POST', + headers, + body: JSON.stringify({ + base_tree: baseBranchSha, + tree: updatedTree + }) + }); + + if (!response.ok) { + console.error('Failed to update the git tree', response.statusText); + } + const treeUpdateData = await response.json(); + + response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/commits`, { + method: 'POST', + headers, + body: JSON.stringify({ + message: `Deleting subdirectory ${subDirectory} for contribution ${contributionName}`, + tree: treeUpdateData.sha, + parents: [baseBranchSha] + }) + }); + + if (!response.ok) { + console.error('Failed to create a delete commit for contribution:', contributionName); + } + + const commitData = await response.json(); + console.log('Delete commit created:', commitData); + + // Update branch reference + const updateBranchResponse = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/refs/heads/${branchName}`, { + method: 'PATCH', + headers, + body: JSON.stringify({ sha: commitData.sha }) + }); + + if (!updateBranchResponse.ok) { + const errorText = await updateBranchResponse.text(); + console.error('Failed to update branch reference:', updateBranchResponse.status, errorText); + throw new Error('Failed to update branch reference'); + } + console.log('Branch reference updated'); + console.log(`${contributionName}: new branch sha :`, commitData.sha); + } catch (error) { + console.error(`Failed to delete files for the contribution : ${contributionName}`, error); + } +} + export async function GET(req: NextRequest) { try { + const url = new URL(req.url); + const commitSHA = url.searchParams.get('commitSHA'); + if (commitSHA == null) { + return NextResponse.json({ files: [] }, { status: 200 }); + } + const token = await getToken({ req, secret: process.env.NEXTAUTH_SECRET! }); if (!token || !token.accessToken) { @@ -269,92 +358,141 @@ export async function GET(req: NextRequest) { const repoPath = TAXONOMY_DOCUMENTS_REPO.replace('github.com/', ''); const [_, repoName] = repoPath.split('/'); - const files = await fetchMarkdownFiles(headers, githubUsername, repoName, BASE_BRANCH); - const knowledgeFiles: KnowledgeFile[] = []; - - for (const file of files) { - const commitInfo = await fetchCommitInfo(headers, githubUsername, repoName, file.path); - if (commitInfo) { - const { sha, date } = commitInfo; - const content = await fetchFileContent(headers, githubUsername, repoName, file.path); - knowledgeFiles.push({ - filename: file.path, - content: content, - commitSha: sha, - commitDate: date - }); + // Read the preview commit sha + let contributionName = ''; + let existingSubDirectory = ''; + const finalFiles: KnowledgeFile[] = []; + + const commitMsg = await readCommit(headers, githubUsername, repoName, commitSHA); + if (commitMsg.length != 0) { + const name = extractContributionName(commitMsg); + if (name.length === 0) { + console.warn('Contribution name not found. Looks like the SHA is not correct.'); + console.log('Continue uploading the newly provided document.'); + } else { + console.log(`Contribution name found for the sha ${commitSHA}:`, contributionName); + contributionName = name; + } + + const directory = extractSubDirectoryName(commitMsg); + if (directory.length === 0) { + console.warn('No subdirectory exist. Either the commit sha is invalid, or the docs are manually deleted.'); + console.log('Continue uploading the newly provided document.'); + } else { + console.log(`Document sub directory exist for the sha ${commitSHA}:`, directory); + existingSubDirectory = directory; } + } else { + console.log('Uploading the documents for the first time for the contribution'); + } + + if (existingSubDirectory.length != 0) { + // Read all the files from the existing directory. + const existingFiles = await fetchAllFilesFromDir(headers, githubUsername, repoName, path.join(contributionName, existingSubDirectory)); + existingFiles.map((file: { fileName: string; filePath: string; fileSha: string; fileContent: string }) => { + console.log('Existing file found : ', file.filePath); + finalFiles.push({ + filename: file.fileName, + content: file.fileContent, + commitSha: file.fileSha, + commitDate: '' + }); + }); } - return NextResponse.json({ files: knowledgeFiles }, { status: 200 }); + return NextResponse.json({ files: finalFiles }, { status: 200 }); } catch (error) { console.error('Failed to process GET request:', error); return NextResponse.json({ error: (error as Error).message }, { status: 500 }); } } -// Fetch all markdown files from the main branch -async function fetchMarkdownFiles( - headers: HeadersInit, - owner: string, - repo: string, - branchName: string -): Promise<{ path: string; content: string }[]> { +// Fetch the content of a file from the repository +async function fetchFileContent(headers: HeadersInit, owner: string, repo: string, filePath: string): Promise { try { - const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${branchName}?recursive=1`, { headers }); + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${filePath}`, { headers }); if (!response.ok) { const errorText = await response.text(); - console.error('Failed to fetch files from knowledge document repository:', response.status, errorText); - throw new Error('Failed to fetch file from knowledge document repository:'); + console.error(`Failed to fetch content of file ${filePath} :`, response.status, errorText); + throw new Error(`Failed to fetch content of file ${filePath}.`); } const data = await response.json(); - const files = data.tree.filter( - (item: { type: string; path: string }) => item.type === 'blob' && item.path.endsWith('.md') && item.path !== 'README.md' - ); - return files.map((file: { path: string; content: string }) => ({ path: file.path, content: file.content })); - } catch (error) { - console.error('Error fetching files from knowledge document repository:', error); - return []; - } -} - -// Fetch the latest commit info for a file -async function fetchCommitInfo(headers: HeadersInit, owner: string, repo: string, filePath: string): Promise<{ sha: string; date: string } | null> { - try { - const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/commits?path=${filePath}`, { headers }); - if (!response.ok) { - const errorText = await response.text(); - console.error('Failed to fetch commit information for file:', response.status, errorText); - throw new Error('Failed to fetch commit information for file.'); + if (data.content) { + return Buffer.from(data.content, 'base64').toString('utf-8'); + } + if (!data.sha) { + console.error(`Failed to fetch content of file ${filePath}, no file sha found.`); + throw new Error(`Failed to fetch content of file ${filePath}.`); } - const data = await response.json(); - if (data.length === 0) return null; + const blobResponse = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${data.sha}`, { headers }); - return { - sha: data[0].sha, - date: data[0].commit.committer.date - }; + if (!blobResponse.ok) { + const errorText = await blobResponse.text(); + console.error(`Failed to fetch blob content of file ${filePath} :`, blobResponse.status, errorText); + throw new Error(`Failed to fetch content of file ${filePath}.`); + } + + const blobData = await blobResponse.json(); + return Buffer.from(blobData.content, 'base64').toString('utf-8'); } catch (error) { - console.error(`Error fetching commit info for ${filePath}:`, error); - return null; + console.error(`Error fetching content for ${filePath}:`, error); + return ''; } } -// Fetch the content of a file from the repository -async function fetchFileContent(headers: HeadersInit, owner: string, repo: string, filePath: string): Promise { +async function fetchAllFilesFromDir( + headers: HeadersInit, + owner: string, + repo: string, + filePath: string +): Promise<{ fileName: string; filePath: string; fileSha: string; fileContent: string }[]> { try { - const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${filePath}`, { headers }); - if (!response.ok) { - const errorText = await response.text(); - console.error(`Failed to fetch content of file ${filePath} :`, response.status, errorText); - throw new Error(`Failed to fetch content of file ${filePath}.`); - } + const contentsResponse = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${filePath}`, { + method: 'GET', + headers + }); - const data = await response.json(); - return Buffer.from(data.content, 'base64').toString('utf-8'); + const contentsData = await contentsResponse.json(); + if (!Array.isArray(contentsData)) { + throw new Error(`${filePath} is not a directory`); + } + const results = Promise.all( + contentsData.map(async (file) => { + const content = await fetchFileContent(headers, owner, repo, file.path); + return { + fileName: file.name, + filePath: file.path, + fileSha: file.sha, + fileContent: content + }; + }) + ); + return results; } catch (error) { - console.error(`Error fetching content for ${filePath}:`, error); - return ''; + console.error(`Failed to read content from :${filePath}`, error); } + return []; +} + +/** + * Extract the sub directory name where documents are stored. + * @param commitMessage Commit message from the provided SHA + * @returns + */ +function extractSubDirectoryName(commitMessage: string): string { + const match = commitMessage.match(/Sub-Directory:(.+)/); + if (!match) return ''; + return match[1].trim(); +} + +/** + * Extract the contribution name where documents are stored. + * @param commitMessage Commit message from the provided SHA + * @returns + */ +function extractContributionName(commitMessage: string): string { + const match = commitMessage.match(/Contribution Name:(.+)/); + if (!match) return ''; + return match[1].trim(); } diff --git a/src/app/api/github/pr/knowledge/route.ts b/src/app/api/github/pr/knowledge/route.ts index baa727ced..d43871462 100644 --- a/src/app/api/github/pr/knowledge/route.ts +++ b/src/app/api/github/pr/knowledge/route.ts @@ -6,6 +6,7 @@ import { KnowledgeYamlData, AttributionData } from '@/types'; import { GITHUB_API_URL, BASE_BRANCH } from '@/types/const'; import { dumpYaml } from '@/utils/yamlConfig'; import { checkUserForkExists, createBranch, createFilesInSingleCommit, createFork, getBaseBranchSha, getGitHubUsername } from '@/utils/github'; +import { prInfoFromSummary } from '@/app/api/github/utils'; const KNOWLEDGE_DIR = 'knowledge'; const UPSTREAM_REPO_OWNER = process.env.NEXT_PUBLIC_TAXONOMY_REPO_OWNER!; @@ -29,7 +30,7 @@ export async function POST(req: NextRequest) { try { const body = await req.json(); - const { content, attribution, name, email, submissionSummary, documentOutline, filePath } = body; + const { branchName, content, attribution, name, email, submissionSummary, filePath } = body; const knowledgeData: KnowledgeYamlData = yaml.load(content) as KnowledgeYamlData; const attributionData: AttributionData = attribution; @@ -44,7 +45,6 @@ export async function POST(req: NextRequest) { await createFork(headers, UPSTREAM_REPO_OWNER, UPSTREAM_REPO_NAME, githubUsername); } - const branchName = `knowledge-contribution-${Date.now()}`; const newYamlFilePath = `${KNOWLEDGE_DIR}/${filePath}qna.yaml`; const newAttributionFilePath = `${KNOWLEDGE_DIR}/${filePath}attribution.txt`; @@ -64,6 +64,8 @@ Creator names: ${attributionData.creator_names} // Create a new branch in the user's fork await createBranch(headers, githubUsername, UPSTREAM_REPO_NAME, branchName, baseBranchSha); + const { prTitle, prBody, commitMessage } = prInfoFromSummary(submissionSummary); + // Create both files in a single commit with DCO sign-off await createFilesInSingleCommit( headers, @@ -74,11 +76,11 @@ Creator names: ${attributionData.creator_names} { path: newAttributionFilePath, content: attributionContent } ], branchName, - `${submissionSummary}\n\nSigned-off-by: ${name} <${email}>` + `${commitMessage}\n\nSigned-off-by: ${name} <${email}>` ); // Create a pull request from the user's fork to the upstream repository - const pr = await createPullRequest(headers, githubUsername, branchName, submissionSummary, documentOutline); + const pr = await createPullRequest(headers, githubUsername, branchName, prTitle, prBody); return NextResponse.json(pr, { status: 201 }); } catch (error) { @@ -87,14 +89,14 @@ Creator names: ${attributionData.creator_names} } } -async function createPullRequest(headers: HeadersInit, username: string, branchName: string, knowledgeSummary: string, documentOutline: string) { +async function createPullRequest(headers: HeadersInit, username: string, branchName: string, prTitle: string, prBody?: string) { const response = await fetch(`${GITHUB_API_URL}/repos/${UPSTREAM_REPO_OWNER}/${UPSTREAM_REPO_NAME}/pulls`, { method: 'POST', headers, body: JSON.stringify({ - title: `Knowledge: ${knowledgeSummary}`, + title: prTitle, head: `${username}:${branchName}`, - body: documentOutline, + body: prBody, base: BASE_BRANCH }) }); diff --git a/src/app/api/github/pr/skill/route.ts b/src/app/api/github/pr/skill/route.ts index 380a8f8ba..9646d3fd8 100644 --- a/src/app/api/github/pr/skill/route.ts +++ b/src/app/api/github/pr/skill/route.ts @@ -6,6 +6,7 @@ import { SkillYamlData, AttributionData } from '@/types'; import { GITHUB_API_URL, BASE_BRANCH } from '@/types/const'; import { dumpYaml } from '@/utils/yamlConfig'; import { checkUserForkExists, createBranch, createFilesInSingleCommit, createFork, getBaseBranchSha, getGitHubUsername } from '@/utils/github'; +import { prInfoFromSummary } from '@/app/api/github/utils'; const SKILLS_DIR = 'compositional_skills'; const UPSTREAM_REPO_OWNER = process.env.NEXT_PUBLIC_TAXONOMY_REPO_OWNER!; @@ -29,7 +30,7 @@ export async function POST(req: NextRequest) { try { const body = await req.json(); - const { content, attribution, name, email, submissionSummary, documentOutline, filePath } = body; + const { branchName, content, attribution, name, email, submissionSummary, filePath } = body; const githubUsername = await getGitHubUsername(headers); console.log('Skill contribution from gitHub Username:', githubUsername); @@ -40,7 +41,6 @@ export async function POST(req: NextRequest) { await createFork(headers, UPSTREAM_REPO_OWNER, UPSTREAM_REPO_NAME, githubUsername); } - const branchName = `skill-contribution-${Date.now()}`; const newYamlFilePath = `${SKILLS_DIR}/${filePath}qna.yaml`; const newAttributionFilePath = `${SKILLS_DIR}/${filePath}attribution.txt`; @@ -62,6 +62,8 @@ Creator names: ${attributionData.creator_names} // Create a new branch in the user's fork await createBranch(headers, githubUsername, UPSTREAM_REPO_NAME, branchName, baseBranchSha); + const { prTitle, prBody, commitMessage } = prInfoFromSummary(submissionSummary); + // Create both files in a single commit await createFilesInSingleCommit( headers, @@ -72,11 +74,11 @@ Creator names: ${attributionData.creator_names} { path: newAttributionFilePath, content: attributionString } ], branchName, - `${submissionSummary}\n\nSigned-off-by: ${name} <${email}>` + `${commitMessage}\n\nSigned-off-by: ${name} <${email}>` ); // Create a pull request from the user's fork to the upstream repository - const pr = await createPullRequest(headers, githubUsername, branchName, submissionSummary, documentOutline); + const pr = await createPullRequest(headers, githubUsername, branchName, prTitle, prBody); return NextResponse.json(pr, { status: 201 }); } catch (error) { @@ -85,14 +87,14 @@ Creator names: ${attributionData.creator_names} } } -async function createPullRequest(headers: HeadersInit, username: string, branchName: string, skillSummary: string, skillDescription: string) { +async function createPullRequest(headers: HeadersInit, username: string, branchName: string, prTitle: string, prBody?: string) { const response = await fetch(`${GITHUB_API_URL}/repos/${UPSTREAM_REPO_OWNER}/${UPSTREAM_REPO_NAME}/pulls`, { method: 'POST', headers, body: JSON.stringify({ - title: `Skill: ${skillSummary}`, + title: prTitle, + body: prBody, head: `${username}:${branchName}`, - body: skillDescription, base: BASE_BRANCH }) }); diff --git a/src/app/api/github/tree/route.ts b/src/app/api/github/tree/route.ts new file mode 100644 index 000000000..7aaf6b4d8 --- /dev/null +++ b/src/app/api/github/tree/route.ts @@ -0,0 +1,123 @@ +// src/app/api/github/tree/route.ts +import { NextRequest, NextResponse } from 'next/server'; +import * as git from 'isomorphic-git'; +import http from 'isomorphic-git/http/node'; +import path from 'path'; +import fs from 'fs'; + +const LOCAL_TAXONOMY_ROOT_DIR = process.env.NEXT_PUBLIC_LOCAL_TAXONOMY_ROOT_DIR || `${process.env.HOME}/.instructlab-ui`; +const TAXONOMY_REPO_URL = process.env.NEXT_PUBLIC_TAXONOMY_REPO_URL || 'https://github.com/instructlab/taxonomy.git'; +const SKILLS = 'compositional_skills'; +const KNOWLEDGE = 'knowledge'; +const CHECK_INTERVAL = 300000; // 5 minute +let lastChecked = 0; + +async function cloneTaxonomyRepo(): Promise { + const taxonomyDirectoryPath = path.join(LOCAL_TAXONOMY_ROOT_DIR, '/taxonomy'); + + if (fs.existsSync(taxonomyDirectoryPath)) { + fs.rmdirSync(taxonomyDirectoryPath, { recursive: true }); + } + + try { + await git.clone({ + fs, + http, + dir: taxonomyDirectoryPath, + url: TAXONOMY_REPO_URL, + singleBranch: true + }); + + // Include the full path in the response for client display + console.log(`Local Taxonomy repository cloned successfully to ${taxonomyDirectoryPath}.`); + return true; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'; + console.error(`Failed to clone local taxonomy repository: ${errorMessage}`); + return false; + } +} + +async function deleteTaxonomyRepo(): Promise { + const taxonomyDirectoryPath = path.join(LOCAL_TAXONOMY_ROOT_DIR, '/taxonomy'); + + if (fs.existsSync(taxonomyDirectoryPath)) { + fs.rmdirSync(taxonomyDirectoryPath, { recursive: true }); + } +} + +async function getRemoteHeadHash(): Promise { + try { + const remoteRefs = await git.listServerRefs({ http, url: TAXONOMY_REPO_URL }); + const mainRef = remoteRefs.find((ref) => ref.ref.endsWith('refs/heads/main')); + return mainRef?.oid || null; + } catch (error) { + console.error('Failed to get remote head hash:', error); + return null; + } +} + +async function getLocalHeadHash(): Promise { + try { + const taxonomyDirectoryPath = path.join(LOCAL_TAXONOMY_ROOT_DIR, '/taxonomy'); + + const head = await git.resolveRef({ fs, dir: taxonomyDirectoryPath, ref: 'HEAD' }); + return head || null; + } catch (error) { + console.error('Failed to get local head hash:', error); + return null; + } +} + +async function checkForUpdates(): Promise { + const currentTime = Date.now(); + if (currentTime - lastChecked < CHECK_INTERVAL) { + return; + } + lastChecked = currentTime; + const timestamp = new Date().toISOString(); + console.log(`${timestamp}: Checking for updates... `); + const remoteHash = await getRemoteHeadHash(); + const localHash = await getLocalHeadHash(); + + if (remoteHash && localHash && remoteHash !== localHash) { + console.log(`${timestamp}: New changes detected, updating repository...`); + await deleteTaxonomyRepo(); + await cloneTaxonomyRepo(); + } else { + console.log(`${timestamp}: No new changes detected in taxonomy repo.`); + } +} + +function getFirstLevelDirectories(directoryPath: string): string[] { + try { + checkForUpdates(); + return fs + .readdirSync(directoryPath) + .map((name) => path.join(directoryPath, name)) + .filter((source) => fs.statSync(source).isDirectory()) + .map((dir) => path.basename(dir)); + } catch (error) { + console.error('Error reading directory:', error); + return []; + } +} + +export async function POST(req: NextRequest) { + const body = await req.json(); + const { root_path, dir_name } = body; + + try { + let dirPath = ''; + if (root_path === 'skills') { + dirPath = path.join(LOCAL_TAXONOMY_ROOT_DIR, 'taxonomy', SKILLS, dir_name); + } else { + dirPath = path.join(LOCAL_TAXONOMY_ROOT_DIR, 'taxonomy', KNOWLEDGE, dir_name); + } + const dirs = getFirstLevelDirectories(dirPath); + return NextResponse.json({ data: dirs }, { status: 201 }); + } catch (error) { + console.error('Failed to get the tree for path:', root_path, error); + return NextResponse.json({ error: 'Failed to get the tree for path' }, { status: 500 }); + } +} diff --git a/src/app/api/github/utils.ts b/src/app/api/github/utils.ts new file mode 100644 index 000000000..7a08403ef --- /dev/null +++ b/src/app/api/github/utils.ts @@ -0,0 +1,150 @@ +export const GITHUB_API_URL = 'https://api.github.com'; +export const TAXONOMY_DOCUMENTS_REPO = + process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs'; +export const BASE_BRANCH = 'main'; + +export const prInfoFromSummary = (summaryString: string): { prTitle: string; prBody?: string; commitMessage: string } => { + const prTitle = summaryString.length > 60 ? `${summaryString.slice(0, 60)}...` : summaryString; + const prBody = summaryString.length > 60 ? `...${summaryString.slice(60)}` : undefined; + const commitMessage = `${prTitle}${prBody ? `\n\n${prBody}` : ''}`; + return { prTitle, prBody, commitMessage }; +}; + +export const checkIfRepoExists = async (headers: HeadersInit, owner: string, repo: string): Promise => { + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}`, { headers }); + const exists = response.ok; + if (!exists) { + const errorText = await response.text(); + console.error('Repository does not exist:', response.status, errorText); + } + return exists; +}; + +export const forkRepo = async (headers: HeadersInit, owner: string, repo: string, forkOwner: string) => { + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/forks`, { + method: 'POST', + headers + }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to fork repository:', response.status, errorText); + throw new Error('Failed to fork repository'); + } + + // Wait for the fork to be created + let forkCreated = false; + for (let i = 0; i < 10; i++) { + const forkExists = await checkIfRepoExists(headers, forkOwner, repo); + if (forkExists) { + forkCreated = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 3000)); + } + + if (!forkCreated) { + throw new Error('Failed to confirm fork creation'); + } +}; + +export const getBranchSha = async (headers: HeadersInit, owner: string, repo: string, branch: string): Promise => { + console.log(`Fetching branch SHA for ${branch}...`); + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/ref/heads/${branch}`, { headers }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to get branch SHA:', response.status, errorText); + if (response.status === 409 && errorText.includes('Git Repository is empty')) { + throw new Error('Git Repository is empty.'); + } + throw new Error('Failed to get branch SHA'); + } + + const data = await response.json(); + console.log('Branch SHA:', data.object.sha); + return data.object.sha; +}; + +// Fetch all markdown files from the main branch +export const fetchMarkdownFiles = async ( + headers: HeadersInit, + owner: string, + repo: string, + branchName: string +): Promise<{ path: string; content: string }[]> => { + try { + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${branchName}?recursive=1`, { headers }); + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to fetch files from knowledge document repository:', response.status, errorText); + throw new Error('Failed to fetch file from knowledge document repository:'); + } + + const data = await response.json(); + const files = data.tree.filter( + (item: { type: string; path: string }) => item.type === 'blob' && item.path.endsWith('.md') && item.path !== 'README.md' + ); + return files.map((file: { path: string; content: string }) => ({ path: file.path, content: file.content })); + } catch (error) { + console.error('Error fetching files from knowledge document repository:', error); + return []; + } +}; + +// Fetch the latest commit info for a file +export const fetchCommitInfo = async ( + headers: HeadersInit, + owner: string, + repo: string, + filePath: string +): Promise<{ sha: string; date: string } | null> => { + try { + const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/commits?path=${filePath}`, { headers }); + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to fetch commit information for file:', response.status, errorText); + throw new Error('Failed to fetch commit information for file.'); + } + + const data = await response.json(); + if (data.length === 0) return null; + + return { + sha: data[0].sha, + date: data[0].commit.committer.date + }; + } catch (error) { + console.error(`Error fetching commit info for ${filePath}:`, error); + return null; + } +}; + +export async function getGitHubUsernameAndEmail(headers: HeadersInit): Promise<{ githubUsername: string; userEmail: string }> { + const response = await fetch(`${GITHUB_API_URL}/user`, { headers }); + + if (!response.ok) { + const errorText = await response.text(); + console.error('Failed to fetch GitHub username and email:', response.status, errorText); + throw new Error('Failed to fetch GitHub username and email'); + } + + const data = await response.json(); + return { githubUsername: data.login, userEmail: data.email }; +} + +export async function readCommit(headers: HeadersInit, owner: string, repo: string, commitSHA: string): Promise { + try { + const commitResponse = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/commits/${commitSHA}`, { + method: 'GET', + headers + }); + + const commitData = await commitResponse.json(); + console.log('Commit Message:', commitData.commit.message); + return commitData.commit.message; + } catch (error) { + console.error(`Failed to fetch commit:${commitSHA}`, error); + } + return ''; +} diff --git a/src/app/api/native/download/route.ts b/src/app/api/native/download/route.ts new file mode 100644 index 000000000..e22587c22 --- /dev/null +++ b/src/app/api/native/download/route.ts @@ -0,0 +1,64 @@ +// src/app/api/native/download/route.ts +'use server'; +import { NextRequest, NextResponse } from 'next/server'; +import { spawn } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import * as git from 'isomorphic-git'; + +const LOCAL_TAXONOMY_ROOT_DIR = process.env.NEXT_PUBLIC_LOCAL_TAXONOMY_ROOT_DIR || `${process.env.HOME}/.instructlab-ui`; + +export async function POST(req: NextRequest) { + const rootDir = LOCAL_TAXONOMY_ROOT_DIR; + if (!rootDir) { + return NextResponse.json({ error: 'Failed to find the local taxonomy that contains the contribution.' }, { status: 500 }); + } + const { branchName } = await req.json(); + + const taxonomyDir = path.join(rootDir, 'taxonomy'); + try { + await fs.promises.access(taxonomyDir, fs.constants.R_OK); + } catch { + return NextResponse.json({ error: 'Taxonomy directory not found or not readable' }, { status: 500 }); + } + + // Checkout the new branch + await git.checkout({ fs, dir: taxonomyDir, ref: branchName }); + + // Spawn tar to write gzipped archive to stdout + const tar = spawn('tar', ['-czf', '-', '-C', rootDir, 'taxonomy'], { + stdio: ['ignore', 'pipe', 'inherit'] + }); + + // If the client aborts, make sure to kill the tar process + req.signal.addEventListener('abort', () => { + tar.kill('SIGTERM'); + }); + + // Helper: convert Node.js Readable into a Web ReadableStream + const stream = new ReadableStream({ + start(controller) { + tar.stdout.on('data', (chunk: Buffer) => { + controller.enqueue(chunk); + }); + tar.stdout.on('end', () => { + controller.close(); + }); + tar.stdout.on('error', (err) => { + controller.error(err); + }); + }, + cancel() { + tar.kill('SIGTERM'); + } + }); + + return new NextResponse(stream, { + status: 200, + headers: { + 'Content-Type': 'application/gzip', + 'Content-Disposition': `attachment`, + 'Cache-Control': 'no-store' + } + }); +} diff --git a/src/app/api/native/git/branches/route.ts b/src/app/api/native/git/branches/route.ts index 2d53b3317..b95c88e40 100644 --- a/src/app/api/native/git/branches/route.ts +++ b/src/app/api/native/git/branches/route.ts @@ -3,11 +3,11 @@ import { NextRequest, NextResponse } from 'next/server'; import * as git from 'isomorphic-git'; import fs from 'fs'; import path from 'path'; +import { findTaxonomyRepoPath } from '@/app/api/native/utils'; // Get the repository path from the environment variable const LOCAL_TAXONOMY_ROOT_DIR = process.env.NEXT_PUBLIC_LOCAL_TAXONOMY_ROOT_DIR || `${process.env.HOME}/.instructlab-ui`; const REMOTE_TAXONOMY_ROOT_DIR = process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || ''; -const REMOTE_TAXONOMY_REPO_CONTAINER_MOUNT_DIR = '/tmp/.instructlab-ui'; interface CommitDetails { message: string; @@ -76,23 +76,10 @@ export async function POST(req: NextRequest) { } if (action === 'publish') { - let remoteTaxonomyRepoDirFinal: string = ''; + const remoteTaxonomyRepoDirFinal: string = findTaxonomyRepoPath(); - const remoteTaxonomyRepoContainerMountDir = path.join(REMOTE_TAXONOMY_REPO_CONTAINER_MOUNT_DIR, '/taxonomy'); - const remoteTaxonomyRepoDir = path.join(REMOTE_TAXONOMY_ROOT_DIR, '/taxonomy'); - - // Check if there is taxonomy repository mounted in the container - if (fs.existsSync(remoteTaxonomyRepoContainerMountDir) && fs.readdirSync(remoteTaxonomyRepoContainerMountDir).length !== 0) { - remoteTaxonomyRepoDirFinal = remoteTaxonomyRepoContainerMountDir; - console.log('Remote taxonomy repository ', remoteTaxonomyRepoDir, ' is mounted at:', remoteTaxonomyRepoDirFinal); - } else { - // If remote taxonomy is not mounted, it means it's local deployment and we can directly use the paths - if (fs.existsSync(remoteTaxonomyRepoDir) && fs.readdirSync(remoteTaxonomyRepoDir).length !== 0) { - remoteTaxonomyRepoDirFinal = remoteTaxonomyRepoDir; - } - } if (remoteTaxonomyRepoDirFinal === '') { - return NextResponse.json({ error: 'Remote taxonomy repository path does not exist.' }, { status: 400 }); + return NextResponse.json({ error: 'Unable to locate taxonomy repository path.' }, { status: 400 }); } console.log('Remote taxonomy repository path:', remoteTaxonomyRepoDirFinal); @@ -257,7 +244,7 @@ async function handlePublish(branchName: string, localTaxonomyDir: string, remot return NextResponse.json({ error: 'Invalid contribution name for publish' }, { status: 400 }); } - console.log(`Publishing contribution from ${branchName} to remote taxonomy repo at ${REMOTE_TAXONOMY_ROOT_DIR}/taxonomy`); + console.log(`Publishing contribution from ${branchName} to remote taxonomy repo at ${remoteTaxonomyDir}`); const changes = await findDiff(branchName, localTaxonomyDir); // Check if there are any changes to publish, create a new branch at remoteTaxonomyDir and diff --git a/src/app/api/native/git/knowledge-files/route.ts b/src/app/api/native/git/knowledge-files/route.ts deleted file mode 100644 index 55d1626c0..000000000 --- a/src/app/api/native/git/knowledge-files/route.ts +++ /dev/null @@ -1,252 +0,0 @@ -// src/app/api/native/git/knowledge-files/route.ts - -'use server'; -import { NextRequest, NextResponse } from 'next/server'; -import * as git from 'isomorphic-git'; -import fs from 'fs'; -import path from 'path'; -import http from 'isomorphic-git/http/node'; - -// Constants for repository paths -const TAXONOMY_DOCS_ROOT_DIR = process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || ''; -const TAXONOMY_DOCS_CONTAINER_MOUNT_DIR = '/tmp/.instructlab-ui'; -const TAXONOMY_KNOWLEDGE_DOCS_REPO_URL = - process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs'; -const BASE_BRANCH = 'main'; - -// Interface for the response -interface KnowledgeFile { - filename: string; - content: string; - commitSha: string; - commitDate: string; -} - -function findTaxonomyDocRepoPath(): string { - // Check the location of the taxonomy docs repository . - let remoteTaxonomyDocsRepoDirFinal: string = ''; - // Check if the taxonomy docs repo directory is mounted in the container (for container deployment) or present locally (for local deployment). - const remoteTaxonomyDocsRepoContainerMountDir = path.join(TAXONOMY_DOCS_CONTAINER_MOUNT_DIR, '/taxonomy-knowledge-docs'); - const remoteTaxonomyDocsRepoDir = path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy-knowledge-docs'); - if (fs.existsSync(remoteTaxonomyDocsRepoContainerMountDir) && fs.readdirSync(remoteTaxonomyDocsRepoContainerMountDir).length !== 0) { - remoteTaxonomyDocsRepoDirFinal = TAXONOMY_DOCS_CONTAINER_MOUNT_DIR; - } else { - if (fs.existsSync(remoteTaxonomyDocsRepoDir) && fs.readdirSync(remoteTaxonomyDocsRepoDir).length !== 0) { - remoteTaxonomyDocsRepoDirFinal = TAXONOMY_DOCS_ROOT_DIR; - } - } - if (remoteTaxonomyDocsRepoDirFinal === '') { - return ''; - } - - const taxonomyDocsDirectoryPath = path.join(remoteTaxonomyDocsRepoDirFinal, '/taxonomy-knowledge-docs'); - return taxonomyDocsDirectoryPath; -} - -/** - * Function to retrieve knowledge files from a specific branch. - * @param branchName - The name of the branch to retrieve files from. - * @returns An array of KnowledgeFile objects. - */ -const getKnowledgeFiles = async (branchName: string): Promise => { - const REPO_DIR = findTaxonomyDocRepoPath(); - - // Ensure the repository path exists - if (!fs.existsSync(REPO_DIR)) { - throw new Error('Repository path does not exist.'); - } - - // Check if the branch exists - const branches = await git.listBranches({ fs, dir: REPO_DIR }); - if (!branches.includes(branchName)) { - throw new Error(`Branch "${branchName}" does not exist.`); - } - - // Checkout the specified branch - await git.checkout({ fs, dir: REPO_DIR, ref: branchName }); - - // Read all files in the repository root directory - const allFiles = fs.readdirSync(REPO_DIR); - - // Filter for Markdown files only - const markdownFiles = allFiles.filter((file) => path.extname(file).toLowerCase() === '.md'); - - const knowledgeFiles: KnowledgeFile[] = []; - - for (const file of markdownFiles) { - const filePath = path.join(REPO_DIR, file); - - // Check if the file is a regular file - const stat = fs.statSync(filePath); - if (!stat.isFile()) { - continue; - } - - try { - // Retrieve the latest commit SHA for the file on the specified branch - const logs = await git.log({ - fs, - dir: REPO_DIR, - ref: branchName, - filepath: file, - depth: 1 // Only the latest commit - }); - - if (logs.length === 0) { - // No commits found for this file; skip it - continue; - } - - const latestCommit = logs[0]; - const commitSha = latestCommit.oid; - const commitDate = new Date(latestCommit.commit.committer.timestamp * 1000).toISOString(); - - // Read the file content - const fileContent = fs.readFileSync(filePath, 'utf-8'); - - knowledgeFiles.push({ - filename: file, - content: fileContent, - commitSha: commitSha, - commitDate: commitDate - }); - } catch (error) { - console.error(`Failed to retrieve commit for file ${file}:`, error); - // Skip files that cause errors - continue; - } - } - - return knowledgeFiles; -}; - -/** - * Handler for GET requests to read the knowledge files and their content. - */ -const getKnowledgeFilesHandler = async (): Promise => { - try { - const knowledgeFiles = await getKnowledgeFiles(BASE_BRANCH); - return NextResponse.json({ files: knowledgeFiles }, { status: 200 }); - } catch (error) { - console.error('Failed to process GET request to fetch knowledge files:', error); - return NextResponse.json({ error: (error as Error).message }, { status: 500 }); - } -}; - -/** - * GET handler to retrieve knowledge files from the taxonomy-knowledge-doc main branch. - */ -export async function GET() { - return await getKnowledgeFilesHandler(); -} - -/** - * POST handler to commit knowledge files to taxonomy-knowledge-doc repo's main branch. - */ -export async function POST(req: NextRequest) { - try { - const body = await req.json(); - const { files } = body; - const docsRepoUrl = await cloneTaxonomyDocsRepo(); - - // If the repository was not cloned, return an error - if (!docsRepoUrl) { - return NextResponse.json({ error: 'Failed to clone taxonomy knowledge docs repository' }, { status: 500 }); - } - - const timestamp = new Date().toISOString().replace(/[-:.]/g, '').replace('T', 'T').slice(0, -1); - const filesWithTimestamp = files.map((file: { fileName: string; fileContent: string }) => { - const [name, extension] = file.fileName.split(/\.(?=[^.]+$)/); - return { - fileName: `${name}-${timestamp}.${extension}`, - fileContent: file.fileContent - }; - }); - - // Write the files to the repository - const docsRepoUrlTmp = path.join(docsRepoUrl, '/'); - for (const file of filesWithTimestamp) { - const filePath = path.join(docsRepoUrlTmp, file.fileName); - console.log(`Writing file to ${filePath} in taxonomy knowledge docs repository.`); - fs.writeFileSync(filePath, file.fileContent); - } - - // Checkout the main branch - await git.checkout({ fs, dir: docsRepoUrl, ref: 'main' }); - - // Stage the files - await git.add({ fs, dir: docsRepoUrl, filepath: '.' }); - - // Commit the files - const commitSha = await git.commit({ - fs, - dir: docsRepoUrl, - author: { name: 'instructlab-ui', email: 'ui@instructlab.ai' }, - message: `Add files: ${files - .map((file: { fileName: string; fileContent: string }) => file.fileName) - .join(', ')}\n\nSigned-off-by: ui@instructlab.ai` - }); - - console.log(`Successfully committed files to taxonomy knowledge docs repository with commit SHA: ${commitSha}`); - - const origTaxonomyDocsRepoDir = path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy-knowledge-docs'); - return NextResponse.json( - { - repoUrl: origTaxonomyDocsRepoDir, - commitSha, - documentNames: filesWithTimestamp.map((file: { fileName: string }) => file.fileName) - }, - { status: 201 } - ); - } catch (error) { - console.error('Failed to upload knowledge documents:', error); - return NextResponse.json({ error: 'Failed to upload knowledge documents' }, { status: 500 }); - } -} - -async function cloneTaxonomyDocsRepo() { - // Check the location of the taxonomy repository and create the taxonomy-knowledge-doc repository parallel to that. - let remoteTaxonomyRepoDirFinal: string = ''; - // Check if directory pointed by remoteTaxonomyRepoDir exists and not empty - const remoteTaxonomyRepoContainerMountDir = path.join(TAXONOMY_DOCS_CONTAINER_MOUNT_DIR, '/taxonomy'); - const remoteTaxonomyRepoDir = path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy'); - if (fs.existsSync(remoteTaxonomyRepoContainerMountDir) && fs.readdirSync(remoteTaxonomyRepoContainerMountDir).length !== 0) { - remoteTaxonomyRepoDirFinal = TAXONOMY_DOCS_CONTAINER_MOUNT_DIR; - } else { - if (fs.existsSync(remoteTaxonomyRepoDir) && fs.readdirSync(remoteTaxonomyRepoDir).length !== 0) { - remoteTaxonomyRepoDirFinal = TAXONOMY_DOCS_ROOT_DIR; - } - } - if (remoteTaxonomyRepoDirFinal === '') { - return null; - } - - const taxonomyDocsDirectoryPath = path.join(remoteTaxonomyRepoDirFinal, '/taxonomy-knowledge-docs'); - - if (fs.existsSync(taxonomyDocsDirectoryPath)) { - console.log(`Using existing taxonomy knowledge docs repository at ${TAXONOMY_DOCS_ROOT_DIR}/taxonomy-knowledge-docs.`); - return taxonomyDocsDirectoryPath; - } else { - console.log(`Taxonomy knowledge docs repository not found at ${TAXONOMY_DOCS_ROOT_DIR}/taxonomy-knowledge-docs. Cloning...`); - } - - try { - await git.clone({ - fs, - http, - dir: taxonomyDocsDirectoryPath, - url: TAXONOMY_KNOWLEDGE_DOCS_REPO_URL, - singleBranch: true - }); - - // Include the full path in the response for client display. Path displayed here is the one - // that user set in the environment variable. - console.log(`Taxonomy knowledge docs repository cloned successfully to ${remoteTaxonomyRepoDir}.`); - // Return the path that the UI sees (direct or mounted) - return taxonomyDocsDirectoryPath; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'; - console.error(`Failed to clone taxonomy docs repository: ${errorMessage}`); - return null; - } -} diff --git a/src/app/api/native/knowledge-files/route.ts b/src/app/api/native/knowledge-files/route.ts new file mode 100644 index 000000000..9952db74b --- /dev/null +++ b/src/app/api/native/knowledge-files/route.ts @@ -0,0 +1,252 @@ +// src/app/api/native/knowledge-files/route.ts + +'use server'; +import { NextRequest, NextResponse } from 'next/server'; +import * as git from 'isomorphic-git'; +import fs from 'fs'; +import path from 'path'; +import { cloneTaxonomyDocsRepo, findTaxonomyDocRepoPath, TAXONOMY_DOCS_ROOT_DIR } from '@/app/api/native/utils'; + +const BASE_BRANCH = 'main'; + +// Interface for the response +interface KnowledgeFile { + filename: string; + content: string; + commitSha: string; + commitDate: string; +} + +/** + * Function to retrieve knowledge files from a specific branch. + * @param contributionCommitSHA - Retrieve files from the SHA. + * @returns An array of KnowledgeFile objects. + */ +const getKnowledgeFiles = async (contributionCommitSHA: string): Promise => { + const REPO_DIR = findTaxonomyDocRepoPath(); + + // Ensure the repository path exists + if (!fs.existsSync(REPO_DIR)) { + throw new Error('Repository path does not exist.'); + } + + // Check if the branch exists + const branches = await git.listBranches({ fs, dir: REPO_DIR }); + if (!branches.includes(BASE_BRANCH)) { + throw new Error(`Branch "${BASE_BRANCH}" does not exist.`); + } + + // Checkout the specified branch + await git.checkout({ fs, dir: REPO_DIR, ref: BASE_BRANCH }); + + // Read all files in the repository root directory + + const allFiles = await fs.promises.readdir(REPO_DIR, { recursive: true }); + + // Filter for Markdown files only + const markdownFiles = allFiles.filter((file) => path.extname(file).toLowerCase() === '.md'); + const knowledgeFiles: KnowledgeFile[] = []; + + for (const file of markdownFiles) { + const filePath = path.join(REPO_DIR, file); + + // Check if the file is a regular file + const stat = fs.statSync(filePath); + if (!stat.isFile()) { + continue; + } + + try { + // Retrieve the latest commit SHA for the file on the specified branch + const logs = await git.log({ + fs, + dir: REPO_DIR, + ref: BASE_BRANCH, + filepath: file, + depth: 1 // Only the latest commit + }); + + if (logs.length === 0) { + // No commits found for this file; skip it + continue; + } + + const latestCommit = logs[0]; + const commitSha = latestCommit.oid; + + if (contributionCommitSHA !== '') { + if (contributionCommitSHA == commitSha) { + const commitDate = new Date(latestCommit.commit.committer.timestamp * 1000).toISOString(); + + // Read the file content + const fileContent = fs.readFileSync(filePath, 'utf-8'); + + knowledgeFiles.push({ + filename: path.basename(file), + content: fileContent, + commitSha: commitSha, + commitDate: commitDate + }); + } + } else { + const commitDate = new Date(latestCommit.commit.committer.timestamp * 1000).toISOString(); + + // Read the file content + const fileContent = fs.readFileSync(filePath, 'utf-8'); + knowledgeFiles.push({ + filename: path.basename(file), + content: fileContent, + commitSha: commitSha, + commitDate: commitDate + }); + } + } catch (error) { + console.error(`Failed to retrieve commit for file ${file}:`, error); + // Skip files that cause errors + continue; + } + } + + return knowledgeFiles; +}; + +/** + * Handler for GET requests to read the knowledge files and their content. + */ +const getKnowledgeFilesHandler = async (commitSHA: string): Promise => { + try { + const knowledgeFiles = await getKnowledgeFiles(commitSHA); + return NextResponse.json({ files: knowledgeFiles }, { status: 200 }); + } catch (error) { + console.error('Failed to process GET request to fetch knowledge files:', error); + return NextResponse.json({ error: (error as Error).message }, { status: 500 }); + } +}; + +/** + * GET handler to retrieve knowledge files from the taxonomy-knowledge-doc main branch. + */ +export async function GET(request: NextRequest) { + const url = new URL(request.url); + const commitSHA = url.searchParams.get('commitSHA'); + return await getKnowledgeFilesHandler(commitSHA ? commitSHA : ''); +} + +/** + * POST handler to commit knowledge files to taxonomy-knowledge-doc repo's main branch. + */ +export async function POST(req: NextRequest) { + try { + const body = await req.json(); + const { branchName, currentCommitSHA, newFiles, updatedExistingFiles } = body; + const docsRepoPath = await cloneTaxonomyDocsRepo(); + + // If the repository was not cloned, return an error + if (!docsRepoPath) { + return NextResponse.json({ error: 'Failed to clone taxonomy knowledge docs repository' }, { status: 500 }); + } + + // Checkout the main branch + await git.checkout({ fs, dir: docsRepoPath, ref: 'main' }); + + const subDirectory = new Date().toISOString().replace(/[-:.]/g, '').replace('T', 'T').slice(0, -1); + + const newDocsDirPath = path.join(docsRepoPath, branchName, subDirectory); + + let oldDocsDirPath = ''; + + if (currentCommitSHA != '') { + const commit = await git.readCommit({ fs, dir: docsRepoPath, oid: currentCommitSHA }); + + const existingSubDirectory = extractSubDirectoryName(commit.commit.message); + if (existingSubDirectory.length === 0) { + console.warn('No subdirectory exist. Either the commit sha is invalid, or the docs are manually deleted.'); + console.log('Continue uploading the newly provided document.'); + } else { + console.log(`Document sub directory exist for the contribution ${branchName}:`, existingSubDirectory); + oldDocsDirPath = path.join(docsRepoPath, branchName, existingSubDirectory); + } + } + + if (!fs.existsSync(newDocsDirPath)) { + fs.mkdirSync(newDocsDirPath, { recursive: true }); + console.log(`New sub directory ${newDocsDirPath} created successfully.`); + } else { + console.log(`Failed to created new sub directory ${docsRepoPath}`); + return NextResponse.json({ error: 'Failed to upload documents.' }, { status: 500 }); + } + + if (oldDocsDirPath != '') { + const existingFiles = fs.readdirSync(oldDocsDirPath); + // Copy existing document to new sub directory + for (const existingFile of existingFiles) { + if (updatedExistingFiles.some((file: { fileName: string; fileContent: string }) => file.fileName === existingFile)) { + const sourcePath = path.join(oldDocsDirPath, existingFile); + const targetPath = path.join(newDocsDirPath, existingFile); + + const stat = fs.statSync(sourcePath); + + if (stat.isFile()) { + fs.copyFileSync(sourcePath, targetPath); + console.log(`Copied file: ${sourcePath} -> ${targetPath}`); + } else { + console.error('Unexpected sub directory found in the existing document directory. Skipping it. : ', sourcePath); + } + } else { + console.log(`${existingFile} is either deleted or replaced with newer version`); + } + } + + //Delete the old directory + fs.rmdirSync(oldDocsDirPath, { recursive: true }); + } + + // Write the files to the repository + for (const file of newFiles) { + const filePath = path.join(newDocsDirPath, file.fileName); + console.log(`Writing file to ${filePath} in taxonomy knowledge docs repository.`); + fs.writeFileSync(filePath, file.fileContent); + } + + const finalFiles = fs.readdirSync(newDocsDirPath); + const filenames = finalFiles.map((file) => path.join(branchName, subDirectory, file)); + + // Stage the files + await git.add({ fs, dir: docsRepoPath, filepath: '.' }); + await git.remove({ fs, dir: docsRepoPath, filepath: '.' }); + + // Commit the files + const commitSha = await git.commit({ + fs, + dir: docsRepoPath, + author: { name: 'instructlab-ui', email: 'ui@instructlab.ai' }, + message: `Contribution Name: ${branchName}\n\nSub-Directory:${subDirectory}\n\nDocument uploaded: ${finalFiles}\n\nSigned-off-by: ui@instructlab.ai` + }); + + console.log(`Successfully committed documents to taxonomy knowledge docs repository with commit SHA: ${commitSha}`); + + const origTaxonomyDocsRepoDir = path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy-knowledge-docs'); + return NextResponse.json( + { + repoUrl: origTaxonomyDocsRepoDir, + commitSha, + documentNames: filenames + }, + { status: 201 } + ); + } catch (error) { + console.error('Failed to upload knowledge documents:', error); + return NextResponse.json({ error: 'Failed to upload knowledge documents' }, { status: 500 }); + } +} + +/** + * Extract the sub directory name where documents are stored. + * @param commitMessage Commit message from the provided SHA + * @returns + */ +function extractSubDirectoryName(commitMessage: string): string { + const match = commitMessage.match(/Sub-Directory:(.+)/); + if (!match) return ''; + return match[1].trim(); +} diff --git a/src/app/api/native/pr/knowledge/route.ts b/src/app/api/native/pr/knowledge/route.ts index 6a2c93637..7187f8c9d 100644 --- a/src/app/api/native/pr/knowledge/route.ts +++ b/src/app/api/native/pr/knowledge/route.ts @@ -8,25 +8,20 @@ import path from 'path'; import { dumpYaml } from '@/utils/yamlConfig'; import { KnowledgeYamlData } from '@/types'; import yaml from 'js-yaml'; +import { prInfoFromSummary } from '@/app/api/github/utils'; // Define paths and configuration const LOCAL_TAXONOMY_ROOT_DIR = process.env.NEXT_PUBLIC_LOCAL_TAXONOMY_ROOT_DIR || `${process.env.HOME}/.instructlab-ui`; const KNOWLEDGE_DIR = 'knowledge'; +// This API submit the knowledge contribution to the local cached taxonomy and not to the remote taxonomy. export async function POST(req: NextRequest) { const REPO_DIR = path.join(LOCAL_TAXONOMY_ROOT_DIR, '/taxonomy'); try { // Extract the data from the request body const { action, branchName, content, name, email, submissionSummary, filePath, oldFilesPath } = await req.json(); - let knowledgeBranchName; - if (action == 'update' && branchName != '') { - knowledgeBranchName = branchName; - } else { - knowledgeBranchName = `knowledge-contribution-${Date.now()}`; - } - // Parse the YAML string into an object const knowledgeData = yaml.load(content) as KnowledgeYamlData; @@ -41,11 +36,11 @@ export async function POST(req: NextRequest) { // Create a new branch if the knowledge is pushed for first time if (action != 'update') { - await git.branch({ fs, dir: REPO_DIR, ref: knowledgeBranchName }); + await git.branch({ fs, dir: REPO_DIR, ref: branchName }); } // Checkout the new branch - await git.checkout({ fs, dir: REPO_DIR, ref: knowledgeBranchName }); + await git.checkout({ fs, dir: REPO_DIR, ref: branchName }); const newYamlFilePath = path.join(KNOWLEDGE_DIR, filePath, 'qna.yaml'); @@ -74,10 +69,11 @@ export async function POST(req: NextRequest) { } // Commit the changes + const { commitMessage } = prInfoFromSummary(submissionSummary); await git.commit({ fs, dir: REPO_DIR, - message: `${submissionSummary}\n\nSigned-off-by: ${name} <${email}>`, + message: `${commitMessage}\n\nSigned-off-by: ${name} <${email}>`, author: { name: name, email: email @@ -86,8 +82,8 @@ export async function POST(req: NextRequest) { }); // Respond with success message and branch name - console.log(`Knowledge contribution submitted successfully to local taxonomy repo. Submission Name is ${knowledgeBranchName}.`); - return NextResponse.json({ message: 'Knowledge contribution submitted successfully.', branch: knowledgeBranchName }, { status: 201 }); + console.log(`Knowledge contribution submitted successfully to local taxonomy repo. Submission Name is ${branchName}.`); + return NextResponse.json({ message: 'Knowledge contribution submitted successfully.', branch: branchName }, { status: 201 }); } catch (error) { console.error(`Failed to submit knowledge contribution to local taxonomy repo:`, error); return NextResponse.json({ error: 'Failed to submit knowledge contribution.' }, { status: 500 }); diff --git a/src/app/api/native/pr/skill/route.ts b/src/app/api/native/pr/skill/route.ts index f4446e9e5..8b5a47d2d 100644 --- a/src/app/api/native/pr/skill/route.ts +++ b/src/app/api/native/pr/skill/route.ts @@ -7,24 +7,19 @@ import path from 'path'; import yaml from 'js-yaml'; import { SkillYamlData } from '@/types'; import { dumpYaml } from '@/utils/yamlConfig'; +import { prInfoFromSummary } from '@/app/api/github/utils'; // Define paths and configuration const LOCAL_TAXONOMY_ROOT_DIR = process.env.NEXT_PUBLIC_LOCAL_TAXONOMY_ROOT_DIR || `${process.env.HOME}/.instructlab-ui`; const SKILLS_DIR = 'compositional_skills'; +// This API submit the skill contribution to the local cached taxonomy and not to the remote taxonomy. export async function POST(req: NextRequest) { const REPO_DIR = path.join(LOCAL_TAXONOMY_ROOT_DIR, '/taxonomy'); try { - // Extract the QnA data from the request body TODO: what is documentOutline? - const { action, branchName, content, name, email, submissionSummary, documentOutline, filePath, oldFilesPath } = await req.json(); // eslint-disable-line @typescript-eslint/no-unused-vars - - let skillBranchName; - if (action == 'update' && branchName != '') { - skillBranchName = branchName; - } else { - skillBranchName = `skill-contribution-${Date.now()}`; - } + // Extract the QnA data from the request body + const { action, branchName, content, name, email, submissionSummary, filePath, oldFilesPath } = await req.json(); // eslint-disable-line @typescript-eslint/no-unused-vars const skillData = yaml.load(content) as SkillYamlData; const yamlString = dumpYaml(skillData); @@ -37,11 +32,11 @@ export async function POST(req: NextRequest) { // Create a new branch if the skill is pushed for first time if (action != 'update') { - await git.branch({ fs, dir: REPO_DIR, ref: skillBranchName }); + await git.branch({ fs, dir: REPO_DIR, ref: branchName }); } // Checkout the new branch - await git.checkout({ fs, dir: REPO_DIR, ref: skillBranchName }); + await git.checkout({ fs, dir: REPO_DIR, ref: branchName }); // Define file path const newYamlFilePath = path.join(SKILLS_DIR, filePath, 'qna.yaml'); @@ -71,10 +66,11 @@ export async function POST(req: NextRequest) { } // Commit files + const { commitMessage } = prInfoFromSummary(submissionSummary); await git.commit({ fs, dir: REPO_DIR, - message: `${submissionSummary}\n\nSigned-off-by: ${name} <${email}>`, + message: `${commitMessage}\n\nSigned-off-by: ${name} <${email}>`, author: { name: name, email: email @@ -83,8 +79,8 @@ export async function POST(req: NextRequest) { }); // Respond with success - console.log('Skill contribution submitted successfully. Submission name is ', skillBranchName); - return NextResponse.json({ message: 'Skill contribution submitted successfully.', branch: skillBranchName }, { status: 201 }); + console.log('Skill contribution submitted successfully. Submission name is ', branchName); + return NextResponse.json({ message: 'Skill contribution submitted successfully.', branch: branchName }, { status: 201 }); } catch (error) { console.error('Failed to create local branch and commit:', error); return NextResponse.json({ error: 'Failed to submit skill contribution.' }, { status: 500 }); diff --git a/src/app/api/native/tree/route.ts b/src/app/api/native/tree/route.ts new file mode 100644 index 000000000..889e087f1 --- /dev/null +++ b/src/app/api/native/tree/route.ts @@ -0,0 +1,41 @@ +// src/app/api/native/tree/route.ts +import { NextRequest, NextResponse } from 'next/server'; +import path from 'path'; +import * as fs from 'fs'; +import { findTaxonomyRepoPath } from '@/app/api/native/utils'; + +const SKILLS = 'compositional_skills'; +const KNOWLEDGE = 'knowledge'; + +export async function POST(req: NextRequest) { + const body = await req.json(); + const { root_path, dir_name } = body; + + try { + const taxonomyRootPath = findTaxonomyRepoPath(); + let dirPath = ''; + if (root_path === 'skills') { + dirPath = path.join(taxonomyRootPath, SKILLS, dir_name); + } else { + dirPath = path.join(taxonomyRootPath, KNOWLEDGE, dir_name); + } + const dirs = getFirstLevelDirectories(dirPath); + return NextResponse.json({ data: dirs }, { status: 201 }); + } catch (error) { + console.error('Failed to get the tree for path:', root_path, error); + return NextResponse.json({ error: 'Failed to get the tree for path' }, { status: 500 }); + } +} + +function getFirstLevelDirectories(directoryPath: string): string[] { + try { + return fs + .readdirSync(directoryPath) + .map((name) => path.join(directoryPath, name)) + .filter((source) => fs.statSync(source).isDirectory()) + .map((dir) => path.basename(dir)); + } catch (error) { + console.error('Error reading directory:', error); + return []; + } +} diff --git a/src/app/api/native/utils.ts b/src/app/api/native/utils.ts new file mode 100644 index 000000000..0b915392a --- /dev/null +++ b/src/app/api/native/utils.ts @@ -0,0 +1,94 @@ +// Constants for repository paths +import path from 'path'; +import fs from 'fs'; +import * as git from 'isomorphic-git'; +import http from 'isomorphic-git/http/node'; + +export const TAXONOMY_DOCS_ROOT_DIR = process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || ''; +export const TAXONOMY_DOCS_CONTAINER_MOUNT_DIR = '/tmp/.instructlab-ui'; +export const TAXONOMY_KNOWLEDGE_DOCS_REPO_URL = + process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs'; + +export const cloneTaxonomyDocsRepo = async (): Promise => { + // Check the location of the taxonomy repository and create the taxonomy-knowledge-doc repository parallel to that. + const remoteTaxonomyRepoDirFinal: string = findTaxonomyRepoPath(); + if (remoteTaxonomyRepoDirFinal === '') { + console.warn('Unable to locate taxonomy directory.'); + return null; + } + + const taxonomyDocsDirectoryPath = path.join(remoteTaxonomyRepoDirFinal, '/taxonomy-knowledge-docs'); + + if (fs.existsSync(taxonomyDocsDirectoryPath)) { + console.log(`Using existing taxonomy knowledge docs repository at ${TAXONOMY_DOCS_ROOT_DIR}/taxonomy-knowledge-docs.`); + return taxonomyDocsDirectoryPath; + } else { + console.log(`Taxonomy knowledge docs repository not found at ${TAXONOMY_DOCS_ROOT_DIR}/taxonomy-knowledge-docs. Cloning...`); + } + + try { + await git.clone({ + fs, + http, + dir: taxonomyDocsDirectoryPath, + url: TAXONOMY_KNOWLEDGE_DOCS_REPO_URL, + singleBranch: true + }); + + // Include the full path in the response for client display. Path displayed here is the one + // that user set in the environment variable. + console.log(`Taxonomy knowledge docs repository cloned successfully to ${path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy')}.`); + // Return the path that the UI sees (direct or mounted) + return taxonomyDocsDirectoryPath; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred'; + console.error(`Failed to clone taxonomy docs repository: ${errorMessage}`); + return null; + } +}; + +// Locate the taxonomy-knowledge-docs directory. UI can be deployed locally on host as well as in containers. +// It checks if the directory is mounted at the specified location in container and not empty. +// If it doesn't find the taxonomy-knowledge-docs repo in mounted directory, it checks locally on host. +export const findTaxonomyDocRepoPath = (): string => { + // Check the location of the taxonomy docs repository . + let remoteTaxonomyDocsRepoDirFinal: string = ''; + // Check if the taxonomy docs repo directory is mounted in the container (for container deployment) or present locally (for local deployment). + const remoteTaxonomyDocsRepoContainerMountDir = path.join(TAXONOMY_DOCS_CONTAINER_MOUNT_DIR, '/taxonomy-knowledge-docs'); + const remoteTaxonomyDocsRepoDir = path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy-knowledge-docs'); + if (fs.existsSync(remoteTaxonomyDocsRepoContainerMountDir) && fs.readdirSync(remoteTaxonomyDocsRepoContainerMountDir).length !== 0) { + remoteTaxonomyDocsRepoDirFinal = TAXONOMY_DOCS_CONTAINER_MOUNT_DIR; + } else { + if (fs.existsSync(remoteTaxonomyDocsRepoDir) && fs.readdirSync(remoteTaxonomyDocsRepoDir).length !== 0) { + remoteTaxonomyDocsRepoDirFinal = TAXONOMY_DOCS_ROOT_DIR; + } + } + if (remoteTaxonomyDocsRepoDirFinal === '') { + return ''; + } + + const taxonomyDocsDirectoryPath = path.join(remoteTaxonomyDocsRepoDirFinal, '/taxonomy-knowledge-docs'); + return taxonomyDocsDirectoryPath; +}; + +// Locate the taxonomy directory. UI can be deployed locally on host as well as in containers. +// It checks if the directory is mounted at the specified location in container and not empty. +// If it doesn't find the taxonomy repo in mounted directory, it checks locally on host. +export const findTaxonomyRepoPath = (): string => { + let remoteTaxonomyRepoDirFinal: string = ''; + // Check if directory pointed by remoteTaxonomyRepoDir exists and not empty + const remoteTaxonomyRepoContainerMountDir = path.join(TAXONOMY_DOCS_CONTAINER_MOUNT_DIR, '/taxonomy'); + const remoteTaxonomyRepoDir = path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy'); + if (fs.existsSync(remoteTaxonomyRepoContainerMountDir) && fs.readdirSync(remoteTaxonomyRepoContainerMountDir).length !== 0) { + remoteTaxonomyRepoDirFinal = TAXONOMY_DOCS_CONTAINER_MOUNT_DIR; + console.log('Remote taxonomy repository ', remoteTaxonomyRepoDir, ' is mounted at:', remoteTaxonomyRepoDirFinal); + } else { + if (fs.existsSync(remoteTaxonomyRepoDir) && fs.readdirSync(remoteTaxonomyRepoDir).length !== 0) { + remoteTaxonomyRepoDirFinal = TAXONOMY_DOCS_ROOT_DIR; + } + } + if (remoteTaxonomyRepoDirFinal === '') { + return ''; + } + return path.join(remoteTaxonomyRepoDirFinal, '/taxonomy'); +}; diff --git a/src/app/api/playground/chat/route.ts b/src/app/api/playground/chat/route.ts index beb9f65ee..344779455 100644 --- a/src/app/api/playground/chat/route.ts +++ b/src/app/api/playground/chat/route.ts @@ -10,6 +10,7 @@ export async function POST(req: NextRequest) { const { question, systemRole } = await req.json(); const apiURL = req.nextUrl.searchParams.get('apiURL'); const modelName = req.nextUrl.searchParams.get('modelName'); + const apiKey = req.nextUrl.searchParams.get('apiKey'); if (!apiURL || !modelName) { return new NextResponse('Missing API URL or Model Name', { status: 400 }); @@ -26,16 +27,34 @@ export async function POST(req: NextRequest) { stream: true }; - const agent = new https.Agent({ - rejectUnauthorized: false - }); + let agent: https.Agent; + if (apiKey && apiKey != '') { + agent = new https.Agent({ + rejectUnauthorized: true + }); + } else { + agent = new https.Agent({ + rejectUnauthorized: false + }); + } + + let headers: HeadersInit; + if (apiKey && apiKey != '') { + headers = { + 'Content-Type': 'application/json', + Accept: 'text/event-stream', + Authorization: `Bearer: ${apiKey}` + }; + } else { + headers = { + 'Content-Type': 'application/json', + Accept: 'text/event-stream' + }; + } const chatResponse = await fetch(`${apiURL}/v1/chat/completions`, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - accept: 'application/json' - }, + headers: headers, body: JSON.stringify(requestData), agent: apiURL.startsWith('https') ? agent : undefined }); diff --git a/src/app/api/tree/route.ts b/src/app/api/tree/route.ts deleted file mode 100644 index 55f0901cd..000000000 --- a/src/app/api/tree/route.ts +++ /dev/null @@ -1,22 +0,0 @@ -// src/app/api/tree/route.ts -import axios from 'axios'; -import { NextRequest, NextResponse } from 'next/server'; -import path from 'path'; - -const PATH_SERVICE_URL = process.env.IL_PATH_SERVICE_URL || 'http://pathservice:4000'; - -export async function POST(req: NextRequest) { - const body = await req.json(); - const { root_path, dir_name } = body; - - try { - const apiBaseUrl = path.join(PATH_SERVICE_URL, '/tree'); - const response = await axios.get(apiBaseUrl + root_path, { - params: { dir_name: dir_name } - }); - return NextResponse.json({ data: response.data }, { status: 201 }); - } catch (error) { - console.error('Failed to get the tree for path:', root_path, error); - return NextResponse.json({ error: 'Failed to get the tree for path' }, { status: 500 }); - } -} diff --git a/src/app/contribute/knowledge/page.tsx b/src/app/contribute/knowledge/page.tsx index ea98c8617..f86119a5f 100644 --- a/src/app/contribute/knowledge/page.tsx +++ b/src/app/contribute/knowledge/page.tsx @@ -1,23 +1,46 @@ // src/app/contribute/knowledge/page.tsx 'use client'; +import React, { useEffect, useState } from 'react'; +import { Flex, Spinner } from '@patternfly/react-core'; +import { t_global_spacer_xl as XlSpacerSize } from '@patternfly/react-tokens'; import { AppLayout } from '@/components/AppLayout'; import KnowledgeFormGithub from '@/components/Contribute/Knowledge/Github'; import KnowledgeFormNative from '@/components/Contribute/Knowledge/Native'; -import { useEffect, useState } from 'react'; const KnowledgeFormPage: React.FunctionComponent = () => { const [deploymentType, setDeploymentType] = useState(); + const [loaded, setLoaded] = useState(); useEffect(() => { + let canceled = false; + const getEnvVariables = async () => { const res = await fetch('/api/envConfig'); const envConfig = await res.json(); - setDeploymentType(envConfig.DEPLOYMENT_TYPE); + if (!canceled) { + setDeploymentType(envConfig.DEPLOYMENT_TYPE); + setLoaded(true); + } }; + getEnvVariables(); + + return () => { + canceled = true; + }; }, []); - return {deploymentType === 'native' ? : }; + return ( + + {loaded ? ( + <>{deploymentType === 'native' ? : } + ) : ( + + + + )} + + ); }; export default KnowledgeFormPage; diff --git a/src/app/contribute/skill/page.tsx b/src/app/contribute/skill/page.tsx index b4d31d356..7ed1b1e44 100644 --- a/src/app/contribute/skill/page.tsx +++ b/src/app/contribute/skill/page.tsx @@ -17,7 +17,7 @@ const SkillFormPage: React.FunctionComponent = () => { getEnvVariables(); }, []); - return {deploymentType === 'native' ? : }; + return {deploymentType === 'native' ? : }; }; export default SkillFormPage; diff --git a/src/app/edit-submission/knowledge/github/[id]/page.tsx b/src/app/edit-submission/knowledge/github/[...slug]/page.tsx similarity index 58% rename from src/app/edit-submission/knowledge/github/[id]/page.tsx rename to src/app/edit-submission/knowledge/github/[...slug]/page.tsx index 22fc7639d..a16412f24 100644 --- a/src/app/edit-submission/knowledge/github/[id]/page.tsx +++ b/src/app/edit-submission/knowledge/github/[...slug]/page.tsx @@ -1,19 +1,18 @@ -// src/app/edit-submission/knowledge/[id]/page.tsx +// src/app/edit-submission/knowledge/github/[...slug]/page.tsx import * as React from 'react'; import { AppLayout } from '@/components/AppLayout'; import EditKnowledge from '@/components/Contribute/EditKnowledge/github/EditKnowledge'; type PageProps = { - params: Promise<{ id: string }>; + params: Promise<{ slug: string[] }>; }; const EditKnowledgePage = async ({ params }: PageProps) => { const resolvedParams = await params; - const prNumber = parseInt(resolvedParams.id, 10); return ( - - + + ); }; diff --git a/src/app/edit-submission/knowledge/native/[id]/page.tsx b/src/app/edit-submission/knowledge/native/[...slug]/page.tsx similarity index 52% rename from src/app/edit-submission/knowledge/native/[id]/page.tsx rename to src/app/edit-submission/knowledge/native/[...slug]/page.tsx index 617aa078f..745dedb9d 100644 --- a/src/app/edit-submission/knowledge/native/[id]/page.tsx +++ b/src/app/edit-submission/knowledge/native/[...slug]/page.tsx @@ -1,18 +1,17 @@ -// src/app/edit-submission/knowledge/[id]/page.tsx -import * as React from 'react'; +// src/app/edit-submission/knowledge/native/[...slug]/page.tsx import { AppLayout } from '@/components/AppLayout'; import EditKnowledgeNative from '@/components/Contribute/EditKnowledge/native/EditKnowledge'; +import * as React from 'react'; type PageProps = { - params: Promise<{ id: string }>; + params: Promise<{ slug: string[] }>; }; const EditKnowledgePage = async ({ params }: PageProps) => { - const branchName = await params; - + const contribution = await params; return ( - - + + ); }; diff --git a/src/app/edit-submission/skill/github/[id]/page.tsx b/src/app/edit-submission/skill/github/[...slug]/page.tsx similarity index 67% rename from src/app/edit-submission/skill/github/[id]/page.tsx rename to src/app/edit-submission/skill/github/[...slug]/page.tsx index aca0bbd50..015c38f7a 100644 --- a/src/app/edit-submission/skill/github/[id]/page.tsx +++ b/src/app/edit-submission/skill/github/[...slug]/page.tsx @@ -4,16 +4,15 @@ import { AppLayout } from '@/components/AppLayout'; import EditSkill from '@/components/Contribute/EditSkill/github/EditSkill'; type PageProps = { - params: Promise<{ id: string }>; + params: Promise<{ slug: string[] }>; }; const EditSkillPage = async ({ params }: PageProps) => { const resolvedParams = await params; - const prNumber = parseInt(resolvedParams.id, 10); return ( - - + + ); }; diff --git a/src/app/edit-submission/skill/native/[id]/page.tsx b/src/app/edit-submission/skill/native/[...slug]/page.tsx similarity index 60% rename from src/app/edit-submission/skill/native/[id]/page.tsx rename to src/app/edit-submission/skill/native/[...slug]/page.tsx index efb2ec130..1b79309e4 100644 --- a/src/app/edit-submission/skill/native/[id]/page.tsx +++ b/src/app/edit-submission/skill/native/[...slug]/page.tsx @@ -4,15 +4,15 @@ import { AppLayout } from '@/components/AppLayout'; import EditSkillNative from '@/components/Contribute/EditSkill/native/EditSkill'; type PageProps = { - params: Promise<{ id: string }>; + params: Promise<{ slug: string[] }>; }; const EditSkillPage = async ({ params }: PageProps) => { - const branchName = await params; + const contribution = await params; return ( - - + + ); }; diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 89ce187e4..e159f6e5a 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -7,7 +7,7 @@ import '@patternfly/react-styles/css/components/Menu/menu.css'; export const metadata = { title: 'InstructLab UI', - description: 'InstructLab UI' + description: 'User interface for creating InstructLab contributions' }; interface RootLayoutProps { diff --git a/src/app/login/LoginLinks.tsx b/src/app/login/LoginLinks.tsx new file mode 100644 index 000000000..8d9c78191 --- /dev/null +++ b/src/app/login/LoginLinks.tsx @@ -0,0 +1,32 @@ +import React from 'react'; +import './login-page.css'; +import { Content } from '@patternfly/react-core'; + +const LoginLinks: React.FC = () => ( +
+ + + GitHub + {' '} + |{' '} + + Collaborate + {' '} + |{' '} + + Code Of Conduct + + + + + Terms of use + {' '} + |{' '} + + Privacy Policy + + +
+); + +export default LoginLinks; diff --git a/src/app/login/devmodelogin.tsx b/src/app/login/devmodelogin.tsx deleted file mode 100644 index 63b972787..000000000 --- a/src/app/login/devmodelogin.tsx +++ /dev/null @@ -1,152 +0,0 @@ -// src/app/login/DevModeLogin.tsx -import React, { useState } from 'react'; -import { signIn } from 'next-auth/react'; -import './githublogin.css'; -import { Button, Content, Form, FormGroup, Grid, GridItem, HelperText, HelperTextItem, TextInput } from '@patternfly/react-core'; -import { GithubIcon } from '@patternfly/react-icons'; - -const DevModeLogin: React.FunctionComponent = () => { - const [, setShowHelperText] = useState(false); - const [username, setUsername] = useState(''); - const [isValidUsername, setIsValidUsername] = useState(true); - const [password, setPassword] = useState(''); - const [isValidPassword, setIsValidPassword] = useState(true); - - const handleLogin = async (e: React.FormEvent) => { - e.preventDefault(); - const result = await signIn('credentials', { redirect: false, username, password }); - if (result?.error) { - setShowHelperText(true); - setIsValidUsername(false); - setIsValidPassword(false); - } else { - window.location.href = '/dashboard'; - } - }; - - const handleUsernameChange = (_event: React.FormEvent, value: string) => { - setUsername(value); - }; - - const handlePasswordChange = (_event: React.FormEvent, value: string) => { - setPassword(value); - }; - - const handleGitHubLogin = () => { - signIn('github', { callbackUrl: '/dashboard' }); - }; - - return ( -
- - - - - Login locally with a username and password or via GitHub OAuth - - - - - Join the novel, community-based movement to create truly open-source LLMs - - -
- -
- - - {!isValidUsername && ( - - Invalid Username - - )} - - - - {!isValidPassword && ( - - Invalid password - - )} - - -
-
- - - - GitHub - {' '} - |{' '} - - Collaborate - {' '} - |{' '} - - Code Of Conduct - - - - - Terms of use - {' '} - |{' '} - - Privacy Policy - - - - - -
- ); -}; - -export default DevModeLogin; diff --git a/src/app/login/githublogin.css b/src/app/login/githublogin.css deleted file mode 100644 index a96f5da19..000000000 --- a/src/app/login/githublogin.css +++ /dev/null @@ -1,68 +0,0 @@ -a { - color: white; - text-decoration: underline; -} - -.login-page-background { - background-image: url('../../../public/InstructLab-Background-Image.svg'); - background-size: contain; - background-position: center; - background-repeat: no-repeat; - background-color: #454b9a; - min-height: 100vh; - display: flex; - align-items: center; - justify-content: center; - - overflow-y: auto; - overflow-x: auto; -} - -.login-container { - text-align: center; - font-size: medium !important; - width: 100%; - max-width: 1200px; - padding: 20px; -} - -.sign-in-text { - margin-top: 1rem; - color: white !important; - padding: 20px; - font-size: large !important; -} - -.description-text { - margin-top: 1rem; - color: white !important; - padding-bottom: 50px; - font-size: large !important; - font-style: italic; -} - -.urls-text { - margin-top: 1rem; - color: white; - padding: 20px; - font-size: large; -} - -.urls-text-medium { - margin-top: 1rem; - color: white; - padding: 20px; - text-align: center; - font-size: medium; -} - -.policy-text { - margin-top: 1rem; - color: white; - padding-top: 20px; - font-size: large; -} - -.login-label { - color: white; -} diff --git a/src/app/login/githublogin.tsx b/src/app/login/githublogin.tsx index 09c35fb99..32d773616 100644 --- a/src/app/login/githublogin.tsx +++ b/src/app/login/githublogin.tsx @@ -1,9 +1,9 @@ -import React, { useEffect, useState, Suspense } from 'react'; -import './githublogin.css'; +import React, { useEffect, useState } from 'react'; import { signIn } from 'next-auth/react'; import { useRouter, useSearchParams } from 'next/navigation'; -import { Button, Content, Grid, GridItem, Modal, ModalBody, ModalFooter, ModalHeader, ModalVariant } from '@patternfly/react-core'; +import { Button, Content, Modal, ModalBody, ModalFooter, ModalHeader, ModalVariant, Alert } from '@patternfly/react-core'; import { GithubIcon } from '@patternfly/react-icons'; +import LoginLinks from '@/app/login/LoginLinks'; const GithubLogin: React.FC = () => { const router = useRouter(); @@ -11,6 +11,7 @@ const GithubLogin: React.FC = () => { const [showError, setShowError] = useState(false); const [errorMsg, setErrorMsg] = useState('Something went wrong.'); const [githubUsername, setGithubUsername] = useState(null); + const [invalidToken, setInvalidToken] = useState(false); useEffect(() => { const githubUsername = searchParams.get('user'); @@ -24,7 +25,10 @@ const GithubLogin: React.FC = () => { setErrorMsg(errorMessage); setShowError(true); } - }, []); + if (error === 'InvalidToken') { + setInvalidToken(true); + } + }, [searchParams]); const handleGitHubLogin = () => { signIn('github', { callbackUrl: '/' }); // Redirect to home page after login @@ -66,89 +70,61 @@ const GithubLogin: React.FC = () => { }; return ( - -
- - -
- - - Sign in to your account - - - - - Join the novel, community based movement to

create truly open source LLMs - - -
- -
- - - - GitHub - {' '} - |{' '} - - Collaborate - {' '} - |{' '} - - Code Of Conduct - - - - - Terms of use - {' '} - |{' '} - - Privacy Policy - - - -
- - - {showError && ( -
- handleOnClose()} - aria-labelledby="join-ilab-modal-title" - aria-describedby="join-ilab-body-variant" - > - - -

{errorMsg}

- - - - , - - - -
- )} +
+ + + Sign in to your account + + + + + Join the novel, community based movement to

create truly open source LLMs + + + {invalidToken && } +
+
- + + New to GitHub? + + Create an account + + + + {showError && ( +
+ handleOnClose()} + aria-labelledby="join-ilab-modal-title" + aria-describedby="join-ilab-body-variant" + > + + +

{errorMsg}

+ + + + , + + + +
+ )} +
); }; diff --git a/src/app/login/login-page.css b/src/app/login/login-page.css new file mode 100644 index 000000000..8390ca78e --- /dev/null +++ b/src/app/login/login-page.css @@ -0,0 +1,111 @@ +.login-page-background { + background-image: url('../../../public/InstructLab-Background-Image.svg'); + background-size: contain; + background-position: center; + background-repeat: no-repeat; + background-color: #454b9a; + min-height: 100vh; + display: flex; + align-items: center; + + overflow-y: auto; + overflow-x: auto; +} + +.login-page-container { + width: 50%; +} + +.loading-container { + display: flex; + gap: 0.5rem; + color: white; + font-size: medium !important; + justify-content: center; + width: 50%; +} +.loading-container .pf-v6-c-spinner { + --pf-v6-c-spinner--Color: white; +} + +.login-container { + text-align: center; + font-size: medium !important; + width: 100%; + max-width: 1200px; + padding: 20px; +} + +.native-login-container { + font-size: medium !important; + width: 100%; + max-width: 1200px; + display: flex; + flex-direction: column; + padding-left: 60px; +} + +.login-form .pf-v6-c-form__label-text, +.login-form .pf-v6-c-helper-text__item-text { + color: white; +} + +.login-form .pf-v6-c-form__label-required { + color: #fbc5c5; +} + +.login-form .pf-v6-c-button.pf-m-primary, +.login-button { + background-color: black !important; + font-weight: bold !important; +} + +.login-form .pf-v6-c-button.pf-m-primary:hover, +.login-button:hover { + background-color: #252525 !important; +} + +.login-alert { + margin-bottom: 1rem; +} +.sign-in-text { + margin-top: 1rem; + color: white !important; + padding: 20px 0; + font-size: xx-large !important; + font-weight: bold !important; +} + +.description-text { + margin-top: 1rem; + color: white !important; + padding-bottom: 50px; + font-size: large !important; + font-style: italic; +} + +.urls-text a { + color: white !important; + text-decoration: underline !important; +} + +.policy-text { + margin-top: 1rem; + color: white; + padding-top: 20px; + font-size: large; +} + +.account-create-text { + text-align: center; + margin-top: 1rem; + color: white !important; + padding-bottom: 50px; + font-size: large !important; +} +.account-create-text .create-button { + text-decoration: underline; + color: #b9dafc; + font-size: large; + margin-left: 0.5rem; +} diff --git a/src/app/login/nativelogin.tsx b/src/app/login/nativelogin.tsx index 1ca73c887..e1ae083e3 100644 --- a/src/app/login/nativelogin.tsx +++ b/src/app/login/nativelogin.tsx @@ -1,23 +1,25 @@ // src/app/login/NativeLogin.tsx import React, { useState } from 'react'; import { signIn } from 'next-auth/react'; -import './githublogin.css'; -import { Button, Content, Form, FormGroup, Grid, GridItem, HelperText, HelperTextItem, TextInput } from '@patternfly/react-core'; +import { Alert, Content, LoginForm } from '@patternfly/react-core'; +import LoginLinks from '@/app/login/LoginLinks'; const NativeLogin: React.FunctionComponent = () => { const [, setShowHelperText] = useState(false); const [username, setUsername] = useState(''); - const [isValidUsername, setIsValidUsername] = useState(true); + const [invalidLogin, setInvalidLogin] = useState(false); + const [inProgress, setInProgress] = useState(false); const [password, setPassword] = useState(''); - const [isValidPassword, setIsValidPassword] = useState(true); const handleLogin = async (e: React.FormEvent) => { e.preventDefault(); + setInvalidLogin(false); + setInProgress(true); const result = await signIn('credentials', { redirect: false, username, password }); if (result?.error) { setShowHelperText(true); - setIsValidUsername(false); - setIsValidPassword(false); + setInvalidLogin(true); + setInProgress(false); } else { window.location.href = '/dashboard'; } @@ -32,104 +34,29 @@ const NativeLogin: React.FunctionComponent = () => { }; return ( -
- - - - - Login locally with a username and password or via GitHub OAuth - - - - - Join the novel, community-based movement to create truly open-source LLMs - - -
-
- - - {!isValidUsername && ( - - Invalid Username - - )} - - - - {!isValidPassword && ( - - Invalid password - - )} - - -
-
- - - - GitHub - {' '} - |{' '} - - Collaborate - {' '} - |{' '} - - Code Of Conduct - - - - - Terms of use - {' '} - |{' '} - - Privacy Policy - - - - - +
+
+ + Log in to your account + + + Join the novel, community-based movement to create truly open-source LLMs + + {invalidLogin ? : null} + + +
); }; diff --git a/src/app/login/page.tsx b/src/app/login/page.tsx index 961cbbc0e..f238c5eb4 100644 --- a/src/app/login/page.tsx +++ b/src/app/login/page.tsx @@ -1,15 +1,14 @@ // src/app/login/page.tsx 'use client'; -import React, { useState, useEffect, Suspense } from 'react'; -import './githublogin.css'; +import React, { useState, useEffect } from 'react'; +import { Spinner } from '@patternfly/react-core'; import NativeLogin from '@/app/login/nativelogin'; import GithubLogin from '@/app/login/githublogin'; -import DevModeLogin from './devmodelogin'; +import './login-page.css'; const Login: React.FunctionComponent = () => { const [deploymentType, setDeploymentType] = useState(); - const [isDevModeEnabled, setIsDevModeEnabled] = useState(false); const [isLoading, setIsLoading] = useState(true); useEffect(() => { @@ -18,7 +17,6 @@ const Login: React.FunctionComponent = () => { const res = await fetch('/api/envConfig'); const envConfig = await res.json(); setDeploymentType(envConfig.DEPLOYMENT_TYPE); - setIsDevModeEnabled(envConfig.ENABLE_DEV_MODE === 'true'); } catch (error) { console.error('Error fetching environment config:', error); setDeploymentType('github'); @@ -29,21 +27,17 @@ const Login: React.FunctionComponent = () => { chooseLoginPage(); }, []); - // Don't render the page until the useEffect finishes fetching environment data - if (isLoading || deploymentType === null) { - return
Loading...
; - } - - if (isDevModeEnabled) { - return ; - } - if (deploymentType === 'native') { - return ; - } return ( - - - +
+ {isLoading ? ( +
+ + Loading... +
+ ) : ( +
{deploymentType === 'native' ? : }
+ )} +
); }; diff --git a/src/app/page.tsx b/src/app/page.tsx index 75a82eda4..8324beef3 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -10,11 +10,9 @@ import { AppLayout } from '../components/AppLayout'; const HomePage: React.FC = () => { const [isWarningConditionAccepted, setIsWarningConditionAccepted] = useState(false); - const handleWarningConditionAccepted = () => { - if (!isWarningConditionAccepted) { - setIsWarningConditionAccepted(true); - } - }; + const handleWarningConditionAccepted = React.useCallback(() => { + setIsWarningConditionAccepted(true); + }, []); return ( diff --git a/src/app/playground/endpoints/DeleteEndpoinModal.tsx b/src/app/playground/endpoints/DeleteEndpoinModal.tsx new file mode 100644 index 000000000..240cf2564 --- /dev/null +++ b/src/app/playground/endpoints/DeleteEndpoinModal.tsx @@ -0,0 +1,69 @@ +'use client'; + +import React from 'react'; +import { Button, Content, Modal, ModalBody, ModalFooter, ModalHeader, ModalVariant, TextInput } from '@patternfly/react-core'; +import { Endpoint } from '@/types'; + +interface Props { + endpoint: Endpoint; + onClose: (deleteEndpoint: boolean) => void; +} + +const DeleteEndpointModal: React.FC = ({ endpoint, onClose }) => { + const [deleteEndpointName, setDeleteEndpointName] = React.useState(''); + + return ( + onClose(false)} + aria-labelledby="confirm-delete-custom-model-endpoint" + aria-describedby="show-yaml-body-variant" + > + + The {endpoint.name} custom model endpoint will be deleted. + + } + /> + + + Type {endpoint.name} to confirm deletion: + * + + setDeleteEndpointName(value)} + /> + + + + + + + ); +}; + +export default DeleteEndpointModal; diff --git a/src/app/playground/endpoints/EditEndpointModal.tsx b/src/app/playground/endpoints/EditEndpointModal.tsx new file mode 100644 index 000000000..f6272808f --- /dev/null +++ b/src/app/playground/endpoints/EditEndpointModal.tsx @@ -0,0 +1,246 @@ +'use client'; + +import React from 'react'; +import { Endpoint, ModelEndpointStatus } from '@/types'; +import { + Button, + Form, + FormGroup, + FormHelperText, + HelperText, + HelperTextItem, + Modal, + ModalBody, + ModalFooter, + ModalHeader, + ModalVariant, + Popover, + TextInput, + ValidatedOptions +} from '@patternfly/react-core'; +import { ExclamationCircleIcon, OutlinedQuestionCircleIcon } from '@patternfly/react-icons'; +import { fetchEndpointStatus } from '@/components/Chat/modelService'; + +const removeTrailingSlash = (inputUrl: string): string => { + if (inputUrl.slice(-1) === '/') { + return inputUrl.slice(0, -1); + } + return inputUrl; +}; + +const validateUrl = (link: string): boolean => { + try { + new URL(link); + return true; + } catch (e) { + return false; + } +}; + +interface Props { + endpoint: Endpoint; + onClose: (endpoint?: Endpoint) => void; +} + +const EditEndpointModal: React.FC = ({ endpoint, onClose }) => { + const [endpointName, setEndpointName] = React.useState(endpoint.name || ''); + const [endpointDescription, setEndpointDescription] = React.useState(endpoint.description || ''); + const [url, setUrl] = React.useState(endpoint.url || ''); + const [modelName, setModelName] = React.useState(endpoint.modelName || ''); + const [modelDescription, setModelDescription] = React.useState(endpoint.modelDescription || ''); + const [apiKey, setApiKey] = React.useState(endpoint.apiKey || ''); + const [nameTouched, setNameTouched] = React.useState(); + const [urlTouched, setUrlTouched] = React.useState(); + const [modelNameTouched, setModelNameTouched] = React.useState(); + const [apiKeyTouched, setApiKeyTouched] = React.useState(); + + const validName = endpointName.trim().length > 0; + const validUrl = validateUrl(url); + const validModelName = modelName.trim().length > 0; + const validApiKey = apiKey.trim().length > 0; + + const isValid = validName && validUrl && validModelName && validApiKey; + + return ( + onClose()} + aria-labelledby="endpoint-modal-title" + aria-describedby="endpoint-body-variant" + > + + +
+ + setEndpointName(value)} + placeholder="Enter name" + onBlur={() => setNameTouched(true)} + /> + {nameTouched && !validName ? ( + + + } variant={ValidatedOptions.error}> + Required field + + + + ) : null} + + + setEndpointDescription(value)} + placeholder="Enter description" + /> + + + + + } + > + setUrl(value)} + placeholder="Enter URL" + onBlur={() => setUrlTouched(true)} + /> + {urlTouched && !validUrl ? ( + + + } variant={ValidatedOptions.error}> + Please enter a valid URL. + + + + ) : null} + + + setModelName(value)} + placeholder="Enter model name" + onBlur={() => setModelNameTouched(true)} + /> + {modelNameTouched && !validModelName ? ( + + + } variant={ValidatedOptions.error}> + Required field + + + + ) : null} + + + setModelDescription(value)} + placeholder="Enter description" + /> + + + + + } + > + setApiKey(value)} + placeholder="Enter API key" + onBlur={() => setApiKeyTouched(true)} + /> + {apiKeyTouched && !validApiKey ? ( + + + } variant={ValidatedOptions.error}> + Required field + + + + ) : null} + +
+ + + + + + + ); +}; + +export default EditEndpointModal; diff --git a/src/app/playground/endpoints/EndpointActions.tsx b/src/app/playground/endpoints/EndpointActions.tsx new file mode 100644 index 000000000..ffa9fff20 --- /dev/null +++ b/src/app/playground/endpoints/EndpointActions.tsx @@ -0,0 +1,47 @@ +'use client'; + +import React from 'react'; +import { Button, Dropdown, DropdownList, DropdownItem, MenuToggle, Divider } from '@patternfly/react-core'; +import { EllipsisVIcon } from '@patternfly/react-icons'; +import { Endpoint } from '@/types'; + +interface Props { + endpoint: Endpoint; + onToggleEnabled: () => void; + onEdit: () => void; + onDelete: () => void; +} + +const EndpointActions: React.FC = ({ endpoint, onToggleEnabled, onEdit, onDelete }) => { + const [menuOpen, setMenuOpen] = React.useState(false); + + return ( + <> + + setMenuOpen(false)} + onOpenChange={(isOpen: boolean) => setMenuOpen(isOpen)} + toggle={(toggleRef) => ( + setMenuOpen((prev) => !prev)} isExpanded={menuOpen}> + + + )} + popperProps={{ position: 'right' }} + ouiaId="ModelEndpointDropdown" + > + + Edit + + + Delete endpoint + + + + + ); +}; + +export default EndpointActions; diff --git a/src/app/playground/endpoints/page.tsx b/src/app/playground/endpoints/page.tsx index 2f5e2e168..c8bc6402e 100644 --- a/src/app/playground/endpoints/page.tsx +++ b/src/app/playground/endpoints/page.tsx @@ -1,12 +1,8 @@ 'use client'; -import React, { useState, useEffect } from 'react'; +import React from 'react'; import { v4 as uuidv4 } from 'uuid'; -import { AppLayout } from '@/components/AppLayout'; -import { Endpoint } from '@/types'; import { - Breadcrumb, - BreadcrumbItem, Button, Content, DataList, @@ -17,99 +13,163 @@ import { DataListItemRow, Flex, FlexItem, - Form, - FormGroup, - InputGroup, - Modal, - ModalBody, - ModalFooter, - ModalHeader, - ModalVariant, - PageBreadcrumb, PageSection, - TextInput, - Title + Title, + Truncate, + ClipboardCopy } from '@patternfly/react-core'; -import { EyeSlashIcon, EyeIcon } from '@patternfly/react-icons'; +import { BanIcon, CheckCircleIcon, EyeSlashIcon, EyeIcon, QuestionCircleIcon } from '@patternfly/react-icons'; +import { AppLayout } from '@/components/AppLayout'; +import { Endpoint, ModelEndpointStatus } from '@/types'; +import EditEndpointModal from '@/app/playground/endpoints/EditEndpointModal'; +import DeleteEndpointModal from '@/app/playground/endpoints/DeleteEndpoinModal'; +import EndpointActions from '@/app/playground/endpoints/EndpointActions'; +import { fetchEndpointStatus } from '@/components/Chat/modelService'; + +const iconForStatus = (status: ModelEndpointStatus) => { + switch (status) { + case ModelEndpointStatus.available: + return ; + case ModelEndpointStatus.unavailable: + return ; + case ModelEndpointStatus.disabled: + return ; + case ModelEndpointStatus.unknown: + return ; + default: + return ; + } +}; interface ExtendedEndpoint extends Endpoint { isApiKeyVisible?: boolean; } +const getEndpointsStatus = async (endpoints: ExtendedEndpoint[]): Promise => + Promise.all( + endpoints.map(async (endpoint) => { + const status = await fetchEndpointStatus(endpoint); + + return { + ...endpoint, + status + }; + }) + ); + const EndpointsPage: React.FC = () => { - const [endpoints, setEndpoints] = useState([]); - const [isModalOpen, setIsModalOpen] = useState(false); - const [currentEndpoint, setCurrentEndpoint] = useState | null>(null); - const [url, setUrl] = useState(''); - const [modelName, setModelName] = useState(''); - const [apiKey, setApiKey] = useState(''); - - useEffect(() => { - const storedEndpoints = localStorage.getItem('endpoints'); - if (storedEndpoints) { - setEndpoints(JSON.parse(storedEndpoints)); + const [endpoints, setEndpoints] = React.useState([]); + const [deleteEndpoint, setDeleteEndpoint] = React.useState(); + const [editEndpoint, setEditEndpoint] = React.useState(); + const [actionsWidth, setActionsWidth] = React.useState(); + + const setActionRef = (ref: HTMLElement | null) => { + if (!ref) { + return; } - }, []); - const handleModalToggle = () => { - setIsModalOpen(!isModalOpen); + const rect = ref.getBoundingClientRect(); + const style = window.getComputedStyle(ref); + const marginLeft = parseFloat(style.marginLeft); + const marginRight = parseFloat(style.marginRight); + + setActionsWidth(rect.width + marginLeft + marginRight); }; - const removeTrailingSlash = (inputUrl: string): string => { - if (typeof inputUrl !== 'string') { - throw new Error('Invalid url'); - } - if (inputUrl.slice(-1) === '/') { - return inputUrl.slice(0, -1); + React.useEffect(() => { + const loadEndpoints = async () => { + const storedEndpoints = localStorage.getItem('endpoints'); + if (storedEndpoints) { + const loadedEndpoints = await getEndpointsStatus(JSON.parse(storedEndpoints)); + setEndpoints(loadedEndpoints); + } + }; + loadEndpoints(); + }, []); + + React.useEffect(() => { + async function updateEndpointStatuses() { + const updatedEndpoints = await Promise.all( + endpoints.map(async (endpoint) => { + const status = await fetchEndpointStatus(endpoint); + + return { + ...endpoint, + status + }; + }) + ); + + setEndpoints(updatedEndpoints); } - return inputUrl; + + const interval = setInterval( + () => { + console.log('Running update endpoints'); + updateEndpointStatuses(); + }, + 10 * 60 * 1000 + ); // run every 10 minutes in milliseconds + + return () => clearInterval(interval); // cleanup on unmount + }, [endpoints]); + + const toggleEndpointEnabled = (endpointId: string) => { + const updatedEndpoints = endpoints.map((endpoint) => { + if (endpoint.id === endpointId) { + return { + ...endpoint, + enabled: !endpoint.enabled + }; + } + return endpoint; + }); + setEndpoints(updatedEndpoints); + localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); }; - const handleSaveEndpoint = () => { - const updatedUrl = removeTrailingSlash(url); - if (currentEndpoint) { + const handleSaveEndpoint = async (endPoint?: Endpoint) => { + if (endPoint) { + const status = await fetchEndpointStatus(endPoint); const updatedEndpoint: ExtendedEndpoint = { - id: currentEndpoint.id || uuidv4(), - url: updatedUrl, - modelName: modelName, - apiKey: apiKey, - isApiKeyVisible: false + ...endPoint, + isApiKeyVisible: false, + status: status, + enabled: true }; - const updatedEndpoints = currentEndpoint.id - ? endpoints.map((ep) => (ep.id === currentEndpoint.id ? updatedEndpoint : ep)) - : [...endpoints, updatedEndpoint]; + const updatedEndpoints = updatedEndpoint?.id + ? endpoints.map((ep) => (ep.id === updatedEndpoint.id ? updatedEndpoint : ep)) + : [...endpoints, { ...updatedEndpoint, id: uuidv4() }]; setEndpoints(updatedEndpoints); localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); - setCurrentEndpoint(null); - setUrl(''); - setModelName(''); - setApiKey(''); - handleModalToggle(); } + setEditEndpoint(undefined); }; - const handleDeleteEndpoint = (id: string) => { - const updatedEndpoints = endpoints.filter((ep) => ep.id !== id); - setEndpoints(updatedEndpoints); - localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); - }; + const handleDeleteEndpoint = (doDelete: boolean) => { + if (doDelete && deleteEndpoint) { + const updatedEndpoints = endpoints.filter((ep) => ep.id !== deleteEndpoint.id); + setEndpoints(updatedEndpoints); + localStorage.setItem('endpoints', JSON.stringify(updatedEndpoints)); + } - const handleEditEndpoint = (endpoint: ExtendedEndpoint) => { - setCurrentEndpoint(endpoint); - setUrl(endpoint.url); - setModelName(endpoint.modelName); - setApiKey(endpoint.apiKey); - handleModalToggle(); + setDeleteEndpoint(undefined); }; const handleAddEndpoint = () => { - setCurrentEndpoint({ id: '', url: '', modelName: '', apiKey: '', isApiKeyVisible: false }); - setUrl(''); - setModelName(''); - setApiKey(''); - handleModalToggle(); + setEditEndpoint({ + id: '', + name: '', + description: '', + url: '', + modelName: '', + modelDescription: '', + apiKey: '', + status: ModelEndpointStatus.unknown, + enabled: true + }); }; const toggleApiKeyVisibility = (id: string) => { @@ -126,121 +186,116 @@ const EndpointsPage: React.FC = () => { return isApiKeyVisible ? apiKey : '********'; }; - useEffect(() => {}, [url]); - return ( - - - Dashboard - Custom Model Endpoints - - - - + + + Custom model endpoints + - - Custom Model Endpoints - + Custom model endpoints enable you to interact with and test fine-tuned models using the chat interface. - -
- Manage your own customer model endpoints. If you have a custom model that is served by an endpoint, you can add it here. This will allow you - to use the custom model in the playground chat. - - + - + + + + + Endpoint name + , + + Status + , + + URL + , + + Model name + , + + API key + + ]} + /> + + + + + {endpoints.map((endpoint) => ( + + + + {endpoint.description ? ( + + + + ) : null} + , + + + {iconForStatus(endpoint.status)} + {ModelEndpointStatus[endpoint.status] || 'unknown'} + + , - URL: {endpoint.url} + + {endpoint.url} + , - Model Name: {endpoint.modelName} + + + + {endpoint.modelDescription ? ( + + + + ) : null} , - API Key: {renderApiKey(endpoint.apiKey, endpoint.isApiKeyVisible || false)} + {renderApiKey(endpoint.apiKey, endpoint.isApiKeyVisible || false)} ]} /> - - - + + toggleEndpointEnabled(endpoint.id)} + onEdit={() => setEditEndpoint(endpoint)} + onDelete={() => setDeleteEndpoint(endpoint)} + /> ))} - {isModalOpen && ( - handleModalToggle()} - aria-labelledby="endpoint-modal-title" - aria-describedby="endpoint-body-variant" - > - - -
- - setUrl(value)} placeholder="Enter URL" /> - - - setModelName(value)} - placeholder="Enter Model Name" - /> - - - - setApiKey(value)} - placeholder="Enter API Key" - /> - - -
- - - - - - - )} + {editEndpoint ? : null} + {deleteEndpoint ? : null} ); }; diff --git a/src/components/AppLayout.tsx b/src/components/AppLayout.tsx index 0a2ad3e45..6a7c6f175 100644 --- a/src/components/AppLayout.tsx +++ b/src/components/AppLayout.tsx @@ -103,15 +103,15 @@ const AppLayout: React.FunctionComponent = ({ children, className }) label: 'Playground', children: [ { path: '/playground/chat', label: 'Chat' }, - { path: '/playground/endpoints', label: 'Custom Model Endpoints' } + { path: '/playground/endpoints', label: 'Custom model endpoints' } ] }, isExperimentalEnabled && { path: '/experimental', - label: 'Experimental Features', + label: 'Experimental features', children: [ { path: '/experimental/fine-tune/', label: 'Fine-tuning' }, - { path: '/experimental/chat-eval/', label: 'Model Chat Eval' } + { path: '/experimental/chat-eval/', label: 'Model chat eval' } ] } ].filter(Boolean) as Route[]; @@ -185,7 +185,15 @@ const AppLayout: React.FunctionComponent = ({ children, className }) const PageSkipToContent = Skip to Content; return ( - + {children} ); diff --git a/src/components/Chat/ChatBotComponent.tsx b/src/components/Chat/ChatBotComponent.tsx index f35f75320..c76444c4f 100644 --- a/src/components/Chat/ChatBotComponent.tsx +++ b/src/components/Chat/ChatBotComponent.tsx @@ -2,6 +2,7 @@ 'use client'; import * as React from 'react'; +import { useRouter } from 'next/navigation'; import { Alert, AlertActionCloseButton, @@ -12,6 +13,7 @@ import { DropdownList, MenuToggle, MenuToggleElement, + Popover, Select, SelectList, SelectOption, @@ -33,7 +35,7 @@ import { Model } from '@/types'; import { modelFetcher } from '@/components/Chat/modelService'; const botAvatar = '/bot-icon-chat-32x32.svg'; -import { EllipsisVIcon, TimesIcon } from '@patternfly/react-icons'; +import { EllipsisVIcon, OutlinedQuestionCircleIcon, TimesIcon } from '@patternfly/react-icons'; import styles from '@/components/Chat/chat.module.css'; import { ModelsContext } from '@/components/Chat/ModelsContext'; import { useSession } from 'next-auth/react'; @@ -71,6 +73,7 @@ const ChatBotComponent: React.FunctionComponent = ({ setStopCallback, setController }) => { + const router = useRouter(); const { data: session } = useSession(); const { availableModels } = React.useContext(ModelsContext); const [isLoading, setIsLoading] = React.useState(false); @@ -153,7 +156,11 @@ const ChatBotComponent: React.FunctionComponent = ({ }; stopped.current = false; - await modelFetcher(model, input, setCurrentMessage, setController); + try { + await modelFetcher(model, input, setCurrentMessage, setController); + } catch (e) { + console.error(`Model fetch failed: `, e); + } setIsLoading(false); setFetching(false); @@ -184,14 +191,14 @@ const ChatBotComponent: React.FunctionComponent = ({ const toggle = (toggleRef: React.Ref) => ( - {model ? model.name : 'Select a model'} + {model && model.enabled ? model.name : 'Select a model'} ); const dropdownItems = React.useMemo( () => availableModels.map((model, index) => ( - + {model.name} )), @@ -227,6 +234,23 @@ const ChatBotComponent: React.FunctionComponent = ({ > {dropdownItems} + + If your model is not selectable, that means you have disabled the custom model endpoint. To change this please see the{' '} + {' '} + page. +
+ } + > + + {showCompare ? ( diff --git a/src/components/Chat/ChatBotContainer.tsx b/src/components/Chat/ChatBotContainer.tsx index 800ef82f9..926bbfec9 100644 --- a/src/components/Chat/ChatBotContainer.tsx +++ b/src/components/Chat/ChatBotContainer.tsx @@ -77,10 +77,10 @@ const ChatBotContainer: React.FC = () => { const handleStopButton = () => { if (mainChatController) { - mainChatController.abort(); + mainChatController.abort('user stopped'); } if (altChatController) { - altChatController.abort(); + altChatController.abort('user stopped'); } }; @@ -138,7 +138,9 @@ const ChatBotContainer: React.FC = () => { { + setSubmittedMessage(typeof message === 'string' ? message : String(message)); + }} hasMicrophoneButton hasAttachButton={false} isSendButtonDisabled={mainChatFetching || altChatFetching} diff --git a/src/components/Chat/ModelsContext.tsx b/src/components/Chat/ModelsContext.tsx index 57c2c6f1d..63f2faa0c 100644 --- a/src/components/Chat/ModelsContext.tsx +++ b/src/components/Chat/ModelsContext.tsx @@ -29,18 +29,19 @@ const ModelsContextProvider: React.FC = ({ children }) => { const envConfig = await response.json(); const defaultModels: Model[] = [ - { isDefault: true, name: 'Granite-7b', apiURL: envConfig.GRANITE_API, modelName: envConfig.GRANITE_MODEL_NAME }, - { isDefault: true, name: 'Merlinite-7b', apiURL: envConfig.MERLINITE_API, modelName: envConfig.MERLINITE_MODEL_NAME } + { isDefault: true, name: 'Granite-7b', apiURL: envConfig.GRANITE_API, modelName: envConfig.GRANITE_MODEL_NAME, enabled: true }, + { isDefault: true, name: 'Merlinite-7b', apiURL: envConfig.MERLINITE_API, modelName: envConfig.MERLINITE_MODEL_NAME, enabled: true } ]; const storedEndpoints = localStorage.getItem('endpoints'); - const customModels = storedEndpoints ? JSON.parse(storedEndpoints).map((endpoint: Endpoint) => ({ isDefault: false, name: endpoint.modelName, apiURL: `${endpoint.url}`, - modelName: endpoint.modelName + modelName: endpoint.modelName, + enabled: endpoint.enabled, + apiKey: endpoint.apiKey })) : []; diff --git a/src/components/Chat/modelService.ts b/src/components/Chat/modelService.ts index 068d4d6c1..d025c149d 100644 --- a/src/components/Chat/modelService.ts +++ b/src/components/Chat/modelService.ts @@ -1,26 +1,42 @@ -import { Model } from '@/types'; +import { Endpoint, Model, ModelEndpointStatus } from '@/types'; const systemRole = 'You are a cautious assistant. You carefully follow instructions.' + ' You are helpful and harmless and you follow ethical guidelines and promote positive behavior.'; -export const customModelFetcher = async ( - selectedModel: Model, - input: string, - onMessageReceived: (message: string) => void, - setController: (controller: AbortController) => void -) => { +const getCustomModelHeaders = (apiKey?: string): HeadersInit => + apiKey + ? { + 'Content-Type': 'application/json', + Accept: 'text/event-stream', + Authorization: `Bearer: ${apiKey}` + } + : { + 'Content-Type': 'application/json', + Accept: 'text/event-stream' + }; + +const getCustomModelRequestData = (modelName: string, input: string): string => { const messagesPayload = [ { role: 'system', content: systemRole }, { role: 'user', content: input } ]; const requestData = { - model: selectedModel.modelName, + model: modelName, messages: messagesPayload, stream: true }; + return JSON.stringify(requestData); +}; + +export const customModelFetcher = async ( + selectedModel: Model, + input: string, + onMessageReceived: (message: string) => void, + setController: (controller: AbortController) => void +) => { const newController = new AbortController(); setController(newController); @@ -28,11 +44,8 @@ export const customModelFetcher = async ( try { const response = await fetch(`${selectedModel.apiURL}/v1/chat/completions`, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'text/event-stream' - }, - body: JSON.stringify(requestData), + headers: getCustomModelHeaders(selectedModel.apiKey), + body: getCustomModelRequestData(selectedModel.modelName, input), signal: newController.signal }); @@ -115,7 +128,7 @@ export const defaultModelFetcher = async ( headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ input, systemRole }), + body: JSON.stringify({ question: input, systemRole }), signal: newController.signal } ); @@ -149,3 +162,25 @@ export const modelFetcher = async ( selectedModel.isDefault ? defaultModelFetcher(selectedModel, input, onMessageReceived, setController) : customModelFetcher(selectedModel, input, onMessageReceived, setController); + +export const fetchEndpointStatus = async (endpoint: Endpoint): Promise => { + if (!endpoint.enabled) { + return ModelEndpointStatus.disabled; + } + // Attempt a fetch for the custom endpoint + try { + const response = await fetch(`${endpoint.url}/v1/chat/completions`, { + method: 'POST', + headers: getCustomModelHeaders(endpoint.apiKey), + body: getCustomModelRequestData(endpoint.modelName, 'test') + }); + + if (!response.ok) { + console.error(`Failed to serve model from endpoint ${endpoint.url}`); + return ModelEndpointStatus.unavailable; + } + return ModelEndpointStatus.available; + } catch (error) { + return ModelEndpointStatus.unknown; + } +}; diff --git a/src/components/Common/TruncatedText.tsx b/src/components/Common/TruncatedText.tsx new file mode 100644 index 000000000..3dba1c9db --- /dev/null +++ b/src/components/Common/TruncatedText.tsx @@ -0,0 +1,69 @@ +import React from 'react'; +import { Tooltip } from '@patternfly/react-core'; + +type TruncatedTextProps = { + maxLines: number; + content: string; + tooltipMaxWidth?: string; + useTooltip?: boolean; +} & Omit, 'content'>; + +const TruncatedText: React.FC = ({ maxLines, content, tooltipMaxWidth, useTooltip = true, ...props }) => { + const outerElementRef = React.useRef(null); + const textElementRef = React.useRef(null); + const [isTruncated, setIsTruncated] = React.useState(false); + + let shownContent = content; + const splits = content.split('\n'); + if (splits.length > maxLines) { + shownContent = splits.slice(0, maxLines).join('\n') + '...'; + } + + const updateTruncation = React.useCallback(() => { + if (textElementRef.current && outerElementRef.current) { + setIsTruncated(shownContent !== content || textElementRef.current.offsetHeight > outerElementRef.current.offsetHeight); + } + }, [content, shownContent]); + + const truncateBody = ( + { + props.onMouseEnter?.(e); + updateTruncation(); + }} + onFocus={(e) => { + props.onFocus?.(e); + updateTruncation(); + }} + > + {shownContent} + + ); + + if (useTooltip) { + return ( + + ); + } + + return truncateBody; +}; + +export default TruncatedText; diff --git a/src/components/Common/WizardFormGroupLabelHelp.tsx b/src/components/Common/WizardFormGroupLabelHelp.tsx new file mode 100644 index 000000000..c0c5132cf --- /dev/null +++ b/src/components/Common/WizardFormGroupLabelHelp.tsx @@ -0,0 +1,24 @@ +import * as React from 'react'; +import { Button, Popover } from '@patternfly/react-core'; +import { t_global_font_size_xs as XsFontSize, t_global_icon_color_subtle as SubtleIconColor } from '@patternfly/react-tokens'; +import { OutlinedQuestionCircleIcon } from '@patternfly/react-icons'; + +interface Props { + headerContent?: React.ReactNode; + bodyContent: React.ReactNode; + ariaLabel?: string; +} + +const WizardFormGroupLabelHelp: React.FC = ({ headerContent, bodyContent, ariaLabel = 'More info' }) => { + const labelHelpRef = React.useRef(null); + + return ( + + + + ); +}; + +export default WizardFormGroupLabelHelp; diff --git a/src/components/Common/WizardPageHeader.tsx b/src/components/Common/WizardPageHeader.tsx new file mode 100644 index 000000000..708ab626e --- /dev/null +++ b/src/components/Common/WizardPageHeader.tsx @@ -0,0 +1,16 @@ +import * as React from 'react'; +import { Content } from '@patternfly/react-core'; + +interface Props { + title: React.ReactNode; + description?: React.ReactNode; +} + +const WizardPageHeader: React.FC = ({ title, description }) => ( +
+ {title} + {description} +
+); + +export default WizardPageHeader; diff --git a/src/components/Common/WizardSectionHeader.tsx b/src/components/Common/WizardSectionHeader.tsx new file mode 100644 index 000000000..c548c7f77 --- /dev/null +++ b/src/components/Common/WizardSectionHeader.tsx @@ -0,0 +1,38 @@ +import React from 'react'; +import { Content, Popover, Button } from '@patternfly/react-core'; +import { OutlinedQuestionCircleIcon } from '@patternfly/react-icons'; +import { + t_global_spacer_sm as SmallSpacerSize, + t_global_font_size_xs as XsFontSize, + t_global_icon_color_subtle as SubtleIconColor +} from '@patternfly/react-tokens'; + +interface Props { + title: string; + description?: string; + helpInfo?: React.ReactNode; +} +const WizardSectionHeader: React.FC = ({ title, description, helpInfo }) => ( + <> + + {title} + {helpInfo ? ( + <> + {' '} + + + + + ) : null} + + {description ? ( + + {description} + + ) : null} + +); + +export default WizardSectionHeader; diff --git a/src/components/Contribute/AttributionInformation/AttributionInformation.tsx b/src/components/Contribute/AttributionInformation/AttributionInformation.tsx new file mode 100644 index 000000000..b1a7e4b95 --- /dev/null +++ b/src/components/Contribute/AttributionInformation/AttributionInformation.tsx @@ -0,0 +1,216 @@ +import React, { useEffect } from 'react'; +import { ContributionFormData } from '@/types'; +import { ValidatedOptions, FormGroup, TextInput, FormHelperText, HelperText, HelperTextItem, FlexItem, Flex, Form } from '@patternfly/react-core'; +import { ExclamationCircleIcon } from '@patternfly/react-icons'; +import WizardPageHeader from '@/components/Common/WizardPageHeader'; +import WizardFormGroupLabelHelp from '@/components/Common/WizardFormGroupLabelHelp'; + +interface Props { + isEditForm?: boolean; + contributionFormData: ContributionFormData; + titleWork: string; + setTitleWork: (val: string) => void; + linkWork?: string; + setLinkWork?: (val: string) => void; + revision?: string; + setRevision?: (val: string) => void; + licenseWork: string; + setLicenseWork: (val: string) => void; + creators: string; + setCreators: (val: string) => void; +} + +const AttributionInformation: React.FC = ({ + isEditForm, + titleWork, + setTitleWork, + linkWork = '', + setLinkWork, + revision = '', + setRevision, + licenseWork, + setLicenseWork, + creators, + setCreators +}) => { + const [validTitle, setValidTitle] = React.useState(); + const [validLink, setValidLink] = React.useState(); + const [validLicense, setValidLicense] = React.useState(); + const [validCreators, setValidCreators] = React.useState(); + + useEffect(() => { + if (!isEditForm) { + return; + } + setValidTitle(ValidatedOptions.success); + setValidLink(ValidatedOptions.success); + setValidLicense(ValidatedOptions.success); + setValidCreators(ValidatedOptions.success); + }, [isEditForm]); + + const validateTitle = (titleStr: string) => { + const title = titleStr.trim(); + if (title.length > 0) { + setValidTitle(ValidatedOptions.success); + return; + } + setValidTitle(ValidatedOptions.error); + return; + }; + + const validateLink = (linkStr: string) => { + const link = linkStr.trim(); + if (link.length === 0) { + setValidLink(ValidatedOptions.error); + return; + } + try { + new URL(link); + setValidLink(ValidatedOptions.success); + } catch (e) { + setValidLink(ValidatedOptions.warning); + } + }; + + const validateLicense = (licenseStr: string) => { + const license = licenseStr.trim(); + setValidLicense(license.length > 0 ? ValidatedOptions.success : ValidatedOptions.error); + }; + + const validateCreators = (creatorsStr: string) => { + const creators = creatorsStr.trim(); + setValidCreators(creators.length > 0 ? ValidatedOptions.success : ValidatedOptions.error); + }; + + return ( + + + + + +
+ } + > + setTitleWork(value)} + onBlur={() => validateTitle(titleWork)} + /> + {validTitle === ValidatedOptions.error && ( + + + } variant={validTitle}> + Required field + + + + )} + + {setLinkWork ? ( + } + > + setLinkWork(value)} + onBlur={() => validateLink(linkWork)} + /> + {validLink === ValidatedOptions.error && ( + + + } variant={validLink}> + Required field + + + + )} + {validLink === ValidatedOptions.warning && ( + + + } variant={validLink}> + Please enter a valid URL. + + + + )} + + ) : null} + {setRevision ? ( + + setRevision(value)} /> + + ) : null} + + } + > + setLicenseWork(value)} + onBlur={() => validateLicense(licenseWork)} + /> + {validLicense === ValidatedOptions.error && ( + + + } variant={validLicense}> + Required field + + + + )} + + + setCreators(value)} + onBlur={() => validateCreators(creators)} + /> + + + {validCreators === ValidatedOptions.error ? ( + } variant={validCreators}> + Required field + + ) : ( + If listing more than 1 author, separate the names using commas. + )} + + + +
+ + + ); +}; + +export default AttributionInformation; diff --git a/src/components/Contribute/AuthorInformation.tsx b/src/components/Contribute/AuthorInformation.tsx deleted file mode 100644 index bee523012..000000000 --- a/src/components/Contribute/AuthorInformation.tsx +++ /dev/null @@ -1,121 +0,0 @@ -import React, { useEffect, useState } from 'react'; -import { checkSkillFormCompletion } from './Skill/validation'; -import { checkKnowledgeFormCompletion } from './Knowledge/validation'; -import { ValidatedOptions, FormGroup, TextInput, FormHelperText, HelperText, HelperTextItem } from '@patternfly/react-core'; -import { ExclamationCircleIcon } from '@patternfly/react-icons'; - -export enum FormType { - Knowledge, - Skill -} - -interface Props { - formType: FormType; - reset: boolean; - formData: object; - setDisableAction: React.Dispatch>; - email: string; - setEmail: React.Dispatch>; - name: string; - setName: React.Dispatch>; -} -const AuthorInformation: React.FC = ({ formType, reset, formData, setDisableAction, email, setEmail, name, setName }) => { - const [validEmail, setValidEmail] = useState(); - const [validName, setValidName] = useState(); - const [validEmailError, setValidEmailError] = useState('Required Field'); - - const validateEmail = (emailStr: string) => { - const email = emailStr.trim(); - const re = /\S+@\S+\.\S+/; - if (re.test(email)) { - setValidEmail(ValidatedOptions.success); - setValidEmailError(''); - if (formType === FormType.Knowledge) { - setDisableAction(!checkKnowledgeFormCompletion(formData)); - return; - } - setDisableAction(!checkSkillFormCompletion(formData)); - return; - } - const errMsg = email ? 'Please enter a valid email address.' : 'Required field'; - setDisableAction(true); - setValidEmail(ValidatedOptions.error); - setValidEmailError(errMsg); - return; - }; - - const validateName = (nameStr: string) => { - const name = nameStr.trim(); - if (name.length > 0) { - setValidName(ValidatedOptions.success); - if (formType === FormType.Knowledge) { - setDisableAction(!checkKnowledgeFormCompletion(formData)); - return; - } - setDisableAction(!checkSkillFormCompletion(formData)); - return; - } - setDisableAction(true); - setValidName(ValidatedOptions.error); - return; - }; - - useEffect(() => { - setValidEmail(ValidatedOptions.default); - setValidName(ValidatedOptions.default); - }, [reset]); - - return ( - <> -

- Author Information - * -

-

Provide your information required for a GitHub DCO sign-off.

- - setEmail(value)} - onBlur={() => validateEmail(email)} - /> - {validEmail === ValidatedOptions.error && ( - - - } variant={validEmail}> - {validEmailError} - - - - )} - - - setName(value)} - onBlur={() => validateName(name)} - /> - {validName === ValidatedOptions.error && ( - - - } variant={validName}> - Required field - - - - )} - - - ); -}; - -export default AuthorInformation; diff --git a/src/components/Contribute/Knowledge/AutoFill.ts b/src/components/Contribute/AutoFill.ts similarity index 79% rename from src/components/Contribute/Knowledge/AutoFill.ts rename to src/components/Contribute/AutoFill.ts index e8583d259..1e39acf3b 100644 --- a/src/components/Contribute/Knowledge/AutoFill.ts +++ b/src/components/Contribute/AutoFill.ts @@ -1,4 +1,4 @@ -import { KnowledgeFormData, KnowledgeSeedExample, QuestionAndAnswerPair } from '@/types'; +import { KnowledgeFormData, KnowledgeSeedExample, QuestionAndAnswerPair, SkillFormData, SkillSeedExample } from '@/types'; import { ValidatedOptions } from '@patternfly/react-core'; const questionAndAnswerPairs1: QuestionAndAnswerPair[] = [ @@ -127,7 +127,7 @@ const questionAndAnswerPairs5: QuestionAndAnswerPair[] = [ } ]; -const seedExamples: KnowledgeSeedExample[] = [ +const knowledgeSeedExamples: KnowledgeSeedExample[] = [ { immutable: true, isExpanded: true, @@ -251,20 +251,115 @@ const seedExamples: KnowledgeSeedExample[] = [ ]; export const autoFillKnowledgeFields: KnowledgeFormData = { + branchName: `knowledge-contribution-${Date.now()}`, email: 'helloworld@instructlab.com', name: 'juliadenham', submissionSummary: 'Information about the Phoenix Constellation.', - domain: 'astronomy', - documentOutline: - 'Information about the Phoenix Constellation including the history, characteristics, and features of the stars in the constellation.', filePath: 'science/physics/astrophysics/stars', - seedExamples: seedExamples, - knowledgeDocumentRepositoryUrl: 'https://github.com/juliadenham/Summit_knowledge', + seedExamples: knowledgeSeedExamples, + knowledgeDocumentRepositoryUrl: '~/.instructlab-ui/taxonomy-knowledge-docs', knowledgeDocumentCommit: '0a1f2672b9b90582e6115333e3ed62fd628f1c0f', documentName: 'phoenix_constellation.md', titleWork: 'Phoenix (constellation)', linkWork: 'https://en.wikipedia.org/wiki/Phoenix_(constellation)', revision: 'https://en.wikipedia.org/w/index.php?title=Phoenix_(constellation)&oldid=1237187773', licenseWork: 'CC-BY-SA-4.0', - creators: 'Wikipedia Authors' + creators: 'Wikipedia Authors', + filesToUpload: [], + uploadedFiles: [] +}; + +const skillsSeedExamples: SkillSeedExample[] = [ + { + immutable: false, + isExpanded: false, + context: undefined, + isContextValid: ValidatedOptions.success, + validationError: undefined, + questionAndAnswer: { + immutable: false, + question: 'What are 5 words that rhyme with horn?', + isQuestionValid: ValidatedOptions.success, + questionValidationError: undefined, + answer: 'warn, torn, born, thorn, and corn.', + isAnswerValid: ValidatedOptions.success, + answerValidationError: undefined + } + }, + { + immutable: false, + isExpanded: false, + context: undefined, + isContextValid: ValidatedOptions.success, + validationError: undefined, + questionAndAnswer: { + immutable: false, + question: 'What are 5 words that rhyme with cat?', + isQuestionValid: ValidatedOptions.success, + questionValidationError: undefined, + answer: 'bat, gnat, rat, vat, and mat.', + isAnswerValid: ValidatedOptions.success, + answerValidationError: undefined + } + }, + { + immutable: false, + isExpanded: false, + context: undefined, + isContextValid: ValidatedOptions.success, + validationError: undefined, + questionAndAnswer: { + immutable: false, + question: 'What are 5 words that rhyme with poor?', + isQuestionValid: ValidatedOptions.success, + questionValidationError: undefined, + answer: 'door, shore, core, bore, and tore.', + isAnswerValid: ValidatedOptions.success, + answerValidationError: undefined + } + }, + { + immutable: false, + isExpanded: false, + context: undefined, + isContextValid: ValidatedOptions.success, + validationError: undefined, + questionAndAnswer: { + immutable: false, + question: 'What are 5 words that rhyme with bank?', + isQuestionValid: ValidatedOptions.success, + questionValidationError: undefined, + answer: 'tank, rank, prank, sank, and drank.', + isAnswerValid: ValidatedOptions.success, + answerValidationError: undefined + } + }, + { + immutable: false, + isExpanded: false, + context: undefined, + isContextValid: ValidatedOptions.success, + validationError: undefined, + questionAndAnswer: { + immutable: false, + question: 'What are 5 words that rhyme with bake?', + isQuestionValid: ValidatedOptions.success, + questionValidationError: undefined, + answer: 'wake, lake, steak, make, and quake.', + isAnswerValid: ValidatedOptions.success, + answerValidationError: undefined + } + } +]; + +export const autoFillSkillsFields: SkillFormData = { + branchName: `knowledge-contribution-${Date.now()}`, + email: 'helloworld@instructlab.com', + name: 'juliadenham', + submissionSummary: 'Teaching a model to rhyme.', + filePath: 'science/physics/astrophysics/stars', + seedExamples: skillsSeedExamples, + titleWork: 'Teaching a model to rhyme.', + licenseWork: 'CC-BY-SA-4.0', + creators: 'juliadenham' }; diff --git a/src/components/Contribute/ContributeAlertGroup.tsx b/src/components/Contribute/ContributeAlertGroup.tsx new file mode 100644 index 000000000..9e8bd472d --- /dev/null +++ b/src/components/Contribute/ContributeAlertGroup.tsx @@ -0,0 +1,65 @@ +// src/components/Contribute/Native/Knowledge/index.tsx +'use client'; +import React from 'react'; +import { AlertGroup, Alert, AlertActionCloseButton, Spinner, Button, Flex, FlexItem } from '@patternfly/react-core'; +import { ActionGroupAlertContent } from '@/components/Contribute/types'; +import { ExternalLinkAltIcon } from '@patternfly/react-icons'; + +export interface ContributeAlertGroupProps { + actionGroupAlertContent?: ActionGroupAlertContent; + onCloseActionGroupAlert: () => void; +} + +export const ContributeAlertGroup: React.FunctionComponent = ({ actionGroupAlertContent, onCloseActionGroupAlert }) => { + if (!actionGroupAlertContent) { + return null; + } + + return ( + + {actionGroupAlertContent ? ( + } + > + + + + {actionGroupAlertContent.waitAlert ? ( + + + + ) : null} + {actionGroupAlertContent.message} + + + {!actionGroupAlertContent.waitAlert && + actionGroupAlertContent.success && + actionGroupAlertContent.url && + actionGroupAlertContent.url.trim().length > 0 ? ( + + + + ) : null} + + + ) : null} + + ); +}; + +export default ContributeAlertGroup; diff --git a/src/components/Contribute/ContributionWizard/ContributionWizard.tsx b/src/components/Contribute/ContributionWizard/ContributionWizard.tsx new file mode 100644 index 000000000..35bc34ce4 --- /dev/null +++ b/src/components/Contribute/ContributionWizard/ContributionWizard.tsx @@ -0,0 +1,238 @@ +// src/components/Contribute/Knowledge/Github/index.tsx +'use client'; +import React from 'react'; +import { useSession } from 'next-auth/react'; +import { ContributionFormData, EditFormData } from '@/types'; +import { useRouter } from 'next/navigation'; +import { autoFillKnowledgeFields, autoFillSkillsFields } from '@/components/Contribute/AutoFill'; +import { + Breadcrumb, + BreadcrumbItem, + Button, + Content, + Flex, + FlexItem, + PageBreadcrumb, + PageGroup, + PageSection, + Title, + Wizard, + WizardStep +} from '@patternfly/react-core'; +import { getGitHubUserInfo } from '@/utils/github'; +import ContributionWizardFooter from '@/components/Contribute/ContributionWizard/ContributionWizardFooter'; +import { deleteDraftData } from '@/components/Contribute/Utils/autoSaveUtils'; + +import './contribute-page.scss'; + +export enum StepStatus { + Default = 'default', + Error = 'error', + Success = 'success' +} + +export interface StepType { + id: string; + name: string; + component?: React.ReactNode; + status?: StepStatus; + subSteps?: { + id: string; + name: string; + component?: React.ReactNode; + status?: StepStatus; + }[]; +} + +export interface Props { + title: React.ReactNode; + description: React.ReactNode; + editFormData?: EditFormData; + formData: ContributionFormData; + setFormData: React.Dispatch>; + isGithubMode: boolean; + isSkillContribution: boolean; + steps: StepType[]; + convertToYaml: (formData: ContributionFormData) => unknown; + onSubmit: (githubUsername: string) => Promise; +} + +export const ContributionWizard: React.FunctionComponent = ({ + title, + description, + editFormData, + formData, + setFormData, + isGithubMode, + isSkillContribution, + steps, + convertToYaml, + onSubmit +}) => { + const [devModeEnabled, setDevModeEnabled] = React.useState(); + const { data: session } = useSession(); + const [githubUsername, setGithubUsername] = React.useState(''); + const [submitEnabled, setSubmitEnabled] = React.useState(false); // **New State Added** + const [activeStepIndex, setActiveStepIndex] = React.useState(0); + + const router = useRouter(); + + const stepIds = React.useMemo( + () => + steps.reduce((acc, nextStep) => { + acc.push(nextStep.id); + if (nextStep.subSteps?.length) { + acc.push(...nextStep.subSteps.map((subStep) => subStep.id)); + } + return acc; + }, []), + [steps] + ); + const getStepIndex = (stepId: string) => stepIds.indexOf(stepId); + + React.useEffect(() => { + const getEnvVariables = async () => { + const res = await fetch('/api/envConfig'); + const envConfig = await res.json(); + setDevModeEnabled(envConfig.ENABLE_DEV_MODE === 'true'); + }; + getEnvVariables(); + }, []); + + React.useEffect(() => { + let canceled = false; + + if (isGithubMode) { + const fetchUserInfo = async () => { + if (session?.accessToken) { + try { + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${session.accessToken}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28' + }; + const fetchedUserInfo = await getGitHubUserInfo(headers); + if (!canceled) { + setGithubUsername(fetchedUserInfo.login); + setFormData((prev) => ({ + ...prev, + name: fetchedUserInfo.name, + email: fetchedUserInfo.email + })); + } + } catch (error) { + console.error('Failed to fetch GitHub user info:', error); + } + } + }; + fetchUserInfo(); + } else { + setFormData((prev) => ({ + ...prev, + name: session?.user?.name ? session.user.name : prev.name, + email: session?.user?.email ? session.user.email : prev.email + })); + } + + return () => { + canceled = true; + }; + }, [isGithubMode, session?.accessToken, session?.user?.name, session?.user?.email, setFormData]); + + const autoFillForm = (): void => { + setFormData(isSkillContribution ? { ...autoFillSkillsFields } : { ...autoFillKnowledgeFields }); + }; + + const handleCancel = () => { + //If there is any draft saved, delete it. + deleteDraftData(formData.branchName); + router.push('/dashboard'); + }; + + React.useEffect(() => { + setSubmitEnabled(!steps.find((step) => step.status === 'error')); + }, [steps]); + + return ( + + + + Contribute + {title} + + + + + + + + + {title} + + + + {devModeEnabled && ( + + )} + + + + + {description} + + + + + setActiveStepIndex(stepIds.indexOf(String(currentStep.id)))} + footer={ + onSubmit(githubUsername)} + showSubmit={submitEnabled} + isEdit={!!editFormData} + convertToYaml={convertToYaml} + /> + } + > + {steps.map((step) => ( + {step.name} }} + status={getStepIndex(step.id) < activeStepIndex ? step.status : StepStatus.Default} + steps={ + step.subSteps + ? step.subSteps.map((subStep) => ( + {subStep.name} }} + status={getStepIndex(subStep.id) < activeStepIndex ? subStep.status : StepStatus.Default} + > + {subStep.component} + + )) + : undefined + } + > + {!step.subSteps ? step.component : null} + + ))} + + + + ); +}; + +export default ContributionWizard; diff --git a/src/components/Contribute/ContributionWizard/ContributionWizardFooter.tsx b/src/components/Contribute/ContributionWizard/ContributionWizardFooter.tsx new file mode 100644 index 000000000..c25693835 --- /dev/null +++ b/src/components/Contribute/ContributionWizard/ContributionWizardFooter.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import { + ActionList, + ActionListGroup, + ActionListItem, + Button, + ButtonVariant, + Flex, + FlexItem, + useWizardContext, + WizardFooterWrapper +} from '@patternfly/react-core'; +import { ArrowRightIcon } from '@patternfly/react-icons'; +import ViewDropdownButton from '@/components/Contribute/ContributionWizard/ViewDropdownButton'; +import { ContributionFormData } from '@/types'; + +interface Props { + formData: ContributionFormData; + isGithubMode: boolean; + showSubmit: boolean; + isEdit: boolean; + onSubmit: () => Promise; + convertToYaml: (formData: ContributionFormData) => unknown; + onCancel: () => void; +} + +const ContributionWizardFooter: React.FC = ({ formData, isGithubMode, showSubmit, onSubmit, convertToYaml, isEdit }) => { + const { steps, activeStep, goToNextStep, goToPrevStep, goToStepByIndex, close } = useWizardContext(); + + const prevDisabled = steps.indexOf(activeStep) < 1; + const isLast = steps.indexOf(activeStep) === steps.length - 1; + + const handleSubmit = async () => { + const result = await onSubmit(); + if (result) { + goToStepByIndex(0); + } + }; + + return ( + + + + + + + + + {!isLast || !showSubmit ? ( + + + + ) : null} + {showSubmit ? ( + + + + ) : null} + + + + + + + + + + + + + + + + + + ); +}; + +export default ContributionWizardFooter; diff --git a/src/components/Contribute/ContributionWizard/ViewDropdownButton.tsx b/src/components/Contribute/ContributionWizard/ViewDropdownButton.tsx new file mode 100644 index 000000000..7e999175a --- /dev/null +++ b/src/components/Contribute/ContributionWizard/ViewDropdownButton.tsx @@ -0,0 +1,155 @@ +import React, { useState } from 'react'; +import YamlCodeModal from '@/components/YamlCodeModal'; +import { AttributionData, ContributionFormData, KnowledgeFormData } from '@/types'; +import { dumpYaml } from '@/utils/yamlConfig'; +import { Dropdown, MenuToggleElement, Icon, DropdownList, DropdownItem, Button, Flex, FlexItem } from '@patternfly/react-core'; +import { CodeIcon, FileIcon, CaretDownIcon } from '@patternfly/react-icons'; + +interface Props { + formData: ContributionFormData; + convertToYaml: (formData: ContributionFormData) => unknown; + isGithubMode: boolean; +} + +export const ViewDropdownButton: React.FunctionComponent = ({ formData, convertToYaml, isGithubMode }) => { + const [isOpen, setIsOpen] = useState(false); + const [isYamlModalOpen, setIsYamlModalOpen] = useState(false); + const [isAttributionModalOpen, setIsAttributionModalOpen] = useState(false); + const [modalContent, setModalContent] = useState(''); + + const handleViewYaml = () => { + const yamlData = convertToYaml(formData); + const yamlString = dumpYaml(yamlData); + setModalContent(yamlString); + setIsYamlModalOpen(true); + }; + + const handleSaveYaml = () => { + const yamlData = convertToYaml(formData); + const yamlString = dumpYaml(yamlData); + const blob = new Blob([yamlString], { type: 'application/x-yaml' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'qna.yaml'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }; + + const handleViewAttribution = () => { + const attributionData: AttributionData = { + title_of_work: formData.titleWork!, + link_to_work: (formData as KnowledgeFormData).linkWork!, + revision: (formData as KnowledgeFormData).revision!, + license_of_the_work: formData.licenseWork!, + creator_names: formData.creators + }; + const attributionString = dumpYaml(attributionData); + setModalContent(attributionString); + setIsAttributionModalOpen(true); + }; + + const handleSaveAttribution = () => { + const attributionData: AttributionData = { + title_of_work: formData.titleWork!, + link_to_work: (formData as KnowledgeFormData).linkWork!, + revision: (formData as KnowledgeFormData).revision!, + license_of_the_work: formData.licenseWork!, + creator_names: formData.creators + }; + const yamlString = dumpYaml(attributionData); + const blob = new Blob([yamlString], { type: 'application/x-yaml' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'attribution.yaml'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }; + + const onToggleClick = () => { + setIsOpen(!isOpen); + }; + + const onSelect = () => { + // eslint-disable-next-line no-console + setIsOpen(false); + }; + + return ( + <> + {isYamlModalOpen ? ( + setIsYamlModalOpen(!isYamlModalOpen)} + yamlContent={modalContent} + onSave={handleSaveYaml} + /> + ) : null} + {isAttributionModalOpen ? ( + setIsAttributionModalOpen(!isAttributionModalOpen)} + yamlContent={modalContent} + onSave={handleSaveAttribution} + /> + ) : null} + {isGithubMode ? ( + setIsOpen(isOpen)} + toggle={(toggleRef: React.Ref) => ( + + )} + ouiaId="DownloadDropdown" + shouldFocusToggleOnSelect + > + + + + + } + > + {' '} + YAML Content + + {isGithubMode && ( + + + + } + > + {' '} + Attribution Content + + )} + + + ) : ( + + )} + + ); +}; + +export default ViewDropdownButton; diff --git a/src/components/Contribute/ContributionWizard/contribute-page.scss b/src/components/Contribute/ContributionWizard/contribute-page.scss new file mode 100644 index 000000000..c3bc29022 --- /dev/null +++ b/src/components/Contribute/ContributionWizard/contribute-page.scss @@ -0,0 +1,10 @@ +.contribute-page { + .pf-v6-c-page__main { + overflow-y: hidden; + } + .pf-v6-c-page__main-section.pf-m-fill { + .pf-v6-c-page__main-body { + height: 100%; + } + } +} diff --git a/src/components/Contribute/DetailsPage/DetailsPage.tsx b/src/components/Contribute/DetailsPage/DetailsPage.tsx new file mode 100644 index 000000000..e429c2f56 --- /dev/null +++ b/src/components/Contribute/DetailsPage/DetailsPage.tsx @@ -0,0 +1,177 @@ +import React, { useEffect } from 'react'; +import { + Button, + Content, + Flex, + FlexItem, + Form, + FormGroup, + FormHelperText, + HelperText, + HelperTextItem, + TextArea, + ValidatedOptions +} from '@patternfly/react-core'; +import { ExclamationCircleIcon, PencilAltIcon } from '@patternfly/react-icons'; +import { + t_global_spacer_sm as SmallSpacerSize, + t_global_font_size_xs as XsFontSize, + t_global_icon_color_subtle as SubtleIconColor +} from '@patternfly/react-tokens'; +import PathService from '@/components/PathService/PathService'; +import WizardPageHeader from '@/components/Common/WizardPageHeader'; +import WizardSectionHeader from '@/components/Common/WizardSectionHeader'; +import { MAX_SUMMARY_CHARS } from '@/components/Contribute/Utils/validationUtils'; +import EditContributorModal from '@/components/Contribute/DetailsPage/EditContributorModal'; + +interface Props { + isGithubMode: boolean; + infoSectionHelp?: React.ReactNode; + isEditForm?: boolean; + email: string; + setEmail: (val: string) => void; + name: string; + setName: (val: string) => void; + submissionSummary: string; + setSubmissionSummary: (val: string) => void; + rootPath: string; + filePath: string; + setFilePath: (val: string) => void; +} +const DetailsPage: React.FC = ({ + isGithubMode, + infoSectionHelp, + isEditForm, + email, + setEmail, + name, + setName, + submissionSummary, + setSubmissionSummary, + rootPath, + filePath, + setFilePath +}) => { + const [editContributorOpen, setEditContributorOpen] = React.useState(); + const [validSummary, setValidSummary] = React.useState(ValidatedOptions.default); + + useEffect(() => { + if (isEditForm) { + setValidSummary(ValidatedOptions.success); + } + }, [isEditForm]); + + const validateSummary = (summaryStr: string) => { + const summary = summaryStr.trim(); + setValidSummary(summary.length > 0 && summary.length <= MAX_SUMMARY_CHARS ? ValidatedOptions.success : ValidatedOptions.error); + }; + + const isSummaryInvalid = + validSummary === ValidatedOptions.error && (submissionSummary.trim().length > MAX_SUMMARY_CHARS || submissionSummary.trim().length === 0); + + return ( + + + + + + + + +
+ + Contributor + + + } + isRequired + > + {name} + {email} + {!name || !email ? ( + + + Name and email are required + + + ) : null} + + {editContributorOpen ? ( + { + setName(name); + setEmail(email); + setEditContributorOpen(false); + }} + onClose={() => setEditContributorOpen(false)} + /> + ) : null} + +
+ + +
+ +