empirical-org · brendanshean · Apr 3, 2025 · Apr 3, 2025
diff --git a/...S/engines/evidence/app/controllers/evidence/research/gen_ai/step_executions_controller.rb b/...S/engines/evidence/app/controllers/evidence/research/gen_ai/step_executions_controller.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Evidence
+  module Research
+    module GenAI
+      class StepExecutionsController < ApplicationController
+        def show
+          @step_execution = StepExecution.find(params[:id])
+        end
+      end
+    end
+  end
+end
diff --git a/...s/QuillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/trials_controller.rb b/...s/QuillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/trials_controller.rb
@@ -16,6 +16,7 @@ def new
           @suboptimal_guidelines = dataset.guidelines.suboptimal.visible
           @prompt_examples = dataset.prompt_examples
           @dataset_relevant_texts = @dataset.dataset_relevant_texts.includes(:relevant_text)
+          @workflows = Workflow.templates.order(created_at: :desc)
 
           respond_to do |format|
             format.html
@@ -34,16 +35,78 @@ def new
         end
 
         def create
-          llm_prompt = LLMPrompt.create_from_template!(
-            dataset_id:, guideline_ids:, llm_prompt_template_id:, prompt_example_ids:, relevant_text_id:
-          )
+          trial = dataset.trials.new(llm_id:, temperature:, notes:)
+
+          # Get the form parameters directly from the request
+          feedback_type = params[:research_gen_ai_trial][:feedback_type]
+          template_type = params[:template_type] # This comes directly from HTML form
+          ad_hoc_prompt = params[:ad_hoc_prompt] # This comes directly from HTML form
+          prompt_template1 = params[:prompt_template1] # This comes directly from HTML form
+          prompt_template2 = params[:prompt_template2] # This comes directly from HTML form
+
+          if feedback_type == 'workflow'
+            trial.llm_prompt_id = nil
+
+            if template_type == 'template'
+              # Use existing workflow template
+              workflow = Workflow.find(trial_create_params[:workflow_id])
+
+              if workflow
+                trial.workflow_id = workflow.duplicate.id
+              else
+                flash[:alert] = 'Failed to create a copy of the selected workflow'
+                redirect_to new_research_gen_ai_trial_path(dataset) and return
+              end
+            else
+              workflow = Workflow.find_by(template_id: nil, name: 'Ad Hoc Workflow').duplicate
+
+              # Parse config values
+              prompt_chain_step1_config = JSON.parse(params[:prompt_chain_step1_config] || '{}')
+              prompt_chain_step2_config = JSON.parse(params[:prompt_chain_step2_config] || '{}')
+
+              step1 = workflow.workflow_steps.create!(
+                name: 'Step 1',
+                type: 'Evidence::Research::GenAI::PromptChainStep',
+                prompt_template: prompt_template1,
+                config: prompt_chain_step1_config
+              )
+
+              step2 = workflow.workflow_steps.create!(
+                name: 'Step 2',
+                type: 'Evidence::Research::GenAI::PromptChainStep',
+                prompt_template: prompt_template2,
+                config: prompt_chain_step2_config
+              )
+
+              StepConnection.create!(source_step_id: step1.id, target_step_id: step2.id)
+
+              trial.workflow_id = workflow.id
+            end
+          else
+            # Handle prompt template feedback type
+            trial.workflow_id = nil
+
+            if template_type == 'template'
+              trial.llm_prompt = LLMPrompt.create_from_template!(
+                dataset_id:, guideline_ids:, llm_prompt_template_id:, prompt_example_ids:, relevant_text_id:
+              )
+            else
+              trial.llm_prompt = LLMPrompt.create!(
+                prompt: ad_hoc_prompt,
+                optimal_examples_count: 0,
+                optimal_guidelines_count: 0,
+                suboptimal_examples_count: 0,
+                suboptimal_guidelines_count: 0,
+                locked: true
+              )
+
+            end
+          end
 
-          trial = dataset.trials.new(llm_id:, llm_prompt:, temperature:, notes:)
           results = { g_eval_ids: [g_eval_id] }
           results = results.merge(rag_limit:) if trial.classification?
           trial.results = results
 
-          # TODO: clean this up once we have full functionality in the Evidence CMS and no longer need the erb tool
           respond_to do |format|
             format.html do
               if trial.save
@@ -53,6 +116,7 @@ def create
                 render :new
               end
             end
+
             format.json do
               if trial.save
                 RunTrialWorker.perform_async(trial.id)
@@ -79,6 +143,7 @@ def show
           @previous = Trial.where('id < ?', @trial.id).order(id: :desc).first
           @g_evals = GEval.where(id: @trial.g_eval_ids)
           active_activated_llm_prompt = dataset.stem_vault.prompt.active_activated_llm_prompt
+          @workflow_execution_id_lookup = trial.workflow_executions.pluck(:test_example_id, :id).to_h if trial.workflow
 
           @g_eval_scores =
             @trial
@@ -124,6 +189,7 @@ def show
               :llm_id,
               :llm_prompt_template_id,
               :rag_limit,
+              :workflow_id,
               :relevant_text_id,
               :temperature,
               :notes,

diff --git a/...gines/evidence/app/controllers/evidence/research/gen_ai/workflow_executions_controller.rb b/...gines/evidence/app/controllers/evidence/research/gen_ai/workflow_executions_controller.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Evidence
+  module Research
+    module GenAI
+      class WorkflowExecutionsController < ApplicationController
+        before_action :set_workflow, except: [:show]
+        before_action :set_workflow_execution, only: [:show, :run]
+
+        def index
+          @workflow_executions = @workflow ? @workflow.workflow_executions : WorkflowExecution.includes(:workflow)
+          @workflow_executions = @workflow_executions.where(status: params[:status]) if params[:status].present?
+          @workflow_executions = @workflow_executions.order(created_at: :desc)
+        end
+
+        def show
+          @workflow = @workflow_execution.workflow
+        end
+
+        def new
+          @workflow_execution = @workflow.workflow_executions.build
+          @test_examples = TestExample.order(created_at: :desc).limit(100)
+        end
+
+        def create
+          @workflow_execution = @workflow.workflow_executions.build(workflow_execution_params)
+
+          if @workflow_execution.save
+            if params[:run_now] == '1' || params[:commit] =~ /run/i
+              @workflow_execution.run
+            end
+            redirect_to research_gen_ai_workflow_execution_path(@workflow_execution)
+          else
+            @test_examples = TestExample.order(created_at: :desc).limit(100)
+            render :new
+          end
+        end
+
+        private def set_workflow
+          @workflow = Workflow.find(params[:workflow_id]) if params[:workflow_id]
+        end
+
+        private def set_workflow_execution
+          @workflow_execution = WorkflowExecution.find(params[:id])
+        end
+
+        private def workflow_execution_params
+          params.require(:workflow_execution).permit(:test_example_id, :student_response, :misc)
+        end
+      end
+    end
+  end
+end
diff --git a/...MS/engines/evidence/app/controllers/evidence/research/gen_ai/workflow_steps_controller.rb b/...MS/engines/evidence/app/controllers/evidence/research/gen_ai/workflow_steps_controller.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Evidence
+  module Research
+    module GenAI
+      class WorkflowStepsController < ApplicationController
+        before_action :set_workflow
+        before_action :set_workflow_step, only: [:show, :edit, :update, :destroy]
+
+        def index
+          @workflow_steps = @workflow.workflow_steps.order(created_at: :asc)
+        end
+
+        def new
+          @workflow_step = @workflow.workflow_steps.build
+          @available_step_types = WorkflowStep::TYPES
+          @available_steps = @workflow.workflow_steps.where.not(id: params[:id])
+        end
+
+        def create
+          @workflow_step = @workflow.workflow_steps.build(workflow_step_params)
+
+          if @workflow_step.save
+            create_step_connections if params[:previous_step_ids].present?
+            redirect_to research_gen_ai_workflow_path(@workflow)
+          else
+            @available_step_types = WorkflowStep::TYPES
+            @available_steps = @workflow.workflow_steps.where.not(id: @workflow_step.id)
+            render :new, status: :unprocessable_entity
+          end
+        end
+
+        def edit
+          @available_step_types = WorkflowStep::TYPES
+          @available_steps = @workflow.workflow_steps.where.not(id: @workflow_step.id)
+          @current_previous_steps = @workflow_step.previous_steps
+        end
+
+        private
+
+        def set_workflow
+          @workflow = Workflow.find(params[:workflow_id])
+        end
+
+        def set_workflow_step
+          @workflow_step = @workflow.workflow_steps.find(params[:id])
+        end
+
+        def workflow_step_params
+          params
+            .require(:research_gen_ai_workflow_step)
+            .permit(:name, :type, :prompt_template, :config)
+        end
+
+        def create_step_connections
+          previous_step_ids = params[:previous_step_ids] || []
+          previous_step_ids.each do |previous_step_id|
+            StepConnection.create(
+              source_step_id: previous_step_id,
+              target_step_id: @workflow_step.id
+            )
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/...uillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/workflows_controller.rb b/...uillLMS/engines/evidence/app/controllers/evidence/research/gen_ai/workflows_controller.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Evidence
+  module Research
+    module GenAI
+      class WorkflowsController < ApplicationController
+        def index
+          @workflows = Workflow.where(template_id: nil).order(created_at: :desc)
+        end
+
+        def show
+          @workflow = Workflow.find(params[:id])
+          @workflow_steps = @workflow.workflow_steps.includes(:previous_steps, :next_steps)
+          @step_connections = StepConnection.where(source_step_id: @workflow_steps.pluck(:id))
+            .or(StepConnection.where(target_step_id: @workflow_steps.pluck(:id)))
+
+          @workflow_executions = @workflow.workflow_executions.order(created_at: :desc).limit(10)
+        end
+
+        def new
+          @workflow = Workflow.new
+        end
+
+        def create
+          @workflow = Workflow.new(workflow_params)
+
+          if @workflow.save
+            redirect_to research_gen_ai_workflow_path(@workflow)
+          else
+            render :new, status: :unprocessable_entity
+          end
+        end
+
+        private def workflow_params
+          params
+            .require(:research_gen_ai_workflow)
+            .permit(:name, :input)
+        end
+      end
+    end
+  end
+end
diff --git a/...ices/QuillLMS/engines/evidence/app/helpers/evidence/research/gen_ai/application_helper.rb b/...ices/QuillLMS/engines/evidence/app/helpers/evidence/research/gen_ai/application_helper.rb
@@ -44,6 +44,31 @@ def percent_accuracy(numerator, denominator)
           percent = (100.0 * numerator / denominator).round
           "#{percent}% (#{numerator}/#{denominator})"
         end
+
+        def highlight_template_variables(template)
+          return template unless template.is_a?(String)
+
+          # Mark template variables with a special HTML class
+          template.gsub(/%\{([a-zA-Z0-9_]+)\}/) do |match|
+            "<span class='template-variable'>#{match}</span>"
+          end.html_safe
+        end
+
+        def highlight_filled_variables(template, filled_template, variables)
+          return filled_template unless template.is_a?(String) && filled_template.is_a?(String) && variables.is_a?(Hash)
+
+          # Replace template variables with highlighted filled values
+          result = template.dup
+
+          variables.each do |key, value|
+            placeholder = "%{#{key}}"
+            if result.include?(placeholder)
+              result.gsub!(placeholder, "<span class='filled-variable'>#{value}</span>")
+            end
+          end
+
+          result.html_safe
+        end
       end
     end
   end

diff --git a/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/prompt_chain_step.rb b/services/QuillLMS/engines/evidence/app/models/evidence/research/gen_ai/prompt_chain_step.rb
@@ -20,7 +20,22 @@ class PromptChainStep < WorkflowStep
         class MissingTemplateVariableError < StandardError; end
         validates :prompt_template, presence: true
 
-        def run(input) = llm.completion(generate_prompt(input))
+        def run(input, test_example_id)
+          llm_foo = llm_blah(input)
+          llm_output = JSON.parse(llm_foo)
+          feedback = llm_output['feedback']
+
+          return llm_output.to_json unless evaluate_feedback?
+
+          evaluation = Evidence::GenAI::Evaluation::FeedbackEvaluator.run(entry: llm_output, feedback:)
+          evaluation_optimal = evaluation.optimal
+          feedback_errors = evaluation.errors.join(', ')
+          feedback_revision = FeedbackRevision.find_by(title: feedback_errors).prompt_text
+
+          llm_output.merge!(evaluation_optimal:, feedback_errors:, feedback_revision:).compact.to_json
+        end
+
+        def llm_blah(input) = llm.completion(generate_prompt(input))
 
         def generate_prompt(input)
           prompt_template % JSON.parse(input.first['template_variables']).symbolize_keys