Skip to content

Commit

Permalink
Merge pull request #86 from DigitalNZ/rm/activity
Browse files Browse the repository at this point in the history 
See running pipeline jobs
  • Loading branch information
@richardmatthewsdev
richardmatthewsdev authored Sep 1, 2024
2 parents 29c70aa + d6a271f commit 787ea80
Show file tree
Hide file tree
Showing 8 changed files with 225 additions and 93 deletions.
11 changes: 10 additions & 1 deletion app/controllers/pipelines_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,17 @@ class PipelinesController < ApplicationController
before_action :assign_destinations, only: %w[show update]

def index
@pipelines = pipelines
@pipeline = Pipeline.new
status = params['status']

@pipelines = if status == 'queued'
PipelineJob.where.missing(:harvest_reports).map(&:pipeline).uniq
elsif status == 'running'
HarvestReport.running.map { |report| report.pipeline_job.pipeline }
.uniq
else
pipelines
end
end

def show; end
Expand Down
3 changes: 2 additions & 1 deletion app/frontend/stylesheets/bootstrap/_card.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
background-color: white;
width: 100%;
position: relative;
color: $black;

.card-title {
font-size: 1.125rem;
Expand Down Expand Up @@ -51,4 +52,4 @@ form {
.card {
background: $doctor;
}
}
}
13 changes: 11 additions & 2 deletions app/frontend/stylesheets/bootstrap/_typography.scss
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,16 @@
h1, h2, h3, h4 {
h2,
h3,
h4 {
font-weight: 700;
}

h5, h6, h7 {
h5,
h6,
h7 {
font-weight: 600;
}

h1 {
font-size: 2rem;
font-weight: 900;
}
14 changes: 10 additions & 4 deletions app/models/harvest_report.rb
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
# frozen_string_literal: true

class HarvestReport < ApplicationRecord
scope :completed, lambda {
where(extraction_status: 'completed', transformation_status: 'completed',
load_status: 'completed', delete_status: 'completed')
}

scope :active, -> { completed.invert_where.order(created_at: :desc) }
scope :running, -> { active.select { |report| report.status == 'running' } }

belongs_to :pipeline_job, optional: true
belongs_to :harvest_job, optional: true

Expand All @@ -18,8 +26,7 @@ class HarvestReport < ApplicationRecord
enum :kind, { harvest: 0, enrichment: 1 }

METRICS = %w[
pages_extracted records_transformed
records_loaded records_rejected
pages_extracted records_transformed records_loaded records_rejected
records_deleted transformation_workers_queued
transformation_workers_completed load_workers_queued
load_workers_completed delete_workers_queued
Expand All @@ -31,8 +38,7 @@ class HarvestReport < ApplicationRecord
extraction_end_time transformation_start_time
transformation_updated_time transformation_end_time
load_start_time load_updated_time
load_end_time delete_start_time
delete_updated_time delete_end_time
load_end_time delete_start_time delete_updated_time delete_end_time
].freeze

def completed?
Expand Down
2 changes: 1 addition & 1 deletion app/models/pipeline.rb
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# frozen_string_literal: true

class Pipeline < ApplicationRecord
paginates_per 19 # not 20 because of the "Create new pipeline" button
paginates_per 20

has_many :harvest_definitions, dependent: :destroy
has_many :harvest_jobs, through: :harvest_definitions
Expand Down
25 changes: 24 additions & 1 deletion app/views/extraction_definitions/_create_edit_enrichment_modal.html.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,10 @@
<%= form.hidden_field :page, value: 1 %>
<%= form.hidden_field :per_page, value: 20 %>
<%= form.hidden_field :total_selector, value: '$.meta.total_pages' %>
<%= form.hidden_field :source_id, value: @pipeline.harvest.source_id %>

<% if !model.persisted? %>
<%= form.hidden_field :source_id, value: @pipeline.harvest.source_id %>
<% end %>

<% if model.persisted? %>
<div class="col-4">
Expand All @@ -41,6 +44,26 @@
</div>
<% end %>

<% if model.persisted? %>
<div class="col-4">
<%= form.label :source_id, class: 'form-label' do %>
Source ID

<span
data-bs-toggle="tooltip"
data-bs-title="The Source ID of the harvest you want to enrich"><i class="bi bi-question-circle" aria-label="helper text"></i>
</span>
<% end %>
</div>

<div class='col-8'>
<%= form.text_field :source_id,
class: {
'form-control': true
}, required: true %>
</div>
<% end %>

<div class="col-4">
<%= form.label :base_url, class: 'form-label' do %>
Base URL
Expand Down
181 changes: 98 additions & 83 deletions app/views/pipelines/index.html.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,100 +5,115 @@

<h1 class="header__title">Pipelines</h1>

<div class="header__actions">
<button type="button" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#create-modal">
+ Create new pipeline
</button>
</div>

<div class="clearfix"></div>
<% end %>

<%= form_with url: '/pipelines', method: :get, class: 'row justify-content-end mb-5' do |form| %>
<div class="col-auto pe-2">
<div class="search">
<%= form.text_field(
:search,
value: params[:search],
placeholder: 'Search by name, description, source_id, last edited by',
class: 'form-control search__input'
) %>
<%= form.label :search, class: 'search__label' do %>
<i aria-hidden="true" class="bi bi-search"></i>
Search by name, description, source_id, last edited by
<% end %>

<% if params[:search].present? %>
<button type="button" class="btn btn-secondary search__clear" data-clear-field="search">
Clear
<i aria-hidden="true" class="bi bi-plus"></i>
</button>
<% end %>
</div>
</div>
<div class="col-auto ps-0">
<button class="btn btn-primary" type="submit">Search</button>
</div>
<div class='mt-4'></div>
<ul class="nav nav-tabs">

<div class="col-auto pe-0">
<%= form.label :format, 'Format:', class: 'col-form-label' %>
</div>
<div class="col-auto">
<%= form.select(
:format,
[['All formats', '']] + ExtractionDefinition::FORMATS.map { |format| [format, format] },
{ selected: params[:format].in?([nil, '', 'All formats']) ? 'All formats' : params[:format] },
class: 'form-select', 'data-submitting-select': true
) %>
</div>
<li class='nav-item'>
<%= link_to 'All', pipelines_path, class: "nav-link #{'active' if params['status'].nil?}" %>
</li>

<div class="col-auto pe-0">
<%= form.label :sort_by, 'Sort by:', class: 'col-form-label' %>
</div>
<div class="col-auto">
<%= form.select(
:sort_by,
[['Last Edited', 'updated_at'], ['Alphabetical', 'name']],
{ selected: @sort_by.keys.first },
class: 'form-select', 'data-submitting-select': true
) %>
</div>
<% end %>
<li class='nav-item'>
<%= link_to 'Queued', pipelines_path(status: 'queued'),
class: "nav-link #{'active' if params['status'] == 'queued'}" %>
</li>

<div class='row'>
<li class='nav-item'>
<%= link_to 'Running', pipelines_path(status: 'running'),
class: "nav-link #{'active' if params['status'] == 'running'}" %>
</li>
</ul>
<% end %>

<div class='col-3 mb-3'>
<button type="button" class="card mb-3 card--create-cta d-flex h-100" data-bs-toggle="modal" data-bs-target="#create-modal">
<div class="card-body align-items-center d-flex justify-content-center">
<h2 class="card-title">+ Create new pipeline</h5>
<% if params['status'].nil? %>
<%= form_with url: '/pipelines', method: :get, class: 'row mb-5' do |form| %>
<div class="col-auto pe-2">
<div class="search">
<%= form.text_field(
:search,
value: params[:search],
placeholder: 'Search by name, description, source_id, last edited by',
class: 'form-control search__input'
) %>
<%= form.label :search, class: 'search__label' do %>
<i aria-hidden="true" class="bi bi-search"></i>
Search by name, description, source_id, last edited by
<% end %>

<% if params[:search].present? %>
<button type="button" class="btn btn-secondary search__clear" data-clear-field="search">
Clear
<i aria-hidden="true" class="bi bi-plus"></i>
</button>
<% end %>
</div>
</button>
</div>
</div>

<%- @pipelines.each do |pipeline| %>
<div class='col-3 mb-3'>
<%= link_to pipeline, class: 'card card--clickable mb-3 d-flex h-100' do %>
<div class="card-body">
<h2 class="card-title"><%= pipeline.name %></h2>
<h3 class="card-subtitle">
<%= last_edited_by(pipeline) %>
</h3>
<div>
<span class="badge bg-light text-dark">
<%= pipeline.harvest_definitions.harvest.count %> Harvest
</span>
<span class="badge bg-light text-dark">
<%= pipeline.harvest_definitions.enrichment.count %> Enrichments
</span>

<% if pipeline.schedules.any? %>
<span class="badge bg-primary">
Scheduled
</span>
<% end %>
</div>
</div>
<% end %>
<div class="col-4 ps-0">
<button class="btn btn-primary" type="submit">Search</button>
</div>
<%- end %>

</div>
<div class="col-auto pe-0">
<%= form.label :format, 'Format:', class: 'col-form-label' %>
</div>
<div class="col-auto">
<%= form.select(
:format,
[['All formats', '']] + ExtractionDefinition::FORMATS.map { |format| [format, format] },
{ selected: params[:format].in?([nil, '', 'All formats']) ? 'All formats' : params[:format] },
class: 'form-select', 'data-submitting-select': true
) %>
</div>

<%= render 'shared/pagination_below_table', items: @pipelines %>
<div class="col-auto pe-0">
<%= form.label :sort_by, 'Sort by:', class: 'col-form-label' %>
</div>
<div class="col-auto">
<%= form.select(
:sort_by,
[['Last Edited', 'updated_at'], ['Alphabetical', 'name']],
{ selected: @sort_by.keys.first },
class: 'form-select', 'data-submitting-select': true
) %>
</div>
<% end %>
<% end %>

<%- @pipelines.each do |pipeline| %>
<%= link_to pipeline, class: 'card card--clickable mb-3' do %>
<div class="card-body">
<h2 class="card-title"><%= pipeline.name %></h2>
<h3 class="card-subtitle">
<%= last_edited_by(pipeline) %>
</h3>
<div>
<span class="badge bg-light text-dark">
<%= pipeline.harvest_definitions.harvest.count %> Harvest
</span>
<span class="badge bg-light text-dark">
<%= pipeline.harvest_definitions.enrichment.count %> Enrichments
</span>

<% if pipeline.schedules.any? %>
<span class="badge bg-primary">
Scheduled
</span>
<% end %>
</div>
</div>
<% end %>
<%- end %>

<% if params['status'].nil? %>
<%= render 'shared/pagination_below_table', items: @pipelines %>
<% end %>

<%= render layout: 'shared/create_modal',
locals: { modal_heading: 'Create new pipeline', button_text: 'Create pipeline' } do %>
Expand Down
Loading

0 comments on commit 787ea80

Please sign in to comment.