Merge branch 'dev' into fix/AACT-716-Update-Tests-for-TableExporter

app/models/study.rb

@@ -112,50 +112,6 @@ def self.all_nctids
     all.collect{|s|s.nct_id}
   end
 
-  def create
-    ActiveRecord::Base.logger=nil
-    s=Study.where('nct_id=?',nct_id).first
-    s.try(:destroy)
-    update(attribs)
-    groups=DesignGroup.create_all_from(opts)
-    Intervention.create_all_from(opts.merge(:design_groups=>groups))
-    DetailedDescription.new.create_from(opts).try(:save)
-    Design.new.create_from(opts).try(:save)
-    BriefSummary.new.create_from(opts).try(:save)
-    Eligibility.new.create_from(opts).save
-    ParticipantFlow.new.create_from(opts).try(:save)
-
-    BaselineMeasurement.create_all_from(opts)
-    BrowseCondition.create_all_from(opts)
-    BrowseIntervention.create_all_from(opts)
-    CentralContact.create_all_from(opts)
-    Condition.create_all_from(opts)
-    Country.create_all_from(opts)
-    Document.create_all_from(opts)
-    Facility.create_all_from(opts)
-    IdInformation.create_all_from(opts)
-    IpdInformationType.create_all_from(opts)
-    Keyword.create_all_from(opts)
-    Link.create_all_from(opts)
-    Milestone.create_all_from(opts)
-    Outcome.create_all_from(opts)
-    OverallOfficial.create_all_from(opts)
-    DesignOutcome.create_all_from(opts)
-    PendingResult.create_all_from(opts)
-    ProvidedDocument.create_all_from(opts)
-    ReportedEvent.create_all_from(opts)
-    ResponsibleParty.create_all_from(opts)
-    ResultAgreement.create_all_from(opts)
-    ResultContact.create_all_from(opts)
-    Reference.create_all_from(opts)
-    Sponsor.create_all_from(opts)
-    # During full load, indexes are dropped. Populating CalculatedValues requires several db queries - so they're scanned and very slow.
-    # Populate the CalculatedValues after the indexes have been recreated after the full load completes.
-    CalculatedValue.new.create_from(self).save if ENV['load_type'] == 'incremental'
-    self.set_downcase if ENV['load_type'] == 'incremental'
-    self
-  end
-
   def set_downcase
     con=ActiveRecord::Base.connection
     con.execute("UPDATE browse_conditions SET downcase_mesh_term=lower(mesh_term) where nct_id = '#{nct_id}';")
@@ -230,112 +186,6 @@ def name
     brief_title
   end
 
-  def self.mapper(json)
-    return unless json.protocol_section
-
-    ident = json.protocol_section['identificationModule']
-    nct_id = ident['nctId']
-    status = json.protocol_section['statusModule']
-    design = key_check(json.protocol_section['designModule'])
-    oversight = key_check(json.protocol_section['oversightModule'])
-    ipd_sharing = key_check(json.protocol_section['ipdSharingStatementModule'])
-    study_posted = status['studyFirstPostDateStruct']
-    results_posted = key_check(status['resultsFirstPostDateStruct'])
-    disp_posted = key_check(status['dispFirstPostDateStruct'])
-    last_posted = status['lastUpdatePostDateStruct']
-    start_date = key_check(status['startDateStruct'])
-    completion_date = key_check(status['completionDateStruct'])
-    primary_completion_date = key_check(status['primaryCompletionDateStruct'])
-    results = json.results_section || {}
-    more_info = results['moreInfoModule']
-    baseline = key_check(results['baselineCharacteristicsModule'])
-    enrollment = key_check(design['enrollmentInfo'])
-    expanded_access = status.dig('expandedAccessInfo', 'hasExpandedAccess')
-    expanded = key_check(design['expandedAccessTypes'])
-    biospec = key_check(design['bioSpec'])
-    arms_intervention = key_check(json.protocol_section['armsInterventionsModule'])
-    study_type = design['studyType']
-    patient_registry = design['patientRegistry'] || ''
-    study_type = "#{study_type} [Patient Registry]" if patient_registry =~ /Yes/i
-    groups = key_check(arms_intervention['armGroups'])
-    num_of_groups = groups.count == 0 ? nil : groups.count
-    arms_count = study_type =~ /Interventional/i ? num_of_groups : nil
-    groups_count = arms_count ? nil : num_of_groups
-    phase_list = design['phases']
-    phase_list = phase_list.join('/') if phase_list
-
-    {
-      nct_id: nct_id,
-      nlm_download_date_description: nil,
-      study_first_submitted_date: convert_to_date(status['studyFirstSubmitDate']),
-      study_first_submitted_qc_date: convert_to_date(status['studyFirstSubmitQcDate']),
-      study_first_posted_date: convert_to_date(study_posted['date']),
-      study_first_posted_date_type: study_posted['type'],
-      results_first_submitted_date: convert_to_date(status['resultsFirstSubmitDate']),
-      results_first_submitted_qc_date: status['resultsFirstSubmitQcDate'],
-      results_first_posted_date: results_posted['date'],
-      results_first_posted_date_type: results_posted['type'],
-      disposition_first_submitted_date: convert_to_date(status['dispFirstSubmitDate']),
-      disposition_first_submitted_qc_date: status['dispFirstSubmitQcDate'],
-      disposition_first_posted_date: disp_posted['date'],
-      disposition_first_posted_date_type: disp_posted['type'],
-      last_update_submitted_date: convert_to_date(status['lastUpdateSubmitDate']),
-      last_update_submitted_qc_date: convert_to_date(status['lastUpdateSubmitDate']), # this should not go here (Ramiro comment)
-      last_update_posted_date: convert_to_date(last_posted['date']),
-      last_update_posted_date_type: last_posted['type'],
-      start_month_year: start_date['date'],
-      start_date_type: start_date['type'],
-      start_date: convert_to_date(start_date['date']),
-      verification_month_year: status['statusVerifiedDate'],
-      verification_date: convert_to_date(status['statusVerifiedDate']),
-      completion_month_year: completion_date['date'],
-      completion_date_type: completion_date['type'],
-      completion_date: convert_to_date(completion_date['date']),
-      primary_completion_month_year: primary_completion_date['date'],
-      primary_completion_date_type: primary_completion_date['type'],
-      primary_completion_date: convert_to_date(primary_completion_date['date']),
-      baseline_population: baseline['populationDescription'],
-      brief_title: ident['briefTitle'],
-      official_title: ident['officialTitle'],
-      acronym: ident['acronym'],
-      overall_status: status['overallStatus'],
-      last_known_status: status['lastKnownStatus'],
-      why_stopped: status['whyStopped'],
-      delayed_posting: status['delayedPosting'],
-      phase: phase_list,
-      enrollment: enrollment['count'],
-      enrollment_type: enrollment['type'],
-      source: ident.dig('organization', 'fullName'),
-      source_class: ident.dig('organization', 'class'),
-      limitations_and_caveats: key_check(more_info&.dig('limitationsAndCaveats'))&.dig('description'),
-      number_of_arms: arms_count,
-      number_of_groups: groups_count,
-      target_duration: design['targetDuration'],
-      study_type: study_type,
-      has_expanded_access: get_boolean(expanded_access),
-      expanded_access_nctid: status.dig('expandedAccessInfo', 'nctId'),
-      expanded_access_status_for_nctid: status.dig('expandedAccessInfo', 'statusForNctId'),
-      expanded_access_type_individual: get_boolean(expanded['individual']),
-      expanded_access_type_intermediate: get_boolean(expanded['intermediate']),
-      expanded_access_type_treatment: get_boolean(expanded['treatment']),
-      has_dmc: get_boolean(oversight['oversightHasDmc']),
-      is_fda_regulated_drug: get_boolean(oversight['isFdaRegulatedDrug']),
-      is_fda_regulated_device: get_boolean(oversight['isFdaRegulatedDevice']),
-      is_unapproved_device: get_boolean(oversight['isUnapprovedDevice']),
-      is_ppsd: get_boolean(oversight['isPpsd']),
-      is_us_export: get_boolean(oversight['isUsExport']),
-      fdaaa801_violation: get_boolean(oversight['fdaaa801Violation']),
-      biospec_retention: biospec['retention'],
-      biospec_description: biospec['description'],
-      plan_to_share_ipd: ipd_sharing['ipdSharing'],
-      plan_to_share_ipd_description: ipd_sharing['description'],
-      ipd_time_frame: ipd_sharing['timeFrame'],
-      ipd_access_criteria: ipd_sharing['accessCriteria'],
-      ipd_url: ipd_sharing['url'],
-      baseline_type_units_analyzed: baseline['typeUnitsAnalyzed']
-    }
-  end
-
   def get_groups(opts)
     self.groups=ResultGroup.create_all_from(opts)
   end

app/models/study_downloader.rb

@@ -62,4 +62,18 @@ def self.find_studies_to_update
       current = Hash[StudyJsonRecord.where(version: '2').pluck(:nct_id, :updated_at)]
       changed = studies.select{|k| current[k[:nct_id]].nil? || current[k[:nct_id]] < DateTime.parse(k[:updated]) }.map{|k| k[:nct_id]}
     end
+
+    def self.find_studies_to_remove
+      studies = ClinicalTrialsApiV2.all
+
+      with_search_path('ctgov_v2, support, public') do
+        json = StudyJsonRecord.where(version: '2').pluck(:nct_id)
+        removing = json - studies.map{|k| k[:nct_id]}
+        puts "removing #{removing.length} studies from study_json_records".red
+
+        imported = Study.pluck(:nct_id)
+        removing = imported - studies.map{|k| k[:nct_id]}
+        puts "removing #{removing.length} studies from imported".red
+      end
+    end
   end

app/models/study_json_record/worker.rb

@@ -156,6 +156,7 @@ def build
   end
 
   def append_parent(hash, value)
+    return unless hash
     if hash['$parent']
       append_parent(hash['$parent'], value)
     else
@@ -178,6 +179,7 @@ def add_missing_keys(item, keys)
   end
 
   def flatten(path, data, parent=nil)
+    # puts "🛑🛑🛑🛑 FLATTEN #{path} - #{data} - #{parent} 🛑🛑🛑🛑"
     return [] unless data
     child_key = path.first
     result = []
@@ -201,6 +203,9 @@ def flatten(path, data, parent=nil)
         item
       else
         res = flatten(path[1..-1], children, item)
+        if res.first.nil?
+          byebug
+        end
         append_parent(res.first, parent) if parent
         res
       end

app/models/study_relationship.rb

@@ -379,4 +379,16 @@ def self.update_data_definitions
       end
     end
   end
+
+  def self.swhere(opts={}, version='1')
+    table_name = self.table_name.gsub(/ctgov_v2.|ctgov./, '')
+    case version
+    when '1'
+      self.table_name = "ctgov.#{table_name}"
+      where(opts)
+    when '2'
+      self.table_name = "ctgov_v2.#{table_name}"
+      where(opts)
+    end
+  end
 end