scrapeGithub.Rmd

---
title: "scrapeGithub"
output: html_document
author: Simon Goring & Nick McKay
---
### Get a list of libraries from ROpenSci
In this case a user has written a script to link a data resource, and the R package written for that resource, to resources in GitHub.  The intention here is to survey ways in which the data resource is being used in analytic workflows.

```{r ropensci_links, echo=TRUE, warning=FALSE, message=FALSE, results='hide'}
library(tidyverse)

ropensci_registry <- jsonlite::fromJSON("https://raw.githubusercontent.com/ropensci/roregistry/master/registry.json")

packages <- ropensci_registry$packages

dd <- filter(packages,ropensci_category == "data-access")
all.ddcat <- unique(unlist(dd$category))

to.use <- c("biology","archeology","maps","ecology","chemistry","conservation",
            "geospatial","environmental","waether","climate","biodiversity",
            "animals","plants","water","hydrology","noaa","NOAA","ecosystem",
            "remote-sensing")

dd.touse <- filter(dd,category %in% to.use)

packList <- dd.touse$name

```

Add in other repositories...

```{r}
additionalPacks <- read_csv("R_packages_toScrape.csv")

packList <- unique(c(packList,additionalPacks$packages))
#initialize github stuf
gh_token <- scan('gh.token', what = 'character')

script_home <- 'https://github.com/throughput-ec/crossResourceWorklowScraper/blob/master/scrapeGithub.Rmd'
allUrls <- data.frame()


for (i in 1:length(packList)) { 
  
  # I had to serialize this to avoid getting caught by GitHub's abuse detection.
  
  x <- dd.touse$name[i]
  
  
  if (!length(test_pks) >= i) {
    Sys.sleep(15) # This is probably longer than it needs to be. . . 

    repos <- gh::gh(paste0('/search/code?q=library(',x,
                           ')+in:file+language:R+extension:R+extension:Rmd'), 
                     .token = gh_token)
       
    annotation_text <- paste0("The GitHub repository uses the package ",
                              x, " in a `library()` or `require()` call.")
    
    repo_list <- unique(sapply(repos$items, function(x)x$repository$html_url))
    
    target_list <- lapply(repo_list,
                          function(y) { 
                            if(length(repo_list) > 0) {
                              return(data.frame(value = y, type = 'URL')) 
                              } else { return(NULL) }
                            }
                          )
    if(length(target_list)==0){
      next
    }
    theseUrls <- data.frame(urls=sapply(target_list,"[[","value"),package = dd.touse$name[i])
    allUrls <- bind_rows(allUrls,theseUrls)
  #   if (length(target_list) > 0) {
  #   
  #     test_pks[[i]] <- list(
  #       target = target_list,
  #       body = list(object(value = paste0('http://github.com/ropensci/',x), 
  #                                    type = 'URL'),
  #                   object( type = "annotationText",
  #                          value = annotation_text),
  #                   object(type = 'URL',
  #                         value = script_home)),
  #       generator = creator(identifier = '0000-0002-2700-4605',
  #                                   PropertyID = 'orcid',
  #                                     lastName = 'Goring',
  #                                    firstName = 'Simon'),
  #               body_rel = object(type = 'URL',
  #                                value = 'https://ropensci.org/'),
  #               source = object(type = 'URL',
  #                               value = 'http://github.com'))
  #   } else {
  #     test_pks[[i]] <- list()
  #   }
  # }
  # 
  # if (length(test_pks[[i]]) > 0) {
  #   
  #   link_record(con,
  #               target = test_pks[[i]]$target,
  #                 body = test_pks[[i]]$body,
  #            generator = test_pks[[i]]$generator,
  #             body_rel = test_pks[[i]]$body_rel,
  #               source = test_pks[[i]]$source)
   }
   
  saveRDS(object = test_pks, file = 'data/all_github.rds')
  cat(i, '\n')
  
}

```

Now look for package calls that cooccur

```{r}
#mults <- data.frame(repo="NA",packages="NA")
mults <- list()
t=1
uniqueUrls <- unique(allUrls$urls)
for(i in 1:length(uniqueUrls)){
  thisUrl <- uniqueUrls[i]
  wu <- which(thisUrl == allUrls$urls)
  if(length(wu)>1){
    mults[[t]] <- list()
    mults[[t]]$repo <- thisUrl
    mults[[t]]$packages <- allUrls$package[wu]
    t=t+1
  }
  
}
#create a data.frame
repos <- sapply(mults,"[[","repo")
packages <- sapply(mults,function(x){paste(x$packages,collapse=", ")})
out <- data.frame(repos = repos,packages = packages)
write_csv(out,path = here::here("scrapeGitResults.csv"))
```