From 6c0ce729eba718254b8ae9796b5be09b12ba8526 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Wed, 27 Jan 2021 16:19:57 +0100 Subject: [PATCH 01/30] wip adding 3d provider: google people contacts --- controllers/configuration-controller.js | 7 ++ lib/utils/mapping-utils.js | 1 + .../contact-transfer-using-schema-org.ttl | 65 +++++++++++++++++++ rml/mappings-metadata.json | 8 +++ routes/index.js | 9 ++- 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 rml/google/contact-transfer-using-schema-org.ttl diff --git a/controllers/configuration-controller.js b/controllers/configuration-controller.js index 489cecc..10e4567 100644 --- a/controllers/configuration-controller.js +++ b/controllers/configuration-controller.js @@ -13,6 +13,13 @@ const configuration = { }, 'connect' : '/connect/flickr' }, + 'google' : { + 'solid' : { + 'filename': 'google.ttl' + }, + 'connect' : '/connect/google' + }, + 'solid' : { 'storageDirectory' : 'private' } diff --git a/lib/utils/mapping-utils.js b/lib/utils/mapping-utils.js index e4f99d1..309304d 100644 --- a/lib/utils/mapping-utils.js +++ b/lib/utils/mapping-utils.js @@ -90,6 +90,7 @@ exports.createTemplateKeyValues = function(providerCredentials, providerConfig) break; case 'imgur': + case 'google': // The access_token is a Bearer const bearerToken = templateKeyValues['access_token'] // The bearerToken should be filled in in the 'ex:authorizationHeader' as follows: diff --git a/rml/google/contact-transfer-using-schema-org.ttl b/rml/google/contact-transfer-using-schema-org.ttl new file mode 100644 index 0000000..2b23d6d --- /dev/null +++ b/rml/google/contact-transfer-using-schema-org.ttl @@ -0,0 +1,65 @@ +@prefix rr: . +@prefix rdf: . +@prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix d2rq: . +@prefix void: . +@prefix dc: . +@prefix foaf: . +@prefix rml: . +@prefix ql: . +@prefix : . +@prefix ex: . +@prefix geo: . +@prefix ssn-system: . +@prefix ssn: . +@prefix sosa: . +@prefix idlab-fn: . +@prefix grel: . +@prefix schema: . + + +:google_source ex:AuthorizationHeader "{{authorizationHeader}}" ; + schema:name "Google API" ; + schema:url ; + schema:WebAPI . +:rules_000 a void:Dataset; + void:exampleResource :map_google-contacts-parent_000. + :map_google-contacts-parent_000 rml:logicalSource :source_000. +:source_000 a rml:LogicalSource; + rml:source :google_source; + rml:iterator "$.connections[*]"; + rml:referenceFormulation ql:JSONPath. +:map_google-contacts-parent_000 a rr:TriplesMap; + rdfs:label "google-contacts-parent". +:s_000 a rr:SubjectMap. + :map_google-contacts-parent_000 rr:subjectMap :s_000. +:s_000 rr:template "http://example.com/contact/{resourceName}". +:pom_000 a rr:PredicateObjectMap. + :map_google-contacts-parent_000 rr:predicateObjectMap :pom_000. +:pm_000 a rr:PredicateMap. +:pom_000 rr:predicateMap :pm_000. +:pm_000 rr:constant rdf:type. +:pom_000 rr:objectMap :om_000. +:om_000 a rr:ObjectMap; + rr:constant "http://schema.org/Person"; + rr:termType rr:IRI. +:pom_001 a rr:PredicateObjectMap. +:map_google-contacts-parent_000 rr:predicateObjectMap :pom_001. +:pm_001 a rr:PredicateMap. +:pom_001 rr:predicateMap :pm_001. +:pm_001 rr:constant schema:givenName. +:pom_001 rr:objectMap :om_001. +:om_001 a rr:ObjectMap; + rml:reference "names[0].givenName"; + rr:termType rr:Literal. +:pom_002 a rr:PredicateObjectMap. +:map_google-contacts-parent_000 rr:predicateObjectMap :pom_002. +:pm_002 a rr:PredicateMap. +:pom_002 rr:predicateMap :pm_002. +:pm_002 rr:constant schema:familyName. +:pom_002 rr:objectMap :om_002. +:om_002 a rr:ObjectMap; + rml:reference "names[0].familyName"; + rr:termType rr:Literal. diff --git a/rml/mappings-metadata.json b/rml/mappings-metadata.json index 844ecef..ff7abc8 100644 --- a/rml/mappings-metadata.json +++ b/rml/mappings-metadata.json @@ -32,5 +32,13 @@ "label" : "Transfer images from Imgur using DCAT vocabulary", "description" : "Transfer images from Imgur using DCAT vocabulary" } + ], + "google" : [ + { + "filename" : "contact-transfer-using-schema-org.ttl", + "label" : "Transfer Google contacts using Schema.org", + "description" : "" + } + ] } \ No newline at end of file diff --git a/routes/index.js b/routes/index.js index c79e3fa..e0f15c7 100644 --- a/routes/index.js +++ b/routes/index.js @@ -53,6 +53,13 @@ function createRouter(grant, environmentConfig, logConfig = null) { res.redirect('/') }) + // callback for capturing the Google tokens + router.get('/google/callback', (req, res) => { + console.log(req.route.path) + updateTokens(req) + res.redirect('/') + }) + /** * Return RML Rules * If a filename is specified, read and send that file. @@ -230,4 +237,4 @@ function handleStatusCode(req, res, code, optionalMessage = "") { res.sendStatus(code) } -module.exports = createRouter \ No newline at end of file +module.exports = createRouter From 496a24c8fe69437f7c1ebdd60d179f964c4d761e Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 10:55:45 +0100 Subject: [PATCH 02/30] wip: readme --- README.md | 44 ++++++++++++++++++- ...horization-create-credentials-dropdown.png | 3 ++ docs/img/readme-adding-google-enable-api.png | 3 ++ ...g-google-navigate-to-APIs-and-Services.png | 3 ++ ...adding-google-step001-selected-project.png | 3 ++ 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 docs/img/readme-adding-google-authorization-create-credentials-dropdown.png create mode 100644 docs/img/readme-adding-google-enable-api.png create mode 100644 docs/img/readme-adding-google-navigate-to-APIs-and-Services.png create mode 100644 docs/img/readme-adding-google-step001-selected-project.png diff --git a/README.md b/README.md index a84cdb7..9adf144 100644 --- a/README.md +++ b/README.md @@ -32,15 +32,55 @@ Each `app-config` refers to a json file (template at `config.template.json`) tha Run the web-app using: ```bash -npm run start +yarn run start ``` Run the web-app in development mode using: ```bash -npm run start:dev +yarn run start:dev ``` +### Adding data providers + +#### [Google People API](https://developers.google.com/people?hl=en) + +Prerequisites: +- Google Developer Account + +- Go to the [Google Developer Console](https://console.developers.google.com/) + and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` +- For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. + +![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) + +- Navigate to *APIs & Services* + +![Go to APIs and Services](docs/img/readme-adding-google-navigate-to-APIs-and-Services.png) + +- Search and enable the API you wish to integrate. In this case, the [Google People API](https://console.cloud.google.com/apis/library/people.googleapis.com) + +- In order to make requests to the API, you need to set up authorization. + - Since we need to access private data, OAuth 2.0 credentials must be generated. + - Go to the [Credentials page](https://console.developers.google.com/apis/credentials) and click on *Create Credentials* and select *OAuth client ID* +
![Create credentials and select OAuth client ID ](docs/img/readme-adding-google-authorization-create-credentials-dropdown.png) + + - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server + + + + + - OAuth 2.0 Client + - added nearly all scopes w.r.t. reading from people API + - [ ] **can this be easily extended to other Google apis?** + - add testusers (in test-mode, only testusers can use the app (you can register up to 100 test users)) + - create credentials + - API Key: `AIzaSyDNwWIFFpiOmAFiygS7eM-xcltFEWYIthk` + - Oauth Client + - name: `oauthclient-dapsi-dev` + - client id: `704703996998-mm0psiamc628soisd9hkovrjo5tlbd2q.apps.googleusercontent.com` + - client secret: `tatxC9Eli-VWJ5-i12IpSiST` + ## Use cases ### Use case: Transfer data diff --git a/docs/img/readme-adding-google-authorization-create-credentials-dropdown.png b/docs/img/readme-adding-google-authorization-create-credentials-dropdown.png new file mode 100644 index 0000000..9ab2508 --- /dev/null +++ b/docs/img/readme-adding-google-authorization-create-credentials-dropdown.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a30f02c6cc81980a54747dbc854ba05f9b5a558c1717c5649aaadb46266e81 +size 33386 diff --git a/docs/img/readme-adding-google-enable-api.png b/docs/img/readme-adding-google-enable-api.png new file mode 100644 index 0000000..db9d911 --- /dev/null +++ b/docs/img/readme-adding-google-enable-api.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dff25739baae5672956ab616a4ec5ffeeb6cb4feac1db519725b6cb830c7934 +size 160939 diff --git a/docs/img/readme-adding-google-navigate-to-APIs-and-Services.png b/docs/img/readme-adding-google-navigate-to-APIs-and-Services.png new file mode 100644 index 0000000..df68509 --- /dev/null +++ b/docs/img/readme-adding-google-navigate-to-APIs-and-Services.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a3c1fc0900066f8b86b61fb9430b80566a0fe336e142ba86f512b00fb53c1ba +size 201416 diff --git a/docs/img/readme-adding-google-step001-selected-project.png b/docs/img/readme-adding-google-step001-selected-project.png new file mode 100644 index 0000000..e8fe0c5 --- /dev/null +++ b/docs/img/readme-adding-google-step001-selected-project.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89daec99cfab4229c75e779dc611ff75d62029e930776bc9def875dc343f0573 +size 6321 From f5da1b38568c1de34d95ad0886e8e8e409af6a44 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 12:59:05 +0100 Subject: [PATCH 03/30] update readme --- README.md | 45 +++++++++++++------ ...readme-adding-google-adding-test-users.png | 3 ++ 2 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 docs/img/readme-adding-google-adding-test-users.png diff --git a/README.md b/README.md index 9adf144..a49a554 100644 --- a/README.md +++ b/README.md @@ -62,24 +62,43 @@ Prerequisites: - In order to make requests to the API, you need to set up authorization. - Since we need to access private data, OAuth 2.0 credentials must be generated. - - Go to the [Credentials page](https://console.developers.google.com/apis/credentials) and click on *Create Credentials* and select *OAuth client ID* + - Go to the [Credentials page](https://console.developers.google.com/apis/credentials), click on *Create Credentials* and select *OAuth client ID*
![Create credentials and select OAuth client ID ](docs/img/readme-adding-google-authorization-create-credentials-dropdown.png) - - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server + - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server. + - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` + - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. +- Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. +![Navigate to OAuth consent screen](docs/img/readme-adding-google-adding-test-users.png) + +- Finally, add the *Client ID*, *Client secret* to the `config.json`. This may look like + +```json +{ + "defaults": { + "origin": "https://localhost:3000", + "transport": "session" + }, + "imgur": { + // ... + }, + "flickr": { + // ... + }, + "google": { + "key": "", + "secret": "", + "callback" : "https://localhost:3000/google/callback", + "scope" : [ "https://www.googleapis.com/auth/contacts.readonly"] + } +} +``` - - - OAuth 2.0 Client - - added nearly all scopes w.r.t. reading from people API - - [ ] **can this be easily extended to other Google apis?** - - add testusers (in test-mode, only testusers can use the app (you can register up to 100 test users)) - - create credentials - - API Key: `AIzaSyDNwWIFFpiOmAFiygS7eM-xcltFEWYIthk` - - Oauth Client - - name: `oauthclient-dapsi-dev` - - client id: `704703996998-mm0psiamc628soisd9hkovrjo5tlbd2q.apps.googleusercontent.com` - - client secret: `tatxC9Eli-VWJ5-i12IpSiST` +For more information on setting up a Google app that uses the Google People API, check out +- https://developers.google.com/people/v1/how-tos/authorizing +- https://developers.google.com/identity/protocols/oauth2 ## Use cases diff --git a/docs/img/readme-adding-google-adding-test-users.png b/docs/img/readme-adding-google-adding-test-users.png new file mode 100644 index 0000000..7ccedd4 --- /dev/null +++ b/docs/img/readme-adding-google-adding-test-users.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e841ad7c8492bfc49cbad4fc4ada6fa2abdfc5fbefb1a665d30125c3c4756ea1 +size 188920 From c3515237150cd7f1b775ee2a9bb608e89c84dc13 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 13:01:25 +0100 Subject: [PATCH 04/30] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a49a554..b4a9b40 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,10 @@ yarn run start:dev Prerequisites: - Google Developer Account +Steps: - Go to the [Google Developer Console](https://console.developers.google.com/) and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` - For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. - ![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) - Navigate to *APIs & Services* From 139704f2ade96f7b11d3a26b2c8e0b8e2204101c Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 13:02:36 +0100 Subject: [PATCH 05/30] update --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index b4a9b40..2c4d1e0 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Steps: - Go to the [Google Developer Console](https://console.developers.google.com/) and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` - For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. -![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) +
![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) - Navigate to *APIs & Services* @@ -81,10 +81,8 @@ Steps: "transport": "session" }, "imgur": { - // ... }, "flickr": { - // ... }, "google": { "key": "", From c4aab19ea0fdce90f5ad4ff618782469f77f8761 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 13:03:12 +0100 Subject: [PATCH 06/30] update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2c4d1e0..c6340e1 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Steps: - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. + - Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. ![Navigate to OAuth consent screen](docs/img/readme-adding-google-adding-test-users.png) From 5f63e9a00828f9a90fc0841149ca243f4d0af2de Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 13:11:27 +0100 Subject: [PATCH 07/30] Update ChangeLog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8c5838..7c1d846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added + +- Added Data Provider: Google People API (see [issue #44](https://gitlab.ilabt.imec.be/prov4itdata-dapsi/web-app/-/issues/44)) + ## [0.1.1] - 2020-12-21 ### Added From 8db0e53fa119c23cb2fa874e0a37ddab06f1dd88 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 09:18:41 +0100 Subject: [PATCH 08/30] added mapping for transferring google contacts --- .../contact-transfer-using-schema-org.ttl | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 public/rml/google/contact-transfer-using-schema-org.ttl diff --git a/public/rml/google/contact-transfer-using-schema-org.ttl b/public/rml/google/contact-transfer-using-schema-org.ttl new file mode 100644 index 0000000..2b23d6d --- /dev/null +++ b/public/rml/google/contact-transfer-using-schema-org.ttl @@ -0,0 +1,65 @@ +@prefix rr: . +@prefix rdf: . +@prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix d2rq: . +@prefix void: . +@prefix dc: . +@prefix foaf: . +@prefix rml: . +@prefix ql: . +@prefix : . +@prefix ex: . +@prefix geo: . +@prefix ssn-system: . +@prefix ssn: . +@prefix sosa: . +@prefix idlab-fn: . +@prefix grel: . +@prefix schema: . + + +:google_source ex:AuthorizationHeader "{{authorizationHeader}}" ; + schema:name "Google API" ; + schema:url ; + schema:WebAPI . +:rules_000 a void:Dataset; + void:exampleResource :map_google-contacts-parent_000. + :map_google-contacts-parent_000 rml:logicalSource :source_000. +:source_000 a rml:LogicalSource; + rml:source :google_source; + rml:iterator "$.connections[*]"; + rml:referenceFormulation ql:JSONPath. +:map_google-contacts-parent_000 a rr:TriplesMap; + rdfs:label "google-contacts-parent". +:s_000 a rr:SubjectMap. + :map_google-contacts-parent_000 rr:subjectMap :s_000. +:s_000 rr:template "http://example.com/contact/{resourceName}". +:pom_000 a rr:PredicateObjectMap. + :map_google-contacts-parent_000 rr:predicateObjectMap :pom_000. +:pm_000 a rr:PredicateMap. +:pom_000 rr:predicateMap :pm_000. +:pm_000 rr:constant rdf:type. +:pom_000 rr:objectMap :om_000. +:om_000 a rr:ObjectMap; + rr:constant "http://schema.org/Person"; + rr:termType rr:IRI. +:pom_001 a rr:PredicateObjectMap. +:map_google-contacts-parent_000 rr:predicateObjectMap :pom_001. +:pm_001 a rr:PredicateMap. +:pom_001 rr:predicateMap :pm_001. +:pm_001 rr:constant schema:givenName. +:pom_001 rr:objectMap :om_001. +:om_001 a rr:ObjectMap; + rml:reference "names[0].givenName"; + rr:termType rr:Literal. +:pom_002 a rr:PredicateObjectMap. +:map_google-contacts-parent_000 rr:predicateObjectMap :pom_002. +:pm_002 a rr:PredicateMap. +:pom_002 rr:predicateMap :pm_002. +:pm_002 rr:constant schema:familyName. +:pom_002 rr:objectMap :om_002. +:om_002 a rr:ObjectMap; + rml:reference "names[0].familyName"; + rr:termType rr:Literal. From f0fa8b4cc0335bf42ce62c2a0a8d04983251ad34 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 09:22:23 +0100 Subject: [PATCH 09/30] ignore /build --- .gitignore | 2 ++ build/ui/.gitignore | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 build/ui/.gitignore diff --git a/.gitignore b/.gitignore index 3fa3d23..9d506b7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ node_modules/ .idea/ + +build/ diff --git a/build/ui/.gitignore b/build/ui/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/build/ui/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore From 748e1112793c4e796dfd3eabb718d239774f3cf1 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 09:56:55 +0100 Subject: [PATCH 10/30] update readme: added hint on creating google rml mapping --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 27f77fd..ccccf33 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ yarn run start:dev #### [Google People API](https://developers.google.com/people?hl=en) +##### Google Cloud Platform configuration + Prerequisites: - Google Developer Account @@ -112,6 +114,26 @@ For more information on setting up a Google app that uses the Google People API, - https://developers.google.com/people/v1/how-tos/authorizing - https://developers.google.com/identity/protocols/oauth2 +#### Creating an RML Mapping that consumes and transforms the Google People API + +When creating an RML Mapping, you always have to define exactly one [`rml:logicalSource`](https://rml.io/specs/rml/#logical-source) +for every triples map you define. This way, the RML Processor knows where and how to access the data to be mapped. +Using the `rml:logicalSource`'s `rml:source` property we can specify the url of the Web API. + +In order to access protected resources, the RML Processor needs to include the required credentials when consuming the API. +The Google People API uses OAuth 2.0, and an authorization header should be added to the requests. Since these credentials +are managed by our web-app, the value for the authorization header is a template-variable, hence, the web-app will +recognize this and fill in the value. + +The following is an excerpt of the `rml:source` defined in [`rml/google/contact-transfer-using-schema-org.ttl`](/public/rml/google/contact-transfer-using-schema-org.ttl): +```turtle +:google_source + ex:AuthorizationHeader "{{authorizationHeader}}" ; + schema:name "Google API" ; + schema:url ; + schema:WebAPI . +``` + ## Use cases ### Use case: Transfer data From ee6f18d06c2cccdaacb0623406e3ca44ffe8c437 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 10:48:40 +0100 Subject: [PATCH 11/30] update report: google people api --- docs/REPORT.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/REPORT.md b/docs/REPORT.md index 1557431..c1c0e34 100644 --- a/docs/REPORT.md +++ b/docs/REPORT.md @@ -416,6 +416,24 @@ An Imgur image can be mapped to a `schema:ImageObject`, along with the following > We plan to include a more technical section detailing how different Data Providers can be handled uniformly using the RML.io toolchain +### [Google People API](https://developers.google.com/people?hl=en) + +Using the Google People API we can transfer our Google contacts. + +> Please note that this use case can be extended easily to [other Google Products](https://developers.google.com/products?hl=en). + +#### Using [Schema.org] + +A Google contact can be mapped to a `schema:Person`, along with the following properties: + +| Google contact resource | `schema:Person` | +| --------------------- | ----------------------- | +| `givenName` | `schema:givenName` | +| `familyName` | `schema:familyName` | +| `displayName` | `schema:alternativeName` | + + + ## Features Within this project, we envisioned following features. From 2429e888963aac2b079f2474dac5818fa259f3cd Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 10:31:58 +0100 Subject: [PATCH 12/30] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e8aefa..26767e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## Unreleased +## [0.2.0] - 2021-02-05 ### Added From 5e5788346d675912dbeed7ecba3785db65ddce21 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 10:56:28 +0100 Subject: [PATCH 13/30] bump to v0.2.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9c879cf..9a43c58 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@prov4itdata/web-app", - "version": "0.1.1", + "version": "0.2.0", "description": "PROV4ITDaTa Web app", "main": "app.js", "scripts": { From 98c0aa6a2c921fdea6a9e1819722ceff0bb72281 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 11:36:10 +0100 Subject: [PATCH 14/30] update report: adding data providers --- docs/REPORT.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/docs/REPORT.md b/docs/REPORT.md index c1c0e34..597385e 100644 --- a/docs/REPORT.md +++ b/docs/REPORT.md @@ -414,8 +414,6 @@ An Imgur image can be mapped to a `schema:ImageObject`, along with the following - A Imgur Album can contain image resources, and can be modelled by `dcat:Dataset`. - An Imgur Gallery can contain Imgur Albums, which contain image resources. Therefore, an Imgur Gallery can be modelled by a `dcat:Catalog` which can be linked to **zero or more** `dcat:Dataset`s (Albums). -> We plan to include a more technical section detailing how different Data Providers can be handled uniformly using the RML.io toolchain - ### [Google People API](https://developers.google.com/people?hl=en) Using the Google People API we can transfer our Google contacts. @@ -432,6 +430,90 @@ A Google contact can be mapped to a `schema:Person`, along with the following pr | `familyName` | `schema:familyName` | | `displayName` | `schema:alternativeName` | +## Adding data providers + +This section will walk you through the different steps of adding a data provider to the PROV4ITDaTa Web-App, where the Google People API serves as an example. + +#### [Google People API](https://developers.google.com/people?hl=en) + +First, we discuss the configuration steps on the Google Cloud Platform. +Secondly, we elaborate on how to define the Google People API as a logical source in an RML Mapping. + +##### Google Cloud Platform configuration + +Prerequisites: +- Google Developer Account + +Steps: +- Go to the [Google Developer Console](https://console.developers.google.com/) + and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` +- For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. +
![Make sure that the correct project is selected](img/readme-adding-google-step001-selected-project.png) + +- Navigate to *APIs & Services* + +![Go to APIs and Services](img/readme-adding-google-navigate-to-APIs-and-Services.png) + +- Search and enable the API you wish to integrate. In this case, the [Google People API](https://console.cloud.google.com/apis/library/people.googleapis.com) + +- In order to make requests to the API, you need to set up authorization. + - Since we need to access private data, OAuth 2.0 credentials must be generated. + - Go to the [Credentials page](https://console.developers.google.com/apis/credentials), click on *Create Credentials* and select *OAuth client ID* +
![Create credentials and select OAuth client ID ](img/readme-adding-google-authorization-create-credentials-dropdown.png) + + - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server. + - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` + - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. + + +- Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. + ![Navigate to OAuth consent screen](img/readme-adding-google-adding-test-users.png) + +- Finally, add the *Client ID*, *Client secret* to the `config.json`. This may look like + +```json +{ + "defaults": { + "origin": "https://localhost:3000", + "transport": "session" + }, + "imgur": { + }, + "flickr": { + }, + "google": { + "key": "", + "secret": "", + "callback" : "https://localhost:3000/google/callback", + "scope" : [ "https://www.googleapis.com/auth/contacts.readonly"] + } +} + +``` + +For more information on setting up a Google app that uses the Google People API, check out +- https://developers.google.com/people/v1/how-tos/authorizing +- https://developers.google.com/identity/protocols/oauth2 + +#### Creating an RML Mapping that consumes and transforms the Google People API + +When creating an RML Mapping, you always have to define exactly one [`rml:logicalSource`](https://rml.io/specs/rml/#logical-source) +for every triples map you define. This way, the RML Processor knows where and how to access the data to be mapped. +Using the `rml:logicalSource`'s `rml:source` property we can specify the url of the Web API. + +In order to access protected resources, the RML Processor needs to include the required credentials when consuming the API. +The Google People API uses OAuth 2.0, and an authorization header should be added to the requests. Since these credentials +are managed by our web-app, the value for the authorization header is a template-variable, hence, the web-app will +recognize this and fill in the value. + +The following is an excerpt of the `rml:source` defined in [`rml/google/contact-transfer-using-schema-org.ttl`](/public/rml/google/contact-transfer-using-schema-org.ttl): +```turtle +:google_source + ex:AuthorizationHeader "{{authorizationHeader}}" ; + schema:name "Google API" ; + schema:url ; + schema:WebAPI . +``` ## Features From 75e1674b5cdc4e26dfb5f37d6e5444645839cd55 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 11:36:43 +0100 Subject: [PATCH 15/30] removed section about adding data providers (see docs/report) --- README.md | 78 ------------------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/README.md b/README.md index ccccf33..15a69ea 100644 --- a/README.md +++ b/README.md @@ -54,85 +54,7 @@ Run the web-app in development mode using: yarn run start:dev ``` -### Adding data providers - -#### [Google People API](https://developers.google.com/people?hl=en) - -##### Google Cloud Platform configuration - -Prerequisites: -- Google Developer Account - -Steps: -- Go to the [Google Developer Console](https://console.developers.google.com/) - and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` -- For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. -
![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) - -- Navigate to *APIs & Services* - -![Go to APIs and Services](docs/img/readme-adding-google-navigate-to-APIs-and-Services.png) - -- Search and enable the API you wish to integrate. In this case, the [Google People API](https://console.cloud.google.com/apis/library/people.googleapis.com) - -- In order to make requests to the API, you need to set up authorization. - - Since we need to access private data, OAuth 2.0 credentials must be generated. - - Go to the [Credentials page](https://console.developers.google.com/apis/credentials), click on *Create Credentials* and select *OAuth client ID* -
![Create credentials and select OAuth client ID ](docs/img/readme-adding-google-authorization-create-credentials-dropdown.png) - - - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server. - - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` - - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. - - -- Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. -![Navigate to OAuth consent screen](docs/img/readme-adding-google-adding-test-users.png) - -- Finally, add the *Client ID*, *Client secret* to the `config.json`. This may look like - -```json -{ - "defaults": { - "origin": "https://localhost:3000", - "transport": "session" - }, - "imgur": { - }, - "flickr": { - }, - "google": { - "key": "", - "secret": "", - "callback" : "https://localhost:3000/google/callback", - "scope" : [ "https://www.googleapis.com/auth/contacts.readonly"] - } -} -``` - -For more information on setting up a Google app that uses the Google People API, check out -- https://developers.google.com/people/v1/how-tos/authorizing -- https://developers.google.com/identity/protocols/oauth2 - -#### Creating an RML Mapping that consumes and transforms the Google People API - -When creating an RML Mapping, you always have to define exactly one [`rml:logicalSource`](https://rml.io/specs/rml/#logical-source) -for every triples map you define. This way, the RML Processor knows where and how to access the data to be mapped. -Using the `rml:logicalSource`'s `rml:source` property we can specify the url of the Web API. - -In order to access protected resources, the RML Processor needs to include the required credentials when consuming the API. -The Google People API uses OAuth 2.0, and an authorization header should be added to the requests. Since these credentials -are managed by our web-app, the value for the authorization header is a template-variable, hence, the web-app will -recognize this and fill in the value. - -The following is an excerpt of the `rml:source` defined in [`rml/google/contact-transfer-using-schema-org.ttl`](/public/rml/google/contact-transfer-using-schema-org.ttl): -```turtle -:google_source - ex:AuthorizationHeader "{{authorizationHeader}}" ; - schema:name "Google API" ; - schema:url ; - schema:WebAPI . -``` ## Use cases From 27ac04b41e35732edf3cff2164a686ec8615f3ff Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 11:40:05 +0100 Subject: [PATCH 16/30] update ToC --- docs/REPORT.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/REPORT.md b/docs/REPORT.md index 597385e..392ce71 100644 --- a/docs/REPORT.md +++ b/docs/REPORT.md @@ -6,23 +6,64 @@ - [PROV4ITDaTa - Technical Report](#prov4itdata---technical-report) - [Alignment to the milestones](#alignment-to-the-milestones) + - [M1: Design configuration-based DTP exporter, applying it to a use case (deadline 10/2020)](#m1-design-configuration-based-dtp-exporter-applying-it-to-a-use-case-deadline-102020) + - [M2: Improve data provenance of the DTP exporter (deadline 12/2020)](#m2-improve-data-provenance-of-the-dtp-exporter-deadline-122020) + - [M3: Design query-based DTP importer for Solid, applying it to a use case (deadline 01/2020)](#m3-design-query-based-dtp-importer-for-solid-applying-it-to-a-use-case-deadline-012020) + - [M4: Implement configuration-based DTP exporter (deadline 04/2020)](#m4-implement-configuration-based-dtp-exporter-deadline-042020) + - [M5: Implement provenance-based DTP exporter (deadline 05/2020)](#m5-implement-provenance-based-dtp-exporter-deadline-052020) + - [M6: Implement query-based DTP importer for Solid (deadline 04/2020)](#m6-implement-query-based-dtp-importer-for-solid-deadline-042020) + - [M7: Integrate and finalize the exporter and importer, showcasing the use case (deadline 05/2020)](#m7-integrate-and-finalize-the-exporter-and-importer-showcasing-the-use-case-deadline-052020) - [Architecture](#architecture) - [Components](#components) + - [RML Mapping documents](#rml-mapping-documents) + - [RMLMapper](#rmlmapper) + - [Solid pods](#solid-pods) + - [Web App](#web-app) - [Relation to DTP](#relation-to-dtp) - [Requirements](#requirements) - [Demonstrator](#demonstrator) - [Use cases](#use-cases) + - [Best-practice vocabularies](#best-practice-vocabularies) + - [[Schema.org]](#schemaorg) + - [Image](#image) + - [Image Gallery](#image-gallery) + - [Collection](#collection) + - [[DCAT]](#dcat) + - [Image](#image-1) + - [Image gallery](#image-gallery-1) + - [Collection](#collection-1) - [[Flickr]](#flickr) + - [Using [Schema.org]](#using-schemaorg) + - [Using [DCAT]](#using-dcat) - [[Imgur]](#imgur) + - [Using [Schema.org]](#using-schemaorg-1) + - [Using [DCAT]](#using-dcat-1) + - [Google People API](#google-people-api) + - [Using [Schema.org]](#using-schemaorg-2) + - [Adding data providers](#adding-data-providers) + - [Google People API](#google-people-api-1) + - [Google Cloud Platform configuration](#google-cloud-platform-configuration) + - [Creating an RML Mapping that consumes and transforms the Google People API](#creating-an-rml-mapping-that-consumes-and-transforms-the-google-people-api) - [Features](#features) - [Use Open Standards and Open Source](#use-open-standards-and-open-source) - [Mapping files to transfer data](#mapping-files-to-transfer-data) - [Automatic Data Provenance Generation](#automatic-data-provenance-generation) - [Output RDF](#output-rdf) + - [Syntactic Interoperability](#syntactic-interoperability) + - [Semantic Interoperability](#semantic-interoperability) + - [Structural Interoperability](#structural-interoperability) + - [Data Compatibility](#data-compatibility) - [Data Portability](#data-portability) - [Security and Privacy](#security-and-privacy) + - [Data Minimization](#data-minimization) + - [User Control](#user-control) + - [Minimal Scopes for Auth Tokens](#minimal-scopes-for-auth-tokens) + - [Data retention](#data-retention) + - [Abuse](#abuse) - [Personalization](#personalization) - [Quality](#quality) + - [Data Quality](#data-quality) + - [Software Quality](#software-quality) - [GDPR](#gdpr) - [Conclusion](#conclusion) - [References](#references) From aa60fe1f8d67bd7f9b3c5578e27146a3eac28848 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 10:31:58 +0100 Subject: [PATCH 17/30] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e8aefa..26767e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## Unreleased +## [0.2.0] - 2021-02-05 ### Added From 9cd120032c02e42a164f550f2b66473c2bcf53ab Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 10:56:28 +0100 Subject: [PATCH 18/30] bump to v0.2.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9c879cf..9a43c58 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@prov4itdata/web-app", - "version": "0.1.1", + "version": "0.2.0", "description": "PROV4ITDaTa Web app", "main": "app.js", "scripts": { From 82592b32b563a4d0841080e6740eda6e9838ca0a Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 10:55:45 +0100 Subject: [PATCH 19/30] readme --- README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/README.md b/README.md index 15a69ea..d6e3633 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,44 @@ yarn run start:dev +#### [Google People API](https://developers.google.com/people?hl=en) + +Prerequisites: +- Google Developer Account + +- Go to the [Google Developer Console](https://console.developers.google.com/) + and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` +- For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. + +![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) + +- Navigate to *APIs & Services* + +![Go to APIs and Services](docs/img/readme-adding-google-navigate-to-APIs-and-Services.png) + +- Search and enable the API you wish to integrate. In this case, the [Google People API](https://console.cloud.google.com/apis/library/people.googleapis.com) + +- In order to make requests to the API, you need to set up authorization. + - Since we need to access private data, OAuth 2.0 credentials must be generated. + - Go to the [Credentials page](https://console.developers.google.com/apis/credentials) and click on *Create Credentials* and select *OAuth client ID* +
![Create credentials and select OAuth client ID ](docs/img/readme-adding-google-authorization-create-credentials-dropdown.png) + + - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server + + + + + - OAuth 2.0 Client + - added nearly all scopes w.r.t. reading from people API + - [ ] **can this be easily extended to other Google apis?** + - add testusers (in test-mode, only testusers can use the app (you can register up to 100 test users)) + - create credentials + - API Key: `AIzaSyDNwWIFFpiOmAFiygS7eM-xcltFEWYIthk` + - Oauth Client + - name: `oauthclient-dapsi-dev` + - client id: `704703996998-mm0psiamc628soisd9hkovrjo5tlbd2q.apps.googleusercontent.com` + - client secret: `tatxC9Eli-VWJ5-i12IpSiST` + ## Use cases ### Use case: Transfer data From b3ada11fd13e2ebe294395bc0dbe52c118dc53f9 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 12:59:05 +0100 Subject: [PATCH 20/30] update readme --- README.md | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d6e3633..c5baef9 100644 --- a/README.md +++ b/README.md @@ -75,24 +75,43 @@ Prerequisites: - In order to make requests to the API, you need to set up authorization. - Since we need to access private data, OAuth 2.0 credentials must be generated. - - Go to the [Credentials page](https://console.developers.google.com/apis/credentials) and click on *Create Credentials* and select *OAuth client ID* + - Go to the [Credentials page](https://console.developers.google.com/apis/credentials), click on *Create Credentials* and select *OAuth client ID*
![Create credentials and select OAuth client ID ](docs/img/readme-adding-google-authorization-create-credentials-dropdown.png) - - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server + - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server. + - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` + - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. +- Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. +![Navigate to OAuth consent screen](docs/img/readme-adding-google-adding-test-users.png) + +- Finally, add the *Client ID*, *Client secret* to the `config.json`. This may look like + +```json +{ + "defaults": { + "origin": "https://localhost:3000", + "transport": "session" + }, + "imgur": { + // ... + }, + "flickr": { + // ... + }, + "google": { + "key": "", + "secret": "", + "callback" : "https://localhost:3000/google/callback", + "scope" : [ "https://www.googleapis.com/auth/contacts.readonly"] + } +} +``` - - - OAuth 2.0 Client - - added nearly all scopes w.r.t. reading from people API - - [ ] **can this be easily extended to other Google apis?** - - add testusers (in test-mode, only testusers can use the app (you can register up to 100 test users)) - - create credentials - - API Key: `AIzaSyDNwWIFFpiOmAFiygS7eM-xcltFEWYIthk` - - Oauth Client - - name: `oauthclient-dapsi-dev` - - client id: `704703996998-mm0psiamc628soisd9hkovrjo5tlbd2q.apps.googleusercontent.com` - - client secret: `tatxC9Eli-VWJ5-i12IpSiST` +For more information on setting up a Google app that uses the Google People API, check out +- https://developers.google.com/people/v1/how-tos/authorizing +- https://developers.google.com/identity/protocols/oauth2 ## Use cases From a5efb1750d218a1a1b49e5185f745d4d4f5eebe2 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 13:01:25 +0100 Subject: [PATCH 21/30] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c5baef9..00a827f 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,10 @@ yarn run start:dev Prerequisites: - Google Developer Account +Steps: - Go to the [Google Developer Console](https://console.developers.google.com/) and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` - For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. - ![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) - Navigate to *APIs & Services* From 321f541a91dd0eff5957249d79a14e8aa78deb77 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 13:02:36 +0100 Subject: [PATCH 22/30] update --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 00a827f..31e4d10 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Steps: - Go to the [Google Developer Console](https://console.developers.google.com/) and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` - For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. -![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) +
![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) - Navigate to *APIs & Services* @@ -94,10 +94,8 @@ Steps: "transport": "session" }, "imgur": { - // ... }, "flickr": { - // ... }, "google": { "key": "", From 011e6cd051b9fc5913a761fbdcc1eb1fef7001ef Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Thu, 28 Jan 2021 13:03:12 +0100 Subject: [PATCH 23/30] update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 31e4d10..0a8bc1c 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Steps: - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. + - Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. ![Navigate to OAuth consent screen](docs/img/readme-adding-google-adding-test-users.png) From 9f228140b07a28bd428b96d08aaee058b1e317d8 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 09:56:55 +0100 Subject: [PATCH 24/30] update readme: added hint on creating google rml mapping --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 0a8bc1c..eddd8ec 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ yarn run start:dev #### [Google People API](https://developers.google.com/people?hl=en) +##### Google Cloud Platform configuration + Prerequisites: - Google Developer Account @@ -112,6 +114,26 @@ For more information on setting up a Google app that uses the Google People API, - https://developers.google.com/people/v1/how-tos/authorizing - https://developers.google.com/identity/protocols/oauth2 +#### Creating an RML Mapping that consumes and transforms the Google People API + +When creating an RML Mapping, you always have to define exactly one [`rml:logicalSource`](https://rml.io/specs/rml/#logical-source) +for every triples map you define. This way, the RML Processor knows where and how to access the data to be mapped. +Using the `rml:logicalSource`'s `rml:source` property we can specify the url of the Web API. + +In order to access protected resources, the RML Processor needs to include the required credentials when consuming the API. +The Google People API uses OAuth 2.0, and an authorization header should be added to the requests. Since these credentials +are managed by our web-app, the value for the authorization header is a template-variable, hence, the web-app will +recognize this and fill in the value. + +The following is an excerpt of the `rml:source` defined in [`rml/google/contact-transfer-using-schema-org.ttl`](/public/rml/google/contact-transfer-using-schema-org.ttl): +```turtle +:google_source + ex:AuthorizationHeader "{{authorizationHeader}}" ; + schema:name "Google API" ; + schema:url ; + schema:WebAPI . +``` + ## Use cases ### Use case: Transfer data From 2d596bb7398bde4bb578b274acd816637b286d59 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 10:48:40 +0100 Subject: [PATCH 25/30] update report: google people api --- docs/REPORT.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/REPORT.md b/docs/REPORT.md index 7b8b6f3..57d75a0 100644 --- a/docs/REPORT.md +++ b/docs/REPORT.md @@ -541,6 +541,24 @@ The following is an excerpt of the `rml:source` defined in [`rml/google/contact- ``` +### [Google People API](https://developers.google.com/people?hl=en) + +Using the Google People API we can transfer our Google contacts. + +> Please note that this use case can be extended easily to [other Google Products](https://developers.google.com/products?hl=en). + +#### Using [Schema.org] + +A Google contact can be mapped to a `schema:Person`, along with the following properties: + +| Google contact resource | `schema:Person` | +| --------------------- | ----------------------- | +| `givenName` | `schema:givenName` | +| `familyName` | `schema:familyName` | +| `displayName` | `schema:alternativeName` | + + + ## Features Within this project, we envisioned following features. From d10cbb111e8dad311b5696a71a240203fc7fb416 Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 11:36:10 +0100 Subject: [PATCH 26/30] update report: adding data providers --- docs/REPORT.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/docs/REPORT.md b/docs/REPORT.md index 57d75a0..21043cb 100644 --- a/docs/REPORT.md +++ b/docs/REPORT.md @@ -540,7 +540,6 @@ The following is an excerpt of the `rml:source` defined in [`rml/google/contact- schema:WebAPI . ``` - ### [Google People API](https://developers.google.com/people?hl=en) Using the Google People API we can transfer our Google contacts. @@ -557,6 +556,90 @@ A Google contact can be mapped to a `schema:Person`, along with the following pr | `familyName` | `schema:familyName` | | `displayName` | `schema:alternativeName` | +## Adding data providers + +This section will walk you through the different steps of adding a data provider to the PROV4ITDaTa Web-App, where the Google People API serves as an example. + +#### [Google People API](https://developers.google.com/people?hl=en) + +First, we discuss the configuration steps on the Google Cloud Platform. +Secondly, we elaborate on how to define the Google People API as a logical source in an RML Mapping. + +##### Google Cloud Platform configuration + +Prerequisites: +- Google Developer Account + +Steps: +- Go to the [Google Developer Console](https://console.developers.google.com/) + and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` +- For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. +
![Make sure that the correct project is selected](img/readme-adding-google-step001-selected-project.png) + +- Navigate to *APIs & Services* + +![Go to APIs and Services](img/readme-adding-google-navigate-to-APIs-and-Services.png) + +- Search and enable the API you wish to integrate. In this case, the [Google People API](https://console.cloud.google.com/apis/library/people.googleapis.com) + +- In order to make requests to the API, you need to set up authorization. + - Since we need to access private data, OAuth 2.0 credentials must be generated. + - Go to the [Credentials page](https://console.developers.google.com/apis/credentials), click on *Create Credentials* and select *OAuth client ID* +
![Create credentials and select OAuth client ID ](img/readme-adding-google-authorization-create-credentials-dropdown.png) + + - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server. + - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` + - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. + + +- Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. + ![Navigate to OAuth consent screen](img/readme-adding-google-adding-test-users.png) + +- Finally, add the *Client ID*, *Client secret* to the `config.json`. This may look like + +```json +{ + "defaults": { + "origin": "https://localhost:3000", + "transport": "session" + }, + "imgur": { + }, + "flickr": { + }, + "google": { + "key": "", + "secret": "", + "callback" : "https://localhost:3000/google/callback", + "scope" : [ "https://www.googleapis.com/auth/contacts.readonly"] + } +} + +``` + +For more information on setting up a Google app that uses the Google People API, check out +- https://developers.google.com/people/v1/how-tos/authorizing +- https://developers.google.com/identity/protocols/oauth2 + +#### Creating an RML Mapping that consumes and transforms the Google People API + +When creating an RML Mapping, you always have to define exactly one [`rml:logicalSource`](https://rml.io/specs/rml/#logical-source) +for every triples map you define. This way, the RML Processor knows where and how to access the data to be mapped. +Using the `rml:logicalSource`'s `rml:source` property we can specify the url of the Web API. + +In order to access protected resources, the RML Processor needs to include the required credentials when consuming the API. +The Google People API uses OAuth 2.0, and an authorization header should be added to the requests. Since these credentials +are managed by our web-app, the value for the authorization header is a template-variable, hence, the web-app will +recognize this and fill in the value. + +The following is an excerpt of the `rml:source` defined in [`rml/google/contact-transfer-using-schema-org.ttl`](/public/rml/google/contact-transfer-using-schema-org.ttl): +```turtle +:google_source + ex:AuthorizationHeader "{{authorizationHeader}}" ; + schema:name "Google API" ; + schema:url ; + schema:WebAPI . +``` ## Features From 8343e434944b647d9b4fedef73c642666f8eb40e Mon Sep 17 00:00:00 2001 From: Gertjan De Mulder Date: Fri, 5 Feb 2021 11:36:43 +0100 Subject: [PATCH 27/30] removed section about adding data providers (see docs/report) --- README.md | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index eddd8ec..17842ec 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Run the web-app in development mode using: yarn run start:dev ``` +<<<<<<< HEAD #### [Google People API](https://developers.google.com/people?hl=en) @@ -107,32 +108,9 @@ Steps: "scope" : [ "https://www.googleapis.com/auth/contacts.readonly"] } } +======= +>>>>>>> 75e1674 (removed section about adding data providers (see docs/report)) -``` - -For more information on setting up a Google app that uses the Google People API, check out -- https://developers.google.com/people/v1/how-tos/authorizing -- https://developers.google.com/identity/protocols/oauth2 - -#### Creating an RML Mapping that consumes and transforms the Google People API - -When creating an RML Mapping, you always have to define exactly one [`rml:logicalSource`](https://rml.io/specs/rml/#logical-source) -for every triples map you define. This way, the RML Processor knows where and how to access the data to be mapped. -Using the `rml:logicalSource`'s `rml:source` property we can specify the url of the Web API. - -In order to access protected resources, the RML Processor needs to include the required credentials when consuming the API. -The Google People API uses OAuth 2.0, and an authorization header should be added to the requests. Since these credentials -are managed by our web-app, the value for the authorization header is a template-variable, hence, the web-app will -recognize this and fill in the value. - -The following is an excerpt of the `rml:source` defined in [`rml/google/contact-transfer-using-schema-org.ttl`](/public/rml/google/contact-transfer-using-schema-org.ttl): -```turtle -:google_source - ex:AuthorizationHeader "{{authorizationHeader}}" ; - schema:name "Google API" ; - schema:url ; - schema:WebAPI . -``` ## Use cases From ab91fb0d8d6743c4a0f74c130e1b9104a68d5534 Mon Sep 17 00:00:00 2001 From: Ben De Meester Date: Fri, 5 Feb 2021 12:27:06 +0100 Subject: [PATCH 28/30] merge resolve --- README.md | 58 ------------------------------------------------------- 1 file changed, 58 deletions(-) diff --git a/README.md b/README.md index 17842ec..5aef805 100644 --- a/README.md +++ b/README.md @@ -54,64 +54,6 @@ Run the web-app in development mode using: yarn run start:dev ``` -<<<<<<< HEAD - - -#### [Google People API](https://developers.google.com/people?hl=en) - -##### Google Cloud Platform configuration - -Prerequisites: -- Google Developer Account - -Steps: -- Go to the [Google Developer Console](https://console.developers.google.com/) - and the first thing to do, is creating a project. In this example, the project was named `PROV4ITDaTa-DAPSI` -- For any further steps, make sure the project you created in the previous step is active. This is shown at the top of the console, as depicted by the following figure. -
![Make sure that the correct project is selected](docs/img/readme-adding-google-step001-selected-project.png) - -- Navigate to *APIs & Services* - -![Go to APIs and Services](docs/img/readme-adding-google-navigate-to-APIs-and-Services.png) - -- Search and enable the API you wish to integrate. In this case, the [Google People API](https://console.cloud.google.com/apis/library/people.googleapis.com) - -- In order to make requests to the API, you need to set up authorization. - - Since we need to access private data, OAuth 2.0 credentials must be generated. - - Go to the [Credentials page](https://console.developers.google.com/apis/credentials), click on *Create Credentials* and select *OAuth client ID* -
![Create credentials and select OAuth client ID ](docs/img/readme-adding-google-authorization-create-credentials-dropdown.png) - - - Set the *Application Type* to *Web Application*, enter a name, and add authorization redirect URI(s) pointing back to your web-app server. - - For example, assuming the web-app is served locally on port `3000`, the redirect URI would be: `https://localhost:3000/connect/google/callback` - - Don't forget to save the *Client ID* and *Client secret*, as you will need it later on. - - -- Since our Google app's publishing status is "testing", only test users will be able to use it. Navigate to the *OAuth consent screen* using the panel on the left, and add the test users. -![Navigate to OAuth consent screen](docs/img/readme-adding-google-adding-test-users.png) - -- Finally, add the *Client ID*, *Client secret* to the `config.json`. This may look like - -```json -{ - "defaults": { - "origin": "https://localhost:3000", - "transport": "session" - }, - "imgur": { - }, - "flickr": { - }, - "google": { - "key": "", - "secret": "", - "callback" : "https://localhost:3000/google/callback", - "scope" : [ "https://www.googleapis.com/auth/contacts.readonly"] - } -} -======= ->>>>>>> 75e1674 (removed section about adding data providers (see docs/report)) - - ## Use cases ### Use case: Transfer data From ca428e1c673ac7c6a3abe7a1e64a559bfcfdbaea Mon Sep 17 00:00:00 2001 From: Ben De Meester Date: Fri, 5 Feb 2021 12:27:41 +0100 Subject: [PATCH 29/30] 0.2.0 From c259a939a123b6f52c2e9424cff8c87807255db7 Mon Sep 17 00:00:00 2001 From: Ben De Meester Date: Fri, 5 Feb 2021 12:31:53 +0100 Subject: [PATCH 30/30] toc --- docs/REPORT.md | 57 +++++++++++++++++--------------------------------- 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/docs/REPORT.md b/docs/REPORT.md index 21043cb..4bd4ae8 100644 --- a/docs/REPORT.md +++ b/docs/REPORT.md @@ -6,19 +6,8 @@ - [PROV4ITDaTa - Technical Report](#prov4itdata---technical-report) - [Alignment to the milestones](#alignment-to-the-milestones) - - [M1: Design configuration-based DTP exporter, applying it to a use case (deadline 10/2020)](#m1-design-configuration-based-dtp-exporter-applying-it-to-a-use-case-deadline-102020) - - [M2: Improve data provenance of the DTP exporter (deadline 12/2020)](#m2-improve-data-provenance-of-the-dtp-exporter-deadline-122020) - - [M3: Design query-based DTP importer for Solid, applying it to a use case (deadline 01/2020)](#m3-design-query-based-dtp-importer-for-solid-applying-it-to-a-use-case-deadline-012020) - - [M4: Implement configuration-based DTP exporter (deadline 04/2020)](#m4-implement-configuration-based-dtp-exporter-deadline-042020) - - [M5: Implement provenance-based DTP exporter (deadline 05/2020)](#m5-implement-provenance-based-dtp-exporter-deadline-052020) - - [M6: Implement query-based DTP importer for Solid (deadline 04/2020)](#m6-implement-query-based-dtp-importer-for-solid-deadline-042020) - - [M7: Integrate and finalize the exporter and importer, showcasing the use case (deadline 05/2020)](#m7-integrate-and-finalize-the-exporter-and-importer-showcasing-the-use-case-deadline-052020) - [Architecture](#architecture) - [Components](#components) - - [RML Mapping documents](#rml-mapping-documents) - - [RMLMapper](#rmlmapper) - - [Solid pods](#solid-pods) - - [Web App](#web-app) - [Relation to DTP](#relation-to-dtp) - [Requirements](#requirements) - [Demonstrator](#demonstrator) @@ -28,26 +17,17 @@ - [Imgur](#imgur) - [Google People API](#google-people-api) - [Adding data providers](#adding-data-providers) + - [Google People API](#google-people-api-1) + - [Adding data providers](#adding-data-providers-1) - [Features](#features) - [Use Open Standards and Open Source](#use-open-standards-and-open-source) - [Mapping files to transfer data](#mapping-files-to-transfer-data) - [Automatic Data Provenance Generation](#automatic-data-provenance-generation) - [Output RDF](#output-rdf) - - [Syntactic Interoperability](#syntactic-interoperability) - - [Semantic Interoperability](#semantic-interoperability) - - [Structural Interoperability](#structural-interoperability) - - [Data Compatibility](#data-compatibility) - [Data Portability](#data-portability) - [Security and Privacy](#security-and-privacy) - - [Data Minimization](#data-minimization) - - [User Control](#user-control) - - [Minimal Scopes for Auth Tokens](#minimal-scopes-for-auth-tokens) - - [Data retention](#data-retention) - - [Abuse](#abuse) - [Personalization](#personalization) - [Quality](#quality) - - [Data Quality](#data-quality) - - [Software Quality](#software-quality) - [GDPR](#gdpr) - [Conclusion](#conclusion) - [References](#references) @@ -197,15 +177,15 @@ The main component is the PROV4ITDaTa Web App, which performs the different step 2. transforming the fetched data to [RDF]. 3. In the third step, the [Web App](#web-app) finalizes the data transfer by storing the [RDF], generated by the [RMLMapper-JAVA], to the Solid pod. -#### [RML Mapping documents][RML-spec] +#### RML Mapping documents An RML Mapping provides the means to create semantically enriched RDF data from heterogenous and (semi-) structured sources, using a declarative set of rules ([RML][RML-spec]). Describing the Web API as a source in an RML Mapping allows the [RMLMapper][RMLMapper-JAVA] to determine how requests should be made in order to consume the Web API. Furthermore, it requires a one-time effort and avoids hard-coded implementations for each service to be added. The mapping can easily be reused for — and extended to — similar Web APIs. -#### [RMLMapper][RMLMapper-JAVA] +#### RMLMapper -The RMLMapper, proxied by the [RMLMapper Web API](#web-app), processes the selected RML Mapping and yields both the generated [RDF] and [provenance] data. +The [RMLMapper][RMLMapper-JAVA], proxied by the [RMLMapper Web API](#web-app), processes the selected RML Mapping and yields both the generated [RDF] and [provenance] data. In the background, the RML Mapper determines how to consume a Web API based on its description, without the need for integrating code that is tightly coupled to a specific Web API. Such hard-coded implementations are error-prone and time-consuming (consider, for example, the creation of tests and rebuilding the application every time a service is added). @@ -328,7 +308,7 @@ There are three different resources to model ### Best-practice vocabularies -#### [Schema.org] +#### Schema.org ##### Image @@ -342,7 +322,7 @@ An Image Gallery can be represented using `schema:ImageGallery`. Furthermore, th A Collection can be modelled by means of a `schema:Collection`, which can be linked to its Image Gallery resources through the `schema:hasPart` property. -#### [DCAT] +#### DCAT ##### Image @@ -371,12 +351,12 @@ We can model an Image Gallery as a `dcat:Dataset`, which can point to **zero or [DCAT] contains a `dcat:Catalog` class, a curated collection of metadata about resources (e.g., datasets and data services in the context of a data catalog). A `dcat:Catalog` can be linked to **zero or more** `dcat:Dataset`s. -### [Flickr] +### Flickr Flickr is an online photo management and sharing application. Its resources are made available through the [Flickr API][Flickr-API]. Flickr follows the [OAuth1.0a] protocol which requires that requests to protected resources are signed using the Consumer Secret and Token Secret. By specifying the protocol in the [RML Mapping][RML-mapping] the [RMLMapper-JAVA] takes care of the necessary steps for creating requests to protected resources. This also contributes to the extensibility of our solution: when a service decides to change to another protocol, only changes to the [RML Mapping][RML-mapping] must be made. Hence, avoiding the need for rebuilding code. -#### Using [Schema.org] +#### Using Schema.org - A Flickr Photo resource can be mapped to a `schema:ImageObject`. - A Flickr Photoset resource can be mapped to instances of `schema:ImageGallery`. The table below shows how its properties are mapped. @@ -400,7 +380,7 @@ Its resources are made available through the [Flickr API][Flickr-API]. Flickr fo - A Flickr Album can contain image resources - A Flickr Collection can contain Flickr Albums, which contain image resources. Therefore, a Flickr Collection can be modelled by a `dcat:Catalog` which can be linked to **zero or more** `dcat:Dataset`s (Albums) -### [Imgur] +### Imgur Imgur, an image hosting and sharing website, enables its users to quickly upload and share images and GIFs on social media platforms (e.g. Reddit, Twitter, etc.). Unlike the Flickr API, the [Imgur API][Imgur-API] uses OAuth 2.0. @@ -439,9 +419,9 @@ An Imgur image can be mapped to a `schema:ImageObject`, along with the following - A Imgur Album can contain image resources, and can be modelled by `dcat:Dataset`. - An Imgur Gallery can contain Imgur Albums, which contain image resources. Therefore, an Imgur Gallery can be modelled by a `dcat:Catalog` which can be linked to **zero or more** `dcat:Dataset`s (Albums). -### [Google People API](https://developers.google.com/people?hl=en) +### Google People API -Using the Google People API we can transfer our Google contacts. +Using the [Google People API](https://developers.google.com/people?hl=en) we can transfer our Google contacts. > Please note that this use case can be extended easily to [other Google Products](https://developers.google.com/products?hl=en). @@ -459,10 +439,10 @@ A Google contact can be mapped to a `schema:Person`, along with the following pr This section will walk you through the different steps of adding a data provider to the PROV4ITDaTa Web-App, where the Google People API serves as an example. -#### [Google People API](https://developers.google.com/people?hl=en) +#### Google People API First, we discuss the configuration steps on the Google Cloud Platform. -Secondly, we elaborate on how to define the Google People API as a logical source in an RML Mapping. +Secondly, we elaborate on how to define the [Google People API] as a logical source in an RML Mapping. ##### Google Cloud Platform configuration @@ -540,9 +520,9 @@ The following is an excerpt of the `rml:source` defined in [`rml/google/contact- schema:WebAPI . ``` -### [Google People API](https://developers.google.com/people?hl=en) +### Google People API -Using the Google People API we can transfer our Google contacts. +Using the [Google People API] we can transfer our Google contacts. > Please note that this use case can be extended easily to [other Google Products](https://developers.google.com/products?hl=en). @@ -560,10 +540,10 @@ A Google contact can be mapped to a `schema:Person`, along with the following pr This section will walk you through the different steps of adding a data provider to the PROV4ITDaTa Web-App, where the Google People API serves as an example. -#### [Google People API](https://developers.google.com/people?hl=en) +#### Google People API First, we discuss the configuration steps on the Google Cloud Platform. -Secondly, we elaborate on how to define the Google People API as a logical source in an RML Mapping. +Secondly, we elaborate on how to define the [Google People API] as a logical source in an RML Mapping. ##### Google Cloud Platform configuration @@ -904,6 +884,7 @@ and provide integration alternatives with, e.g., [DTP]. [Flickr]: https://www.flickr.com/about [Flickr-API]: https://www.flickr.com/services/developer/api/ [HTTPS]: https://www.ietf.org/rfc/rfc2818.txt +[Google People API]: https://developers.google.com/people?hl=en [Imgur]: https://imgur.com/ [Imgur-API]: https://apidocs.imgur.com/ [LDP]: https://www.w3.org/TR/ldp/