From fc0c038c076c4e02f817dfc2d35a9ea4fcc83887 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Sat, 14 May 2022 19:01:40 -0700 Subject: [PATCH 01/15] Add experimental support for the Composite API --- examples/salesforce/ReadCompositeAPIData.apex | 103 ++++++++++++++++++ .../SalesforceCompositeAPIOutput.py | 58 ++++++++++ 2 files changed, 161 insertions(+) create mode 100644 examples/salesforce/ReadCompositeAPIData.apex create mode 100644 snowfakery/experimental/SalesforceCompositeAPIOutput.py diff --git a/examples/salesforce/ReadCompositeAPIData.apex b/examples/salesforce/ReadCompositeAPIData.apex new file mode 100644 index 00000000..2194af50 --- /dev/null +++ b/examples/salesforce/ReadCompositeAPIData.apex @@ -0,0 +1,103 @@ +// This code can be run in Anonymous Apex to load data from Github Gists +// sfdx force:apex:execute -f paulignore/composite/callout.apex -u SFDX_USERALIAS + +public class JSONSObjectLoader { + public void loadJson(String url) { + String json_records = downloadJSON(url); + loadRecords(json_records); + System.debug('Loaded JSON ' + url); + } + + public String downloadJSON(String url){ + HttpResponse response = makeHTTPCall('GET', url, null); + return response.getBody(); + } + + public HttpResponse makeHTTPCall(String method, String url, String post_body){ + Http h = new Http(); + HttpRequest request = new HttpRequest(); + request.setEndpoint(url); + request.setMethod(method); + if(post_body != null){ + request.setHeader('Content-Type', 'application/json'); + request.setBody(post_body); + } + + request.setHeader('Authorization', 'OAuth ' + UserInfo.getSessionId()); + request.setTimeout(120000); + return h.send(request); + } + + public void loadRecords(String json_records){ + String error = null; + String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; + HttpResponse response = makeHTTPCall('POST', graph_url, json_records); + String response_body = response.getBody(); + parseResponse(response_body); + + if(response.getStatusCode()!=200){ + error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; + }else{ + try{ + error = parseResponse(response_body); + }catch(Exception e){ + error = 'Error creating objects! ' + e.getMessage(); + } + } + + if(error!=null){ + System.debug('Error: ' + error); + System.debug('DOWNLOADED Data'); + System.debug(response_body); + CalloutException e = new CalloutException('Error creating objects! ' + error); + throw e; + } + } + + private String parseResponse(String response) { + String rc = null; + Map graph_parse = (Map)Json.deserializeUntyped(response); + bool success = (bool)graph_parse.get('isSuccessful'); + if (success){ + return null; + }else{ + return parseError(response); + } + } + + private String parseError(Map graph_parse){ + List graphs = (List)graph_parse.get('graphs'); + for(Object graph: graphs){ + Map graphobj = (Map) graph; + Map graphResponse = (Map)graphobj.get('graphResponse'); + List compositeResponse = (List)graphResponse.get('compositeResponse'); + for(Object single_response: compositeResponse){ + Map single_response_obj = (Map)single_response; + Integer status = (Integer)single_response_obj.get('httpStatusCode'); + if(status!=200 && status!=201){ + List body = (List)single_response_obj.get('body'); + Map body_obj = (Map)body[0]; + if(rc==null){ + rc = body_obj.toString(); + break; + } + if((String)body_obj.get('errorCode')!='PROCESSING_HALTED'){ + System.debug('Error: ' + body.toString()); + } + } + } + } + + return rc; + } +} + +JSONSObjectLoader loader = new JSONSObjectLoader(); +for (Integer i = 0; i < 16;) { + loader.LoadJson('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); + System.debug(++i); + loader.LoadJson('https://gist.githubusercontent.com/prescod/86a64c63fe294b8123b895b489852e6f/raw/e8ae51b7f5af5db6b5b800a3f9c40ebd0c7a1998/composite2.json'); + System.debug(++i); + loader.LoadJson('https://gist.githubusercontent.com/prescod/21b4a0b9fe5cac171abb8f6a633cb39a/raw/cd7e0f1025a61946b4f923999afa546f099282fa/composite3.json'); + System.debug(++i); +} diff --git a/snowfakery/experimental/SalesforceCompositeAPIOutput.py b/snowfakery/experimental/SalesforceCompositeAPIOutput.py new file mode 100644 index 00000000..9542a741 --- /dev/null +++ b/snowfakery/experimental/SalesforceCompositeAPIOutput.py @@ -0,0 +1,58 @@ +# Use this experimental OutputStream like this: + +# snowfakery --output-format snowfakery.experimental.SalesforceCompositeOutput recipe.yml > composite.json +# +# Once you have the file you can make it accessible to Salesforce by uploading it +# to some form of server. E.g. Github gist, Heroku, etc. +# +# Then you can use Anon Apex like that in `ReadCompositeAPIData.apex` to load it into +# any org. + +import json +import typing as T +import datetime + +from snowfakery.output_streams import FileOutputStream + + +class SalesforceCompositeOutput(FileOutputStream): + """Output stream that generates records for Salesforce's Composite API""" + + encoders: T.Mapping[type, T.Callable] = { + **FileOutputStream.encoders, + datetime.date: str, + datetime.datetime: str, + bool: bool, + } + is_text = True + + def __init__(self, file, **kwargs): + assert file + super().__init__(file, **kwargs) + self.rows = [] + + def write_single_row(self, tablename: str, row: T.Dict) -> None: + row_without_id = row.copy() + del row_without_id["id"] + values = { + "method": "POST", + "referenceId": f"{tablename}_{row['id']}", + "url": f"/services/data/v50.0/sobjects/{tablename}", + "body": row_without_id, + } + self.rows.append(values) + + def flatten( + self, + sourcetable: str, + fieldname: str, + source_row_dict, + target_object_row, + ) -> T.Union[str, int]: + target_reference = f"{target_object_row._tablename}_{target_object_row.id}" + return "@{%s.id}" % target_reference + + def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: + data = {"graphs": [{"graphId": "graph", "compositeRequest": self.rows}]} + self.write(json.dumps(data, indent=2)) + return super().close() From 71aed6a77d0b3dff16238bd1aec8f253b5b98cd4 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 17 May 2022 15:17:32 -0700 Subject: [PATCH 02/15] Apex loading touch-ups --- ...APIData.apex => LoadCompositeAPIData.apex} | 30 +++++++++---------- .../SalesforceCompositeAPIOutput.py | 14 ++++++--- 2 files changed, 24 insertions(+), 20 deletions(-) rename examples/salesforce/{ReadCompositeAPIData.apex => LoadCompositeAPIData.apex} (85%) diff --git a/examples/salesforce/ReadCompositeAPIData.apex b/examples/salesforce/LoadCompositeAPIData.apex similarity index 85% rename from examples/salesforce/ReadCompositeAPIData.apex rename to examples/salesforce/LoadCompositeAPIData.apex index 2194af50..376e93c1 100644 --- a/examples/salesforce/ReadCompositeAPIData.apex +++ b/examples/salesforce/LoadCompositeAPIData.apex @@ -1,5 +1,8 @@ // This code can be run in Anonymous Apex to load data from Github Gists -// sfdx force:apex:execute -f paulignore/composite/callout.apex -u SFDX_USERALIAS +// sfdx force:apex:execute -f ./examples/salesforce/LoadCompositeAPIData.apex -u Snowfakery__qa +// or +// cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa + public class JSONSObjectLoader { public void loadJson(String url) { @@ -33,8 +36,6 @@ public class JSONSObjectLoader { String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; HttpResponse response = makeHTTPCall('POST', graph_url, json_records); String response_body = response.getBody(); - parseResponse(response_body); - if(response.getStatusCode()!=200){ error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; }else{ @@ -49,26 +50,23 @@ public class JSONSObjectLoader { System.debug('Error: ' + error); System.debug('DOWNLOADED Data'); System.debug(response_body); - CalloutException e = new CalloutException('Error creating objects! ' + error); + CalloutException e = new CalloutException( error); throw e; } } private String parseResponse(String response) { - String rc = null; Map graph_parse = (Map)Json.deserializeUntyped(response); - bool success = (bool)graph_parse.get('isSuccessful'); - if (success){ - return null; - }else{ - return parseError(response); - } + return parseError(graph_parse); } private String parseError(Map graph_parse){ + String rc = null; List graphs = (List)graph_parse.get('graphs'); for(Object graph: graphs){ Map graphobj = (Map) graph; + boolean success = (boolean)graphobj.get('isSuccessful'); + if(success) continue; Map graphResponse = (Map)graphobj.get('graphResponse'); List compositeResponse = (List)graphResponse.get('compositeResponse'); for(Object single_response: compositeResponse){ @@ -77,13 +75,11 @@ public class JSONSObjectLoader { if(status!=200 && status!=201){ List body = (List)single_response_obj.get('body'); Map body_obj = (Map)body[0]; - if(rc==null){ + if(rc==null && (String)body_obj.get('errorCode')!='PROCESSING_HALTED') { + System.debug('Error: ' + body.toString()); rc = body_obj.toString(); break; } - if((String)body_obj.get('errorCode')!='PROCESSING_HALTED'){ - System.debug('Error: ' + body.toString()); - } } } } @@ -93,7 +89,9 @@ public class JSONSObjectLoader { } JSONSObjectLoader loader = new JSONSObjectLoader(); -for (Integer i = 0; i < 16;) { +// experimentally, much more than 20 hits a cumulative +// maximum time allotted for callout error +for (Integer i = 0; i < 20;) { loader.LoadJson('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); System.debug(++i); loader.LoadJson('https://gist.githubusercontent.com/prescod/86a64c63fe294b8123b895b489852e6f/raw/e8ae51b7f5af5db6b5b800a3f9c40ebd0c7a1998/composite2.json'); diff --git a/snowfakery/experimental/SalesforceCompositeAPIOutput.py b/snowfakery/experimental/SalesforceCompositeAPIOutput.py index 9542a741..0b6cf138 100644 --- a/snowfakery/experimental/SalesforceCompositeAPIOutput.py +++ b/snowfakery/experimental/SalesforceCompositeAPIOutput.py @@ -1,12 +1,18 @@ # Use this experimental OutputStream like this: -# snowfakery --output-format snowfakery.experimental.SalesforceCompositeOutput recipe.yml > composite.json +# snowfakery --output-format snowfakery.experimental.SalesforceCompositeAPIOutput recipe.yml > composite.json # # Once you have the file you can make it accessible to Salesforce by uploading it # to some form of server. E.g. Github gist, Heroku, etc. # -# Then you can use Anon Apex like that in `ReadCompositeAPIData.apex` to load it into -# any org. +# Then you can use Anon Apex like that in `LoadCompositeAPIData.apex` to load it into +# any org. e.g.: + +# sfdx force:apex:execute -f ./examples/salesforce/LoadCompositeAPIData.apex -u Snowfakery__qa +# or +# cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa +# +# Note that Salesforce will complain if the dataset has more than 500 rows. import json import typing as T @@ -15,7 +21,7 @@ from snowfakery.output_streams import FileOutputStream -class SalesforceCompositeOutput(FileOutputStream): +class SalesforceCompositeAPIOutput(FileOutputStream): """Output stream that generates records for Salesforce's Composite API""" encoders: T.Mapping[type, T.Callable] = { From ee22b9f888a2149afd795bac4cacf31a0d6a6499 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 17 May 2022 15:17:32 -0700 Subject: [PATCH 03/15] Apex loading touch-ups --- examples/salesforce/LoadCompositeAPIData.apex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/salesforce/LoadCompositeAPIData.apex b/examples/salesforce/LoadCompositeAPIData.apex index 376e93c1..29fc2f9f 100644 --- a/examples/salesforce/LoadCompositeAPIData.apex +++ b/examples/salesforce/LoadCompositeAPIData.apex @@ -4,7 +4,7 @@ // cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa -public class JSONSObjectLoader { +public class LoadCompositeAPIData { public void loadJson(String url) { String json_records = downloadJSON(url); loadRecords(json_records); @@ -89,6 +89,7 @@ public class JSONSObjectLoader { } JSONSObjectLoader loader = new JSONSObjectLoader(); +LoadCompositeAPIData loader = new LoadCompositeAPIData(); // experimentally, much more than 20 hits a cumulative // maximum time allotted for callout error for (Integer i = 0; i < 20;) { From 9f98330e07a59aabe6cea4b6678d2e4872566c40 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Wed, 18 May 2022 17:36:41 -0700 Subject: [PATCH 04/15] Support multi-file datasets --- examples/salesforce/LoadCompositeAPIData.apex | 49 +++++++++------ snowfakery/api.py | 4 ++ snowfakery/cli.py | 8 --- snowfakery/data_generator_runtime.py | 2 + .../SalesforceCompositeAPIOutput.py | 61 ++++++++++++++++++- snowfakery/output_streams.py | 5 ++ 6 files changed, 102 insertions(+), 27 deletions(-) diff --git a/examples/salesforce/LoadCompositeAPIData.apex b/examples/salesforce/LoadCompositeAPIData.apex index 29fc2f9f..29d19db4 100644 --- a/examples/salesforce/LoadCompositeAPIData.apex +++ b/examples/salesforce/LoadCompositeAPIData.apex @@ -5,18 +5,31 @@ public class LoadCompositeAPIData { - public void loadJson(String url) { + + public void loadSingleJson(String url) { + System.debug('Loading JSON ' + url); String json_records = downloadJSON(url); loadRecords(json_records); System.debug('Loaded JSON ' + url); } - public String downloadJSON(String url){ + public void loadJsonSet(String set_url){ + String json_record_sets = downloadJSON(set_url); + Map data = (Map)Json.deserializeUntyped(json_record_sets); + List tables = (List)data.get('tables'); + for(Object table_url: tables){ + Map url_obj = (Map) table_url; + String url = (String)url_obj.get('url'); + loadSingleJson(url); + } + } + + private String downloadJSON(String url){ HttpResponse response = makeHTTPCall('GET', url, null); return response.getBody(); } - public HttpResponse makeHTTPCall(String method, String url, String post_body){ + private HttpResponse makeHTTPCall(String method, String url, String post_body){ Http h = new Http(); HttpRequest request = new HttpRequest(); request.setEndpoint(url); @@ -31,7 +44,7 @@ public class LoadCompositeAPIData { return h.send(request); } - public void loadRecords(String json_records){ + private void loadRecords(String json_records){ String error = null; String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; HttpResponse response = makeHTTPCall('POST', graph_url, json_records); @@ -39,11 +52,7 @@ public class LoadCompositeAPIData { if(response.getStatusCode()!=200){ error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; }else{ - try{ - error = parseResponse(response_body); - }catch(Exception e){ - error = 'Error creating objects! ' + e.getMessage(); - } + error = parseResponse(response_body); } if(error!=null){ @@ -90,13 +99,17 @@ public class LoadCompositeAPIData { JSONSObjectLoader loader = new JSONSObjectLoader(); LoadCompositeAPIData loader = new LoadCompositeAPIData(); -// experimentally, much more than 20 hits a cumulative + +// load a single Composite Graph JSON +loader.loadSingleJson('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); + +// load a set of 3 Composite Graph JSONs +loader.loadJsonSet('https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json'); +System.debug('SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! '); +// experimentally, much more than 15 hits a cumulative // maximum time allotted for callout error -for (Integer i = 0; i < 20;) { - loader.LoadJson('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); - System.debug(++i); - loader.LoadJson('https://gist.githubusercontent.com/prescod/86a64c63fe294b8123b895b489852e6f/raw/e8ae51b7f5af5db6b5b800a3f9c40ebd0c7a1998/composite2.json'); - System.debug(++i); - loader.LoadJson('https://gist.githubusercontent.com/prescod/21b4a0b9fe5cac171abb8f6a633cb39a/raw/cd7e0f1025a61946b4f923999afa546f099282fa/composite3.json'); - System.debug(++i); -} +// +// for (Integer i = 0; i < 15; i++) { +// loader.loadSingleJson('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); +// System.debug(i); +// } diff --git a/snowfakery/api.py b/snowfakery/api.py index 7b76ac82..20c579d5 100644 --- a/snowfakery/api.py +++ b/snowfakery/api.py @@ -251,6 +251,10 @@ def _get_output_streams(dburls, output_files, output_format, output_folder): if output_stream_cls.uses_folder: output_streams.append(output_stream_cls(output_folder)) + elif output_folder and str(output_folder) != "." and not output_files: + raise exc.DataGenError( + "--output-folder can only be used with --output-file= or --output-format=csv" + ) if output_files: for f in output_files: diff --git a/snowfakery/cli.py b/snowfakery/cli.py index 75b1e67c..7a8326bd 100755 --- a/snowfakery/cli.py +++ b/snowfakery/cli.py @@ -276,14 +276,6 @@ def validate_options( "Sorry, you need to pick --dburl or --output-file " "because they are mutually exclusive." ) - if ( - output_folder - and str(output_folder) != "." - and not (output_files or output_format == "csv") - ): - raise click.ClickException( - "--output-folder can only be used with --output-file= or --output-format=csv" - ) if target_number and reps: raise click.ClickException( diff --git a/snowfakery/data_generator_runtime.py b/snowfakery/data_generator_runtime.py index 4d3dcfd1..9c06ff30 100644 --- a/snowfakery/data_generator_runtime.py +++ b/snowfakery/data_generator_runtime.py @@ -375,6 +375,8 @@ def loop_over_templates_until_finished(self, continuing): self.iteration_count += 1 continuing = True self.globals.reset_slots() + # let the output stream know that the recipe was finished + self.output_stream.complete_recipe() def loop_over_templates_once(self, statement_list, continuing: bool): for statement in statement_list: diff --git a/snowfakery/experimental/SalesforceCompositeAPIOutput.py b/snowfakery/experimental/SalesforceCompositeAPIOutput.py index 0b6cf138..bdba4a95 100644 --- a/snowfakery/experimental/SalesforceCompositeAPIOutput.py +++ b/snowfakery/experimental/SalesforceCompositeAPIOutput.py @@ -14,11 +14,14 @@ # # Note that Salesforce will complain if the dataset has more than 500 rows. +# TODO: Add tests + import json import typing as T import datetime +from pathlib import Path -from snowfakery.output_streams import FileOutputStream +from snowfakery.output_streams import FileOutputStream, OutputStream class SalesforceCompositeAPIOutput(FileOutputStream): @@ -62,3 +65,59 @@ def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: data = {"graphs": [{"graphId": "graph", "compositeRequest": self.rows}]} self.write(json.dumps(data, indent=2)) return super().close() + + +class Folder(OutputStream): + uses_folder = True + + def __init__(self, output_folder, **kwargs): + super().__init__(None, **kwargs) + self.target_path = Path(output_folder) + if not Path.exists(self.target_path): + Path.mkdir(self.target_path, exist_ok=True) + self.recipe_sets = [[]] + self.filenum = 0 + self.filenames = [] + + def write_single_row(self, tablename: str, row: T.Dict) -> None: + self.recipe_sets[-1].append((tablename, row)) + + def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: + self.flush_sets() + table_metadata = [{"url": str(filename)} for filename in self.filenames] + metadata = { + "@context": "http://www.w3.org/ns/csvw", + "tables": table_metadata, + } + metadata_filename = self.target_path / "csvw_metadata.json" + with open(metadata_filename, "w") as f: + json.dump(metadata, f, indent=2) + return [f"Created {self.target_path}"] + + def complete_recipe(self, *args): + ready_rows = sum(len(s) for s in self.recipe_sets) + if ready_rows > 500: + self.flush_sets() + self.recipe_sets.append([]) + + def flush_sets(self): + sets = self.recipe_sets + batches = [[]] + for set in sets: + if len(batches[-1]) + len(set) > 500: + batches.append(set.copy()) + else: + batches[-1].extend(set) + for batch in batches: + if len(batch): + self.filenum += 1 + filename = Path(self.target_path) / f"{self.filenum}.composite.json" + self.save_batch(filename, batch) + + def save_batch(self, filename, batch): + with open(filename, "w") as open_file, SalesforceCompositeAPIOutput( + open_file + ) as out: + self.filenames.append(filename) + for tablename, row in batch: + out.write_row(tablename, row) diff --git a/snowfakery/output_streams.py b/snowfakery/output_streams.py index 5a719c70..1097a1ee 100644 --- a/snowfakery/output_streams.py +++ b/snowfakery/output_streams.py @@ -136,6 +136,11 @@ def __enter__(self, *args): def __exit__(self, *args): self.close() + def complete_recipe(self, *args): + """Let the output stream know that a complete recipe + set was generated.""" + pass + class SmartStream: """Common code for managing stream/file opening/closing From aeb5bb0b21a61f74b9a1de1967b3dda4104a7a5a Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Thu, 19 May 2022 18:24:50 -0700 Subject: [PATCH 05/15] Add JSON Bundle Feature --- examples/salesforce/LoadCompositeAPIData.apex | 85 ++++++++++++++++--- .../SalesforceCompositeAPIOutput.py | 72 +++++++++++----- 2 files changed, 125 insertions(+), 32 deletions(-) diff --git a/examples/salesforce/LoadCompositeAPIData.apex b/examples/salesforce/LoadCompositeAPIData.apex index 29d19db4..8913169c 100644 --- a/examples/salesforce/LoadCompositeAPIData.apex +++ b/examples/salesforce/LoadCompositeAPIData.apex @@ -6,21 +6,73 @@ public class LoadCompositeAPIData { - public void loadSingleJson(String url) { + // Load one of three JSON formats. + // + // 1. One with a top-level key called "tables" which links to other + // composite graph payload jsons, like this: + // + // https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Folder + // + // 2. One with a top-level key called "data" which embeds compsite graph + // payloads as strings. Like this: + // + // https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json' + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Bundle + // + // 3. One which is just a single composite graph payload like this: + // + // https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json + // + // Which is recognizable by its top-level "graphs" key. + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput + + public void loadJsonSet(String set_url){ + String json_record_sets = downloadJSON(set_url); + Map data = (Map)Json.deserializeUntyped(json_record_sets); + List tables = (List)data.get('tables'); + if(tables != null){ + loadDistributedJsonSet(tables); + return; + } + + List graph_jsons = (List)data.get('data'); + if(graph_jsons != null){ + loadBundledJsonSet(graph_jsons); + return; + } + + List graphs = (List)data.get('graphs'); + if(graphs != null){ + loadRecords(json_record_sets); + return; + } + + } + + // optimized method for a single composite graph (<500 records) + // This method doesn't parse the JSON to see what's in it. + public void loadSingleJsonGraphPayload(String url) { System.debug('Loading JSON ' + url); String json_records = downloadJSON(url); loadRecords(json_records); System.debug('Loaded JSON ' + url); } - public void loadJsonSet(String set_url){ - String json_record_sets = downloadJSON(set_url); - Map data = (Map)Json.deserializeUntyped(json_record_sets); - List tables = (List)data.get('tables'); + public void loadDistributedJsonSet(List tables){ for(Object table_url: tables){ Map url_obj = (Map) table_url; String url = (String)url_obj.get('url'); - loadSingleJson(url); + loadSingleJsonGraphPayload(url); + } + } + + public void loadBundledJsonSet(List graph_jsons){ + for(Object graph_json: graph_jsons){ + loadRecords((String)graph_json); } } @@ -41,6 +93,7 @@ public class LoadCompositeAPIData { request.setHeader('Authorization', 'OAuth ' + UserInfo.getSessionId()); request.setTimeout(120000); + System.debug(url); return h.send(request); } @@ -57,8 +110,8 @@ public class LoadCompositeAPIData { if(error!=null){ System.debug('Error: ' + error); - System.debug('DOWNLOADED Data'); - System.debug(response_body); + // System.debug('DOWNLOADED Data'); + // System.debug(response_body); CalloutException e = new CalloutException( error); throw e; } @@ -97,19 +150,25 @@ public class LoadCompositeAPIData { } } -JSONSObjectLoader loader = new JSONSObjectLoader(); LoadCompositeAPIData loader = new LoadCompositeAPIData(); -// load a single Composite Graph JSON -loader.loadSingleJson('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); +// load a single Composite Graph JSON Payload +loader.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); + +// load anoter single one slightly less efficiently +loader.loadJsonSet('https://gist.githubusercontent.com/prescod/ffa992a7218906ab0dcf160b5d755259/raw/f9d40587a2ba9b04275241723637ed571bd55617/Graph%2520Gist'); -// load a set of 3 Composite Graph JSONs +// load a set of 3 Composite Graph JSONs in a distributed set loader.loadJsonSet('https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json'); + +// load a single-file bundle of 3 Composite Graph JSONs +loader.loadJsonSet('https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json'); + System.debug('SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! '); // experimentally, much more than 15 hits a cumulative // maximum time allotted for callout error // // for (Integer i = 0; i < 15; i++) { -// loader.loadSingleJson('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); +// loader.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); // System.debug(i); // } diff --git a/snowfakery/experimental/SalesforceCompositeAPIOutput.py b/snowfakery/experimental/SalesforceCompositeAPIOutput.py index bdba4a95..62da3d92 100644 --- a/snowfakery/experimental/SalesforceCompositeAPIOutput.py +++ b/snowfakery/experimental/SalesforceCompositeAPIOutput.py @@ -20,6 +20,7 @@ import typing as T import datetime from pathlib import Path +from tempfile import TemporaryDirectory from snowfakery.output_streams import FileOutputStream, OutputStream @@ -62,6 +63,8 @@ def flatten( return "@{%s.id}" % target_reference def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: + # NOTE: Could improve loading performance by breaking graphs up + # to allow server-side parallelization, but I'd risk locking issues data = {"graphs": [{"graphId": "graph", "compositeRequest": self.rows}]} self.write(json.dumps(data, indent=2)) return super().close() @@ -76,14 +79,19 @@ def __init__(self, output_folder, **kwargs): if not Path.exists(self.target_path): Path.mkdir(self.target_path, exist_ok=True) self.recipe_sets = [[]] + self.current_batch = [] self.filenum = 0 self.filenames = [] + def write_row(self, tablename: str, row_with_references: T.Dict) -> None: + self.recipe_sets[-1].append((tablename, row_with_references)) + def write_single_row(self, tablename: str, row: T.Dict) -> None: - self.recipe_sets[-1].append((tablename, row)) + assert 0, "Shouldn't be called. write_row should be called instead" def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: self.flush_sets() + self.flush_batch() table_metadata = [{"url": str(filename)} for filename in self.filenames] metadata = { "@context": "http://www.w3.org/ns/csvw", @@ -95,29 +103,55 @@ def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: return [f"Created {self.target_path}"] def complete_recipe(self, *args): - ready_rows = sum(len(s) for s in self.recipe_sets) - if ready_rows > 500: - self.flush_sets() + self.flush_sets() self.recipe_sets.append([]) def flush_sets(self): - sets = self.recipe_sets - batches = [[]] - for set in sets: - if len(batches[-1]) + len(set) > 500: - batches.append(set.copy()) - else: - batches[-1].extend(set) - for batch in batches: - if len(batch): - self.filenum += 1 - filename = Path(self.target_path) / f"{self.filenum}.composite.json" - self.save_batch(filename, batch) - - def save_batch(self, filename, batch): + while self.recipe_sets: + next_set = self.recipe_sets.pop(0) + if len(self.current_batch) + len(next_set) > 500: + self.flush_batch() + self.current_batch.extend(next_set) + + def flush_batch(self): + self.filenum += 1 + filename = Path(self.target_path) / f"{self.filenum}.composite.json" + with open(filename, "w") as open_file, SalesforceCompositeAPIOutput( open_file ) as out: self.filenames.append(filename) - for tablename, row in batch: + print(len(self.current_batch)) + for tablename, row in self.current_batch: out.write_row(tablename, row) + + self.current_batch = [] + + +class Bundle(FileOutputStream): + def __init__(self, file, **kwargs): + super().__init__(file, **kwargs) + self.tempdir = TemporaryDirectory() + self.folder_os = Folder(self.tempdir.name) + + def write_row(self, tablename: str, row_with_references: T.Dict) -> None: + self.folder_os.write_row(tablename, row_with_references) + + def write_single_row(self, tablename: str, row: T.Dict) -> None: + assert 0, "Shouldn't be called. write_row should be called instead" + + def complete_recipe(self, *args): + self.folder_os.complete_recipe() + + def close(self): + self.folder_os.close() + data = self.organize_bundle() + self.write(json.dumps(data, indent=2)) + self.tempdir.cleanup() + return super().close() + + def organize_bundle(self): + files = Path(self.tempdir.name).glob("*.composite.json") + data = [file.read_text() for file in files] + assert data + return {"bundle_format": 1, "data": data} From 17c205b6d0b93787a0393e56d86953e7792f6f29 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 7 Jun 2022 12:19:03 -0700 Subject: [PATCH 06/15] WIP: LoadCompositeAPIData Demo Program --- .gitignore | 1 - cumulusci.yml | 1 + examples/salesforce/DemoJSONData.apex | 23 +++ .../default/classes/LoadCompositeAPIData.cls | 153 ++++++++++++++++++ .../classes/LoadCompositeAPIData.cls-meta.xml | 5 + .../classes/LoadSnowfakeryJSONData.cls | 9 ++ .../LoadSnowfakeryJSONData.cls-meta.xml | 5 + .../main/default/flows/Test.flow-meta.xml | 47 ++++++ sfdx-project.json | 12 ++ unpackaged/site-settings/package.xml | 8 + .../Github_Gists.remoteSite | 7 + 11 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 examples/salesforce/DemoJSONData.apex create mode 100644 force-app/main/default/classes/LoadCompositeAPIData.cls create mode 100644 force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml create mode 100644 force-app/main/default/classes/LoadSnowfakeryJSONData.cls create mode 100644 force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml create mode 100644 force-app/main/default/flows/Test.flow-meta.xml create mode 100644 sfdx-project.json create mode 100644 unpackaged/site-settings/package.xml create mode 100644 unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite diff --git a/.gitignore b/.gitignore index 828e9003..c5b85cb9 100644 --- a/.gitignore +++ b/.gitignore @@ -47,5 +47,4 @@ coverage.xml .cci .sfdx /src.orig -/src myvenv \ No newline at end of file diff --git a/cumulusci.yml b/cumulusci.yml index 4f9f07c2..ef842213 100644 --- a/cumulusci.yml +++ b/cumulusci.yml @@ -5,6 +5,7 @@ project: api_version: "50.0" dependencies: - github: https://github.com/SalesforceFoundation/NPSP + source_format: sfdx sources: npsp: diff --git a/examples/salesforce/DemoJSONData.apex b/examples/salesforce/DemoJSONData.apex new file mode 100644 index 00000000..b33896e6 --- /dev/null +++ b/examples/salesforce/DemoJSONData.apex @@ -0,0 +1,23 @@ +// LoadCompositeAPIData loader = new LoadCompositeAPIData(); + +// load a single Composite Graph JSON Payload +LoadCompositeAPIData.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); + +// load anoter single one slightly less efficiently +LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/ffa992a7218906ab0dcf160b5d755259/raw/f9d40587a2ba9b04275241723637ed571bd55617/Graph%2520Gist'); + +// load a set of 3 Composite Graph JSONs in a distributed set +LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json'); + +// load a single-file bundle of 3 Composite Graph JSONs +LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json'); + +System.debug('SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! '); +// experimentally, much more than 15 hits a cumulative +// maximum time allotted for callout error +// +// for (Integer i = 0; i < 15; i++) { +// loader.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); +// System.debug(i); +// } + diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls b/force-app/main/default/classes/LoadCompositeAPIData.cls new file mode 100644 index 00000000..c380738c --- /dev/null +++ b/force-app/main/default/classes/LoadCompositeAPIData.cls @@ -0,0 +1,153 @@ +// This code can be run in Anonymous Apex to load data from Github Gists +// sfdx force:apex:execute -f src/classes/LoadCompositeAPIData.cls -f ./examples/salesforce/LoadCompositeAPIData.apex -u Snowfakery__qa +// or +// cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa +// +// Or called from other Apex, like the LoadSnowfakeryJSONData which exposes a +// an Invocable endpoint + +public class LoadCompositeAPIData { + + // Load one of three JSON formats. + // + // 1. One with a top-level key called "tables" which links to other + // composite graph payload jsons, like this: + // + // https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Folder + // + // 2. One with a top-level key called "data" which embeds compsite graph + // payloads as strings. Like this: + // + // https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json' + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Bundle + // + // 3. One which is just a single composite graph payload like this: + // + // https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json + // + // Which is recognizable by its top-level "graphs" key. + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput + + public static void loadJsonSet(String set_url){ + String json_record_sets = downloadJSON(set_url); + Map data = (Map)Json.deserializeUntyped(json_record_sets); + List tables = (List)data.get('tables'); + if(tables != null){ + loadDistributedJsonSet(tables); + return; + } + + List graph_jsons = (List)data.get('data'); + if(graph_jsons != null){ + loadBundledJsonSet(graph_jsons); + return; + } + + List graphs = (List)data.get('graphs'); + if(graphs != null){ + loadRecords(json_record_sets); + return; + } + + } + + // optimized method for a single composite graph (<500 records) + // This method doesn't parse the JSON to see what's in it. + public static void loadSingleJsonGraphPayload(String url) { + System.debug('Loading JSON ' + url); + String json_records = downloadJSON(url); + loadRecords(json_records); + System.debug('Loaded JSON ' + url); + } + + public static void loadDistributedJsonSet(List tables){ + for(Object table_url: tables){ + Map url_obj = (Map) table_url; + String url = (String)url_obj.get('url'); + loadSingleJsonGraphPayload(url); + } + } + + public static void loadBundledJsonSet(List graph_jsons){ + for(Object graph_json: graph_jsons){ + loadRecords((String)graph_json); + } + } + + private static String downloadJSON(String url){ + HttpResponse response = makeHTTPCall('GET', url, null); + return response.getBody(); + } + + private static HttpResponse makeHTTPCall(String method, String url, String post_body){ + Http h = new Http(); + HttpRequest request = new HttpRequest(); + request.setEndpoint(url); + request.setMethod(method); + if(post_body != null){ + request.setHeader('Content-Type', 'application/json'); + request.setBody(post_body); + } + + request.setHeader('Authorization', 'OAuth ' + UserInfo.getSessionId()); + request.setTimeout(120000); + System.debug(url); + return h.send(request); + } + + private static void loadRecords(String json_records){ + String error = null; + String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; + HttpResponse response = makeHTTPCall('POST', graph_url, json_records); + String response_body = response.getBody(); + if(response.getStatusCode()!=200){ + error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; + }else{ + error = parseResponse(response_body); + } + + if(error!=null){ + System.debug('Error: ' + error); + // System.debug('DOWNLOADED Data'); + // System.debug(response_body); + CalloutException e = new CalloutException( error); + throw e; + } + } + + private static String parseResponse(String response) { + Map graph_parse = (Map)Json.deserializeUntyped(response); + return parseError(graph_parse); + } + + private static String parseError(Map graph_parse){ + String rc = null; + List graphs = (List)graph_parse.get('graphs'); + for(Object graph: graphs){ + Map graphobj = (Map) graph; + boolean success = (boolean)graphobj.get('isSuccessful'); + if(success) continue; + Map graphResponse = (Map)graphobj.get('graphResponse'); + List compositeResponse = (List)graphResponse.get('compositeResponse'); + for(Object single_response: compositeResponse){ + Map single_response_obj = (Map)single_response; + Integer status = (Integer)single_response_obj.get('httpStatusCode'); + if(status!=200 && status!=201){ + List body = (List)single_response_obj.get('body'); + Map body_obj = (Map)body[0]; + if(rc==null && (String)body_obj.get('errorCode')!='PROCESSING_HALTED') { + System.debug('Error: ' + body.toString()); + rc = body_obj.toString(); + break; + } + } + } + } + + return rc; + } +} \ No newline at end of file diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml b/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml new file mode 100644 index 00000000..40d67933 --- /dev/null +++ b/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml @@ -0,0 +1,5 @@ + + + 54.0 + Active + diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls new file mode 100644 index 00000000..54f506d5 --- /dev/null +++ b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls @@ -0,0 +1,9 @@ +public class LoadSnowfakeryJSONData { + @InvocableMethod(label='Load Snowfakery Data Bundle' + description='Load a Snowfakery data bundle file into an Org by URL (JSON Graph API format)') + public static void loadJsonSet(List json_urls){ + for(String json_url: json_urls){ + LoadCompositeAPIData.loadJsonSet(json_url); + } + } +} \ No newline at end of file diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml new file mode 100644 index 00000000..40d67933 --- /dev/null +++ b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml @@ -0,0 +1,5 @@ + + + 54.0 + Active + diff --git a/force-app/main/default/flows/Test.flow-meta.xml b/force-app/main/default/flows/Test.flow-meta.xml new file mode 100644 index 00000000..4225f6db --- /dev/null +++ b/force-app/main/default/flows/Test.flow-meta.xml @@ -0,0 +1,47 @@ + + + + Load_Data + + 176 + 158 + LoadSnowfakeryJSONData + apex + + json_urls + + https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json + + + + 54.0 + Test {!$Flow.CurrentDateTime} + + + BuilderType + + LightningFlowBuilder + + + + CanvasMode + + AUTO_LAYOUT_CANVAS + + + + OriginBuilderType + + LightningFlowBuilder + + + Flow + + 50 + 0 + + Load_Data + + + Draft + diff --git a/sfdx-project.json b/sfdx-project.json new file mode 100644 index 00000000..7ccd2382 --- /dev/null +++ b/sfdx-project.json @@ -0,0 +1,12 @@ +{ + "packageDirectories": [ + { + "path": "force-app", + "default": true + } + ], + "name": "Snowfakery", + "namespace": "", + "sfdcLoginUrl": "https://login.salesforce.com", + "sourceApiVersion": "54.0" +} diff --git a/unpackaged/site-settings/package.xml b/unpackaged/site-settings/package.xml new file mode 100644 index 00000000..c735167c --- /dev/null +++ b/unpackaged/site-settings/package.xml @@ -0,0 +1,8 @@ + + + + Github_Gists + RemoteSiteSetting + + 50.0 + \ No newline at end of file diff --git a/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite b/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite new file mode 100644 index 00000000..b3322a68 --- /dev/null +++ b/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite @@ -0,0 +1,7 @@ + + + Github Gists for loading Snowfakery Data Bundles + false + true + https://gist.githubusercontent.com + From bfabdc1e5b79d1a41b41beadbc88bd04c69ae628 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 7 Jun 2022 12:19:03 -0700 Subject: [PATCH 07/15] WIP: LoadCompositeAPIData Demo Program --- .gitignore | 1 - cumulusci.yml | 1 + examples/salesforce/DemoJSONData.apex | 23 +++ .../default/classes/LoadCompositeAPIData.cls | 153 ++++++++++++++++++ .../classes/LoadCompositeAPIData.cls-meta.xml | 5 + .../classes/LoadSnowfakeryJSONData.cls | 9 ++ .../LoadSnowfakeryJSONData.cls-meta.xml | 5 + .../main/default/flows/Test.flow-meta.xml | 47 ++++++ sfdx-project.json | 12 ++ unpackaged/site-settings/package.xml | 8 + .../Github_Gists.remoteSite | 7 + 11 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 examples/salesforce/DemoJSONData.apex create mode 100644 force-app/main/default/classes/LoadCompositeAPIData.cls create mode 100644 force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml create mode 100644 force-app/main/default/classes/LoadSnowfakeryJSONData.cls create mode 100644 force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml create mode 100644 force-app/main/default/flows/Test.flow-meta.xml create mode 100644 sfdx-project.json create mode 100644 unpackaged/site-settings/package.xml create mode 100644 unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite diff --git a/.gitignore b/.gitignore index 828e9003..c5b85cb9 100644 --- a/.gitignore +++ b/.gitignore @@ -47,5 +47,4 @@ coverage.xml .cci .sfdx /src.orig -/src myvenv \ No newline at end of file diff --git a/cumulusci.yml b/cumulusci.yml index 4f9f07c2..ef842213 100644 --- a/cumulusci.yml +++ b/cumulusci.yml @@ -5,6 +5,7 @@ project: api_version: "50.0" dependencies: - github: https://github.com/SalesforceFoundation/NPSP + source_format: sfdx sources: npsp: diff --git a/examples/salesforce/DemoJSONData.apex b/examples/salesforce/DemoJSONData.apex new file mode 100644 index 00000000..b33896e6 --- /dev/null +++ b/examples/salesforce/DemoJSONData.apex @@ -0,0 +1,23 @@ +// LoadCompositeAPIData loader = new LoadCompositeAPIData(); + +// load a single Composite Graph JSON Payload +LoadCompositeAPIData.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); + +// load anoter single one slightly less efficiently +LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/ffa992a7218906ab0dcf160b5d755259/raw/f9d40587a2ba9b04275241723637ed571bd55617/Graph%2520Gist'); + +// load a set of 3 Composite Graph JSONs in a distributed set +LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json'); + +// load a single-file bundle of 3 Composite Graph JSONs +LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json'); + +System.debug('SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! '); +// experimentally, much more than 15 hits a cumulative +// maximum time allotted for callout error +// +// for (Integer i = 0; i < 15; i++) { +// loader.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); +// System.debug(i); +// } + diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls b/force-app/main/default/classes/LoadCompositeAPIData.cls new file mode 100644 index 00000000..c380738c --- /dev/null +++ b/force-app/main/default/classes/LoadCompositeAPIData.cls @@ -0,0 +1,153 @@ +// This code can be run in Anonymous Apex to load data from Github Gists +// sfdx force:apex:execute -f src/classes/LoadCompositeAPIData.cls -f ./examples/salesforce/LoadCompositeAPIData.apex -u Snowfakery__qa +// or +// cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa +// +// Or called from other Apex, like the LoadSnowfakeryJSONData which exposes a +// an Invocable endpoint + +public class LoadCompositeAPIData { + + // Load one of three JSON formats. + // + // 1. One with a top-level key called "tables" which links to other + // composite graph payload jsons, like this: + // + // https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Folder + // + // 2. One with a top-level key called "data" which embeds compsite graph + // payloads as strings. Like this: + // + // https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json' + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Bundle + // + // 3. One which is just a single composite graph payload like this: + // + // https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json + // + // Which is recognizable by its top-level "graphs" key. + // + // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput + + public static void loadJsonSet(String set_url){ + String json_record_sets = downloadJSON(set_url); + Map data = (Map)Json.deserializeUntyped(json_record_sets); + List tables = (List)data.get('tables'); + if(tables != null){ + loadDistributedJsonSet(tables); + return; + } + + List graph_jsons = (List)data.get('data'); + if(graph_jsons != null){ + loadBundledJsonSet(graph_jsons); + return; + } + + List graphs = (List)data.get('graphs'); + if(graphs != null){ + loadRecords(json_record_sets); + return; + } + + } + + // optimized method for a single composite graph (<500 records) + // This method doesn't parse the JSON to see what's in it. + public static void loadSingleJsonGraphPayload(String url) { + System.debug('Loading JSON ' + url); + String json_records = downloadJSON(url); + loadRecords(json_records); + System.debug('Loaded JSON ' + url); + } + + public static void loadDistributedJsonSet(List tables){ + for(Object table_url: tables){ + Map url_obj = (Map) table_url; + String url = (String)url_obj.get('url'); + loadSingleJsonGraphPayload(url); + } + } + + public static void loadBundledJsonSet(List graph_jsons){ + for(Object graph_json: graph_jsons){ + loadRecords((String)graph_json); + } + } + + private static String downloadJSON(String url){ + HttpResponse response = makeHTTPCall('GET', url, null); + return response.getBody(); + } + + private static HttpResponse makeHTTPCall(String method, String url, String post_body){ + Http h = new Http(); + HttpRequest request = new HttpRequest(); + request.setEndpoint(url); + request.setMethod(method); + if(post_body != null){ + request.setHeader('Content-Type', 'application/json'); + request.setBody(post_body); + } + + request.setHeader('Authorization', 'OAuth ' + UserInfo.getSessionId()); + request.setTimeout(120000); + System.debug(url); + return h.send(request); + } + + private static void loadRecords(String json_records){ + String error = null; + String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; + HttpResponse response = makeHTTPCall('POST', graph_url, json_records); + String response_body = response.getBody(); + if(response.getStatusCode()!=200){ + error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; + }else{ + error = parseResponse(response_body); + } + + if(error!=null){ + System.debug('Error: ' + error); + // System.debug('DOWNLOADED Data'); + // System.debug(response_body); + CalloutException e = new CalloutException( error); + throw e; + } + } + + private static String parseResponse(String response) { + Map graph_parse = (Map)Json.deserializeUntyped(response); + return parseError(graph_parse); + } + + private static String parseError(Map graph_parse){ + String rc = null; + List graphs = (List)graph_parse.get('graphs'); + for(Object graph: graphs){ + Map graphobj = (Map) graph; + boolean success = (boolean)graphobj.get('isSuccessful'); + if(success) continue; + Map graphResponse = (Map)graphobj.get('graphResponse'); + List compositeResponse = (List)graphResponse.get('compositeResponse'); + for(Object single_response: compositeResponse){ + Map single_response_obj = (Map)single_response; + Integer status = (Integer)single_response_obj.get('httpStatusCode'); + if(status!=200 && status!=201){ + List body = (List)single_response_obj.get('body'); + Map body_obj = (Map)body[0]; + if(rc==null && (String)body_obj.get('errorCode')!='PROCESSING_HALTED') { + System.debug('Error: ' + body.toString()); + rc = body_obj.toString(); + break; + } + } + } + } + + return rc; + } +} \ No newline at end of file diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml b/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml new file mode 100644 index 00000000..40d67933 --- /dev/null +++ b/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml @@ -0,0 +1,5 @@ + + + 54.0 + Active + diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls new file mode 100644 index 00000000..54f506d5 --- /dev/null +++ b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls @@ -0,0 +1,9 @@ +public class LoadSnowfakeryJSONData { + @InvocableMethod(label='Load Snowfakery Data Bundle' + description='Load a Snowfakery data bundle file into an Org by URL (JSON Graph API format)') + public static void loadJsonSet(List json_urls){ + for(String json_url: json_urls){ + LoadCompositeAPIData.loadJsonSet(json_url); + } + } +} \ No newline at end of file diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml new file mode 100644 index 00000000..40d67933 --- /dev/null +++ b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml @@ -0,0 +1,5 @@ + + + 54.0 + Active + diff --git a/force-app/main/default/flows/Test.flow-meta.xml b/force-app/main/default/flows/Test.flow-meta.xml new file mode 100644 index 00000000..4225f6db --- /dev/null +++ b/force-app/main/default/flows/Test.flow-meta.xml @@ -0,0 +1,47 @@ + + + + Load_Data + + 176 + 158 + LoadSnowfakeryJSONData + apex + + json_urls + + https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json + + + + 54.0 + Test {!$Flow.CurrentDateTime} + + + BuilderType + + LightningFlowBuilder + + + + CanvasMode + + AUTO_LAYOUT_CANVAS + + + + OriginBuilderType + + LightningFlowBuilder + + + Flow + + 50 + 0 + + Load_Data + + + Draft + diff --git a/sfdx-project.json b/sfdx-project.json new file mode 100644 index 00000000..7ccd2382 --- /dev/null +++ b/sfdx-project.json @@ -0,0 +1,12 @@ +{ + "packageDirectories": [ + { + "path": "force-app", + "default": true + } + ], + "name": "Snowfakery", + "namespace": "", + "sfdcLoginUrl": "https://login.salesforce.com", + "sourceApiVersion": "54.0" +} diff --git a/unpackaged/site-settings/package.xml b/unpackaged/site-settings/package.xml new file mode 100644 index 00000000..c735167c --- /dev/null +++ b/unpackaged/site-settings/package.xml @@ -0,0 +1,8 @@ + + + + Github_Gists + RemoteSiteSetting + + 50.0 + \ No newline at end of file diff --git a/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite b/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite new file mode 100644 index 00000000..b3322a68 --- /dev/null +++ b/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite @@ -0,0 +1,7 @@ + + + Github Gists for loading Snowfakery Data Bundles + false + true + https://gist.githubusercontent.com + From 70f4854200cdcc3d70b89acc5a4c45dc8ecf9fd1 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 7 Jun 2022 18:14:53 -0700 Subject: [PATCH 08/15] Move metadata --- .../force-app}/main/default/classes/LoadCompositeAPIData.cls | 0 .../main/default/classes/LoadCompositeAPIData.cls-meta.xml | 0 .../force-app}/main/default/classes/LoadSnowfakeryJSONData.cls | 0 .../main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml | 0 .../salesforce/force-app}/main/default/flows/Test.flow-meta.xml | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {force-app => examples/salesforce/force-app}/main/default/classes/LoadCompositeAPIData.cls (100%) rename {force-app => examples/salesforce/force-app}/main/default/classes/LoadCompositeAPIData.cls-meta.xml (100%) rename {force-app => examples/salesforce/force-app}/main/default/classes/LoadSnowfakeryJSONData.cls (100%) rename {force-app => examples/salesforce/force-app}/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml (100%) rename {force-app => examples/salesforce/force-app}/main/default/flows/Test.flow-meta.xml (100%) diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls b/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls similarity index 100% rename from force-app/main/default/classes/LoadCompositeAPIData.cls rename to examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml b/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml similarity index 100% rename from force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml rename to examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls b/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls similarity index 100% rename from force-app/main/default/classes/LoadSnowfakeryJSONData.cls rename to examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml b/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml similarity index 100% rename from force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml rename to examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml diff --git a/force-app/main/default/flows/Test.flow-meta.xml b/examples/salesforce/force-app/main/default/flows/Test.flow-meta.xml similarity index 100% rename from force-app/main/default/flows/Test.flow-meta.xml rename to examples/salesforce/force-app/main/default/flows/Test.flow-meta.xml From 69cdccf936218499238e718d5abc2db8b67a62bf Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 7 Jun 2022 18:15:57 -0700 Subject: [PATCH 09/15] Tests --- .../SalesforceCompositeAPIOutput.py | 14 +++++++---- tests/test_SalesforceCompositeAPIOutput.py | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 tests/test_SalesforceCompositeAPIOutput.py diff --git a/snowfakery/experimental/SalesforceCompositeAPIOutput.py b/snowfakery/experimental/SalesforceCompositeAPIOutput.py index 62da3d92..3e408b93 100644 --- a/snowfakery/experimental/SalesforceCompositeAPIOutput.py +++ b/snowfakery/experimental/SalesforceCompositeAPIOutput.py @@ -24,6 +24,8 @@ from snowfakery.output_streams import FileOutputStream, OutputStream +MAX_BATCH_SIZE = 500 # https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_composite_graph_limits.htm + class SalesforceCompositeAPIOutput(FileOutputStream): """Output stream that generates records for Salesforce's Composite API""" @@ -77,7 +79,7 @@ def __init__(self, output_folder, **kwargs): super().__init__(None, **kwargs) self.target_path = Path(output_folder) if not Path.exists(self.target_path): - Path.mkdir(self.target_path, exist_ok=True) + Path.mkdir(self.target_path, exist_ok=True) # pragma: no cover self.recipe_sets = [[]] self.current_batch = [] self.filenum = 0 @@ -87,7 +89,9 @@ def write_row(self, tablename: str, row_with_references: T.Dict) -> None: self.recipe_sets[-1].append((tablename, row_with_references)) def write_single_row(self, tablename: str, row: T.Dict) -> None: - assert 0, "Shouldn't be called. write_row should be called instead" + raise NotImplementedError( + "Shouldn't be called. write_row should be called instead" + ) def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: self.flush_sets() @@ -109,7 +113,7 @@ def complete_recipe(self, *args): def flush_sets(self): while self.recipe_sets: next_set = self.recipe_sets.pop(0) - if len(self.current_batch) + len(next_set) > 500: + if len(self.current_batch) + len(next_set) > MAX_BATCH_SIZE: self.flush_batch() self.current_batch.extend(next_set) @@ -138,7 +142,9 @@ def write_row(self, tablename: str, row_with_references: T.Dict) -> None: self.folder_os.write_row(tablename, row_with_references) def write_single_row(self, tablename: str, row: T.Dict) -> None: - assert 0, "Shouldn't be called. write_row should be called instead" + raise NotImplementedError( + "Shouldn't be called. write_row should be called instead" + ) def complete_recipe(self, *args): self.folder_os.complete_recipe() diff --git a/tests/test_SalesforceCompositeAPIOutput.py b/tests/test_SalesforceCompositeAPIOutput.py new file mode 100644 index 00000000..cc45506b --- /dev/null +++ b/tests/test_SalesforceCompositeAPIOutput.py @@ -0,0 +1,24 @@ +from io import StringIO +from unittest.mock import patch + +from snowfakery.data_generator import generate +from snowfakery.data_generator_runtime import StoppingCriteria +from snowfakery.experimental.SalesforceCompositeAPIOutput import Bundle +import json + +## Fill this out when it isn't experimental anymore + + +class TestSalesforceCompositeAPIOutput: + @patch("snowfakery.experimental.SalesforceCompositeAPIOutput.MAX_BATCH_SIZE", 5) + def test_composite(self): + out = StringIO() + output_stream = Bundle(out) + with open("examples/basic-salesforce.recipe.yml") as f: + generate( + f, {}, output_stream, stopping_criteria=StoppingCriteria("Account", 15) + ) + output_stream.close() + data = json.loads(out.getvalue()) + assert data["bundle_format"] == 1 + assert len(data["data"]) == 10 From b45c475f40ecd00f813935fbe9ab7b5fc9526617 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 14 Jun 2022 10:03:56 -0700 Subject: [PATCH 10/15] Cleanups --- .../default/classes/LoadCompositeAPIData.cls | 153 ------------------ .../classes/LoadCompositeAPIData.cls-meta.xml | 5 - .../classes/LoadSnowfakeryJSONData.cls | 9 -- .../LoadSnowfakeryJSONData.cls-meta.xml | 5 - .../main/default/flows/Test.flow-meta.xml | 47 ------ snowfakery/api.py | 2 + ...forceCompositeAPIOutput.py => DataPack.py} | 62 +++++-- tests/test_DataPack.py | 24 +++ tests/upsert-2.yml | 139 ++++++++++++++++ 9 files changed, 216 insertions(+), 230 deletions(-) delete mode 100644 force-app/main/default/classes/LoadCompositeAPIData.cls delete mode 100644 force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml delete mode 100644 force-app/main/default/classes/LoadSnowfakeryJSONData.cls delete mode 100644 force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml delete mode 100644 force-app/main/default/flows/Test.flow-meta.xml rename snowfakery/experimental/{SalesforceCompositeAPIOutput.py => DataPack.py} (71%) create mode 100644 tests/test_DataPack.py diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls b/force-app/main/default/classes/LoadCompositeAPIData.cls deleted file mode 100644 index c380738c..00000000 --- a/force-app/main/default/classes/LoadCompositeAPIData.cls +++ /dev/null @@ -1,153 +0,0 @@ -// This code can be run in Anonymous Apex to load data from Github Gists -// sfdx force:apex:execute -f src/classes/LoadCompositeAPIData.cls -f ./examples/salesforce/LoadCompositeAPIData.apex -u Snowfakery__qa -// or -// cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa -// -// Or called from other Apex, like the LoadSnowfakeryJSONData which exposes a -// an Invocable endpoint - -public class LoadCompositeAPIData { - - // Load one of three JSON formats. - // - // 1. One with a top-level key called "tables" which links to other - // composite graph payload jsons, like this: - // - // https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Folder - // - // 2. One with a top-level key called "data" which embeds compsite graph - // payloads as strings. Like this: - // - // https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json' - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Bundle - // - // 3. One which is just a single composite graph payload like this: - // - // https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json - // - // Which is recognizable by its top-level "graphs" key. - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput - - public static void loadJsonSet(String set_url){ - String json_record_sets = downloadJSON(set_url); - Map data = (Map)Json.deserializeUntyped(json_record_sets); - List tables = (List)data.get('tables'); - if(tables != null){ - loadDistributedJsonSet(tables); - return; - } - - List graph_jsons = (List)data.get('data'); - if(graph_jsons != null){ - loadBundledJsonSet(graph_jsons); - return; - } - - List graphs = (List)data.get('graphs'); - if(graphs != null){ - loadRecords(json_record_sets); - return; - } - - } - - // optimized method for a single composite graph (<500 records) - // This method doesn't parse the JSON to see what's in it. - public static void loadSingleJsonGraphPayload(String url) { - System.debug('Loading JSON ' + url); - String json_records = downloadJSON(url); - loadRecords(json_records); - System.debug('Loaded JSON ' + url); - } - - public static void loadDistributedJsonSet(List tables){ - for(Object table_url: tables){ - Map url_obj = (Map) table_url; - String url = (String)url_obj.get('url'); - loadSingleJsonGraphPayload(url); - } - } - - public static void loadBundledJsonSet(List graph_jsons){ - for(Object graph_json: graph_jsons){ - loadRecords((String)graph_json); - } - } - - private static String downloadJSON(String url){ - HttpResponse response = makeHTTPCall('GET', url, null); - return response.getBody(); - } - - private static HttpResponse makeHTTPCall(String method, String url, String post_body){ - Http h = new Http(); - HttpRequest request = new HttpRequest(); - request.setEndpoint(url); - request.setMethod(method); - if(post_body != null){ - request.setHeader('Content-Type', 'application/json'); - request.setBody(post_body); - } - - request.setHeader('Authorization', 'OAuth ' + UserInfo.getSessionId()); - request.setTimeout(120000); - System.debug(url); - return h.send(request); - } - - private static void loadRecords(String json_records){ - String error = null; - String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; - HttpResponse response = makeHTTPCall('POST', graph_url, json_records); - String response_body = response.getBody(); - if(response.getStatusCode()!=200){ - error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; - }else{ - error = parseResponse(response_body); - } - - if(error!=null){ - System.debug('Error: ' + error); - // System.debug('DOWNLOADED Data'); - // System.debug(response_body); - CalloutException e = new CalloutException( error); - throw e; - } - } - - private static String parseResponse(String response) { - Map graph_parse = (Map)Json.deserializeUntyped(response); - return parseError(graph_parse); - } - - private static String parseError(Map graph_parse){ - String rc = null; - List graphs = (List)graph_parse.get('graphs'); - for(Object graph: graphs){ - Map graphobj = (Map) graph; - boolean success = (boolean)graphobj.get('isSuccessful'); - if(success) continue; - Map graphResponse = (Map)graphobj.get('graphResponse'); - List compositeResponse = (List)graphResponse.get('compositeResponse'); - for(Object single_response: compositeResponse){ - Map single_response_obj = (Map)single_response; - Integer status = (Integer)single_response_obj.get('httpStatusCode'); - if(status!=200 && status!=201){ - List body = (List)single_response_obj.get('body'); - Map body_obj = (Map)body[0]; - if(rc==null && (String)body_obj.get('errorCode')!='PROCESSING_HALTED') { - System.debug('Error: ' + body.toString()); - rc = body_obj.toString(); - break; - } - } - } - } - - return rc; - } -} \ No newline at end of file diff --git a/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml b/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml deleted file mode 100644 index 40d67933..00000000 --- a/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - 54.0 - Active - diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls deleted file mode 100644 index 54f506d5..00000000 --- a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls +++ /dev/null @@ -1,9 +0,0 @@ -public class LoadSnowfakeryJSONData { - @InvocableMethod(label='Load Snowfakery Data Bundle' - description='Load a Snowfakery data bundle file into an Org by URL (JSON Graph API format)') - public static void loadJsonSet(List json_urls){ - for(String json_url: json_urls){ - LoadCompositeAPIData.loadJsonSet(json_url); - } - } -} \ No newline at end of file diff --git a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml b/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml deleted file mode 100644 index 40d67933..00000000 --- a/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - 54.0 - Active - diff --git a/force-app/main/default/flows/Test.flow-meta.xml b/force-app/main/default/flows/Test.flow-meta.xml deleted file mode 100644 index 4225f6db..00000000 --- a/force-app/main/default/flows/Test.flow-meta.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - Load_Data - - 176 - 158 - LoadSnowfakeryJSONData - apex - - json_urls - - https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json - - - - 54.0 - Test {!$Flow.CurrentDateTime} - - - BuilderType - - LightningFlowBuilder - - - - CanvasMode - - AUTO_LAYOUT_CANVAS - - - - OriginBuilderType - - LightningFlowBuilder - - - Flow - - 50 - 0 - - Load_Data - - - Draft - diff --git a/snowfakery/api.py b/snowfakery/api.py index 20c579d5..a2946f9d 100644 --- a/snowfakery/api.py +++ b/snowfakery/api.py @@ -32,6 +32,8 @@ "jpg": "snowfakery.output_streams.ImageOutputStream", "ps": "snowfakery.output_streams.ImageOutputStream", "dot": "snowfakery.output_streams.GraphvizOutputStream", + "datapack": "snowfakery.experimental.DataPack", + "apex": "snowfakery.experimental.DataPack.ApexDataPack", "json": "snowfakery.output_streams.JSONOutputStream", "txt": "snowfakery.output_streams.DebugOutputStream", "csv": "snowfakery.output_streams.CSVOutputStream", diff --git a/snowfakery/experimental/SalesforceCompositeAPIOutput.py b/snowfakery/experimental/DataPack.py similarity index 71% rename from snowfakery/experimental/SalesforceCompositeAPIOutput.py rename to snowfakery/experimental/DataPack.py index 3e408b93..8d43d909 100644 --- a/snowfakery/experimental/SalesforceCompositeAPIOutput.py +++ b/snowfakery/experimental/DataPack.py @@ -1,6 +1,6 @@ # Use this experimental OutputStream like this: -# snowfakery --output-format snowfakery.experimental.SalesforceCompositeAPIOutput recipe.yml > composite.json +# snowfakery --output-format snowfakery.experimental.DataPack recipe.yml > composite.json # # Once you have the file you can make it accessible to Salesforce by uploading it # to some form of server. E.g. Github gist, Heroku, etc. @@ -17,6 +17,8 @@ # TODO: Add tests import json +from logging import warning +from io import StringIO import typing as T import datetime from pathlib import Path @@ -46,10 +48,18 @@ def __init__(self, file, **kwargs): def write_single_row(self, tablename: str, row: T.Dict) -> None: row_without_id = row.copy() del row_without_id["id"] + _sf_update_key = row_without_id.pop("_sf_update_key", None) + if _sf_update_key: + method = "PATCH" + url = f"/services/data/v50.0/sobjects/{tablename}/{_sf_update_key}/{row_without_id[_sf_update_key]}" + else: + method = "POST" + url = f"/services/data/v50.0/sobjects/{tablename}/" + values = { - "method": "POST", + "method": method, "referenceId": f"{tablename}_{row['id']}", - "url": f"/services/data/v50.0/sobjects/{tablename}", + "url": url, "body": row_without_id, } self.rows.append(values) @@ -62,7 +72,7 @@ def flatten( target_object_row, ) -> T.Union[str, int]: target_reference = f"{target_object_row._tablename}_{target_object_row.id}" - return "@{%s.id}" % target_reference + return "@{%s}" % target_reference def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: # NOTE: Could improve loading performance by breaking graphs up @@ -85,10 +95,12 @@ def __init__(self, output_folder, **kwargs): self.filenum = 0 self.filenames = [] - def write_row(self, tablename: str, row_with_references: T.Dict) -> None: + def write_row( + self, tablename: str, row_with_references: T.Dict, *args, **kwargs + ) -> None: self.recipe_sets[-1].append((tablename, row_with_references)) - def write_single_row(self, tablename: str, row: T.Dict) -> None: + def write_single_row(self, tablename: str, row: T.Dict, *args, **kwargs) -> None: raise NotImplementedError( "Shouldn't be called. write_row should be called instead" ) @@ -125,23 +137,25 @@ def flush_batch(self): open_file ) as out: self.filenames.append(filename) - print(len(self.current_batch)) for tablename, row in self.current_batch: out.write_row(tablename, row) self.current_batch = [] -class Bundle(FileOutputStream): +class DataPack(FileOutputStream): def __init__(self, file, **kwargs): super().__init__(file, **kwargs) + warning("DataPack is an experimental data format") self.tempdir = TemporaryDirectory() self.folder_os = Folder(self.tempdir.name) - def write_row(self, tablename: str, row_with_references: T.Dict) -> None: + def write_row( + self, tablename: str, row_with_references: T.Dict, *args, **kwargs + ) -> None: self.folder_os.write_row(tablename, row_with_references) - def write_single_row(self, tablename: str, row: T.Dict) -> None: + def write_single_row(self, tablename: str, row: T.Dict, *args, **kwargs) -> None: raise NotImplementedError( "Shouldn't be called. write_row should be called instead" ) @@ -160,4 +174,30 @@ def organize_bundle(self): files = Path(self.tempdir.name).glob("*.composite.json") data = [file.read_text() for file in files] assert data - return {"bundle_format": 1, "data": data} + return {"datapack_format": 1, "data": data} + + +class ApexDataPack(FileOutputStream): + def __init__(self, file, **kwargs): + super().__init__(file, **kwargs) + self.data = StringIO() + self.datapack = DataPack(self.data) + + def write_row( + self, tablename: str, row_with_references: T.Dict, *args, **kwargs + ) -> None: + self.datapack.write_row(tablename, row_with_references) + + def write_single_row(self, tablename: str, row: T.Dict, *args, **kwargs) -> None: + raise NotImplementedError( + "Shouldn't be called. write_row should be called instead" + ) + + def complete_recipe(self, *args): + self.datapack.complete_recipe() + + def close(self): + self.datapack.close() + quoted_data = repr(self.data.getvalue()) + self.write(f"String json_data = {quoted_data};\n") + self.write("LoadCompositeAPIData.loadBundledJsonSet(json_data);\n") diff --git a/tests/test_DataPack.py b/tests/test_DataPack.py new file mode 100644 index 00000000..ea452eb1 --- /dev/null +++ b/tests/test_DataPack.py @@ -0,0 +1,24 @@ +from io import StringIO +from unittest.mock import patch + +from snowfakery.data_generator import generate +from snowfakery.data_generator_runtime import StoppingCriteria +from snowfakery.experimental.SalesforceCompositeAPIOutput import DataPack +import json + +## Fill this out when it isn't experimental anymore + + +class TestSalesforceCompositeAPIOutput: + @patch("snowfakery.experimental.SalesforceCompositeAPIOutput.MAX_BATCH_SIZE", 5) + def test_composite(self): + out = StringIO() + output_stream = DataPack(out) + with open("examples/basic-salesforce.recipe.yml") as f: + generate( + f, {}, output_stream, stopping_criteria=StoppingCriteria("Account", 15) + ) + output_stream.close() + data = json.loads(out.getvalue()) + assert data["datapack_format"] == 1 + assert len(data["data"]) == 10 diff --git a/tests/upsert-2.yml b/tests/upsert-2.yml index bca7a6a8..c48d2adc 100644 --- a/tests/upsert-2.yml +++ b/tests/upsert-2.yml @@ -1,6 +1,145 @@ +# Accounts + +- object: Account + nickname: Bluth Company + fields: + Name: Bluth Company +- object: Account + nickname: Michael B Company + fields: + Name: Michael B. Company +- object: Account + nickname: Austero Bluth Company + fields: + Name: Austero Bluth Company +- object: Account + nickname: Sitwell Enterprises + fields: + Name: Sitwell Enterprises + +# Contacts + +- object: Contact + update_key: email + nickname: GOB + fields: + firstname: GOB + lastname: Bluth + email: George.Oscar@bluth.com + AccountId: + reference: Bluth Company - object: Contact update_key: email + nickname: Michael + fields: + firstname: Michael + lastname: Bluth + email: michael@bluth.com + AccountId: + reference: Michael B Company +- object: Contact + update_key: email + nickname: Lucille-2 + fields: + firstname: Lucille + lastname: Austero + email: Lucille.Austero@bluth.com + AccountId: + reference: Austero Bluth Company +- object: Contact + update_key: email + nickname: Lindsay + fields: + firstname: Lindsay + lastname: Bluth + email: lindsay@bluth.com + AccountId: + reference: Sitwell Enterprises +- object: Contact + update_key: email + nickname: anyong fields: firstname: Hel-Loh lastname: Bluth email: anyong@bluth.com +- object: Contact + update_key: email + nickname: george + fields: + firstname: George + lastname: Bluth + email: boss@bluth.com +- object: Contact + update_key: email + nickname: buster + fields: + firstname: Buster + lastname: Bluth + email: buster@bluth.com + +# Opportunities + +- object: Opportunity + update_key: Name + fields: + Name: Bluth Company Opp + AccountId: + reference: Bluth Company + # ContactId: + # reference: GOB + CloseDate: + date_between: + start_date: -1y + end_date: today + StageName: + random_choice: + - Prospecting + - Pledged +- object: Opportunity + update_key: Name + fields: + Name: Michael B Company Opp + AccountId: + reference: Michael B Company + # ContactId: + # reference: Michael + CloseDate: + date_between: + start_date: -1y + end_date: today + StageName: + random_choice: + - Prospecting + - Pledged +- object: Opportunity + update_key: Name + fields: + Name: Austero Bluth Company Opp + AccountId: + reference: Austero Bluth Company + # ContactId: + # reference: Lucille-2 + CloseDate: + date_between: + start_date: -1y + end_date: today + StageName: + random_choice: + - Prospecting + - Pledged +- object: Opportunity + update_key: Name + fields: + Name: Sitwell Enterprises Opp + AccountId: + reference: Sitwell Enterprises + # ContactId: + # reference: Lindsay + CloseDate: + date_between: + start_date: -1y + end_date: today + StageName: + random_choice: + - Prospecting + - Pledged From 6b1f4caf8d4b92165265379494c870b9baf7c334 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 14 Jun 2022 13:16:21 -0700 Subject: [PATCH 11/15] Fix tests --- snowfakery/experimental/DataPack.py | 7 +++++ tests/test_DataPack.py | 34 ++++++++++++++++++++-- tests/test_SalesforceCompositeAPIOutput.py | 24 --------------- 3 files changed, 38 insertions(+), 27 deletions(-) delete mode 100644 tests/test_SalesforceCompositeAPIOutput.py diff --git a/snowfakery/experimental/DataPack.py b/snowfakery/experimental/DataPack.py index 8d43d909..a98a8935 100644 --- a/snowfakery/experimental/DataPack.py +++ b/snowfakery/experimental/DataPack.py @@ -77,6 +77,7 @@ def flatten( def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: # NOTE: Could improve loading performance by breaking graphs up # to allow server-side parallelization, but I'd risk locking issues + assert self.rows data = {"graphs": [{"graphId": "graph", "compositeRequest": self.rows}]} self.write(json.dumps(data, indent=2)) return super().close() @@ -125,6 +126,7 @@ def complete_recipe(self, *args): def flush_sets(self): while self.recipe_sets: next_set = self.recipe_sets.pop(0) + assert len(next_set) <= MAX_BATCH_SIZE if len(self.current_batch) + len(next_set) > MAX_BATCH_SIZE: self.flush_batch() self.current_batch.extend(next_set) @@ -137,6 +139,7 @@ def flush_batch(self): open_file ) as out: self.filenames.append(filename) + assert self.current_batch for tablename, row in self.current_batch: out.write_row(tablename, row) @@ -178,6 +181,10 @@ def organize_bundle(self): class ApexDataPack(FileOutputStream): + """Wrap in Anon Apex but note that the amount of data you can load + this way is very limited due to limitations of the REST API (used by CCI) + and SOAP API (used by sfdx)""" + def __init__(self, file, **kwargs): super().__init__(file, **kwargs) self.data = StringIO() diff --git a/tests/test_DataPack.py b/tests/test_DataPack.py index ea452eb1..85787921 100644 --- a/tests/test_DataPack.py +++ b/tests/test_DataPack.py @@ -3,14 +3,14 @@ from snowfakery.data_generator import generate from snowfakery.data_generator_runtime import StoppingCriteria -from snowfakery.experimental.SalesforceCompositeAPIOutput import DataPack +from snowfakery.experimental.DataPack import DataPack, ApexDataPack import json ## Fill this out when it isn't experimental anymore class TestSalesforceCompositeAPIOutput: - @patch("snowfakery.experimental.SalesforceCompositeAPIOutput.MAX_BATCH_SIZE", 5) + @patch("snowfakery.experimental.DataPack.MAX_BATCH_SIZE", 10) def test_composite(self): out = StringIO() output_stream = DataPack(out) @@ -21,4 +21,32 @@ def test_composite(self): output_stream.close() data = json.loads(out.getvalue()) assert data["datapack_format"] == 1 - assert len(data["data"]) == 10 + assert len(data["data"]) == 8 + single_payload = json.loads(data["data"][0]) + print(single_payload) + assert single_payload["graphs"][0]["compositeRequest"][0]["method"] == "POST" + + @patch("snowfakery.experimental.DataPack.MAX_BATCH_SIZE", 50) + def test_composite_upsert(self): + out = StringIO() + output_stream = DataPack(out) + with open("tests/upsert-2.yml") as f: + generate( + f, {}, output_stream, stopping_criteria=StoppingCriteria("Account", 50) + ) + output_stream.close() + data = json.loads(out.getvalue()) + assert data["datapack_format"] == 1 + single_payload = json.loads(data["data"][1]) + assert single_payload["graphs"][0]["compositeRequest"][-1]["method"] == "PATCH" + + def test_apex(self): + out = StringIO() + output_stream = ApexDataPack(out) + with open("examples/basic-salesforce.recipe.yml") as f: + generate( + f, {}, output_stream, stopping_criteria=StoppingCriteria("Account", 50) + ) + output_stream.close() + out = out.getvalue() + assert out.startswith("String json_data") diff --git a/tests/test_SalesforceCompositeAPIOutput.py b/tests/test_SalesforceCompositeAPIOutput.py deleted file mode 100644 index cc45506b..00000000 --- a/tests/test_SalesforceCompositeAPIOutput.py +++ /dev/null @@ -1,24 +0,0 @@ -from io import StringIO -from unittest.mock import patch - -from snowfakery.data_generator import generate -from snowfakery.data_generator_runtime import StoppingCriteria -from snowfakery.experimental.SalesforceCompositeAPIOutput import Bundle -import json - -## Fill this out when it isn't experimental anymore - - -class TestSalesforceCompositeAPIOutput: - @patch("snowfakery.experimental.SalesforceCompositeAPIOutput.MAX_BATCH_SIZE", 5) - def test_composite(self): - out = StringIO() - output_stream = Bundle(out) - with open("examples/basic-salesforce.recipe.yml") as f: - generate( - f, {}, output_stream, stopping_criteria=StoppingCriteria("Account", 15) - ) - output_stream.close() - data = json.loads(out.getvalue()) - assert data["bundle_format"] == 1 - assert len(data["data"]) == 10 From 122af712ac18cef2d66b9279cb6aeb3493443999 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 14 Jun 2022 14:23:54 -0700 Subject: [PATCH 12/15] Delete files that moved to other repo --- examples/salesforce/DemoJSONData.apex | 23 --- examples/salesforce/LoadCompositeAPIData.apex | 174 ------------------ .../default/classes/LoadCompositeAPIData.cls | 153 --------------- .../classes/LoadCompositeAPIData.cls-meta.xml | 5 - .../classes/LoadSnowfakeryJSONData.cls | 9 - .../LoadSnowfakeryJSONData.cls-meta.xml | 5 - .../main/default/flows/Test.flow-meta.xml | 47 ----- sfdx-project.json | 12 -- unpackaged/site-settings/package.xml | 8 - .../Github_Gists.remoteSite | 7 - 10 files changed, 443 deletions(-) delete mode 100644 examples/salesforce/DemoJSONData.apex delete mode 100644 examples/salesforce/LoadCompositeAPIData.apex delete mode 100644 examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls delete mode 100644 examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml delete mode 100644 examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls delete mode 100644 examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml delete mode 100644 examples/salesforce/force-app/main/default/flows/Test.flow-meta.xml delete mode 100644 sfdx-project.json delete mode 100644 unpackaged/site-settings/package.xml delete mode 100644 unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite diff --git a/examples/salesforce/DemoJSONData.apex b/examples/salesforce/DemoJSONData.apex deleted file mode 100644 index b33896e6..00000000 --- a/examples/salesforce/DemoJSONData.apex +++ /dev/null @@ -1,23 +0,0 @@ -// LoadCompositeAPIData loader = new LoadCompositeAPIData(); - -// load a single Composite Graph JSON Payload -LoadCompositeAPIData.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); - -// load anoter single one slightly less efficiently -LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/ffa992a7218906ab0dcf160b5d755259/raw/f9d40587a2ba9b04275241723637ed571bd55617/Graph%2520Gist'); - -// load a set of 3 Composite Graph JSONs in a distributed set -LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json'); - -// load a single-file bundle of 3 Composite Graph JSONs -LoadCompositeAPIData.loadJsonSet('https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json'); - -System.debug('SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! '); -// experimentally, much more than 15 hits a cumulative -// maximum time allotted for callout error -// -// for (Integer i = 0; i < 15; i++) { -// loader.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); -// System.debug(i); -// } - diff --git a/examples/salesforce/LoadCompositeAPIData.apex b/examples/salesforce/LoadCompositeAPIData.apex deleted file mode 100644 index 8913169c..00000000 --- a/examples/salesforce/LoadCompositeAPIData.apex +++ /dev/null @@ -1,174 +0,0 @@ -// This code can be run in Anonymous Apex to load data from Github Gists -// sfdx force:apex:execute -f ./examples/salesforce/LoadCompositeAPIData.apex -u Snowfakery__qa -// or -// cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa - - -public class LoadCompositeAPIData { - - // Load one of three JSON formats. - // - // 1. One with a top-level key called "tables" which links to other - // composite graph payload jsons, like this: - // - // https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Folder - // - // 2. One with a top-level key called "data" which embeds compsite graph - // payloads as strings. Like this: - // - // https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json' - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Bundle - // - // 3. One which is just a single composite graph payload like this: - // - // https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json - // - // Which is recognizable by its top-level "graphs" key. - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput - - public void loadJsonSet(String set_url){ - String json_record_sets = downloadJSON(set_url); - Map data = (Map)Json.deserializeUntyped(json_record_sets); - List tables = (List)data.get('tables'); - if(tables != null){ - loadDistributedJsonSet(tables); - return; - } - - List graph_jsons = (List)data.get('data'); - if(graph_jsons != null){ - loadBundledJsonSet(graph_jsons); - return; - } - - List graphs = (List)data.get('graphs'); - if(graphs != null){ - loadRecords(json_record_sets); - return; - } - - } - - // optimized method for a single composite graph (<500 records) - // This method doesn't parse the JSON to see what's in it. - public void loadSingleJsonGraphPayload(String url) { - System.debug('Loading JSON ' + url); - String json_records = downloadJSON(url); - loadRecords(json_records); - System.debug('Loaded JSON ' + url); - } - - public void loadDistributedJsonSet(List tables){ - for(Object table_url: tables){ - Map url_obj = (Map) table_url; - String url = (String)url_obj.get('url'); - loadSingleJsonGraphPayload(url); - } - } - - public void loadBundledJsonSet(List graph_jsons){ - for(Object graph_json: graph_jsons){ - loadRecords((String)graph_json); - } - } - - private String downloadJSON(String url){ - HttpResponse response = makeHTTPCall('GET', url, null); - return response.getBody(); - } - - private HttpResponse makeHTTPCall(String method, String url, String post_body){ - Http h = new Http(); - HttpRequest request = new HttpRequest(); - request.setEndpoint(url); - request.setMethod(method); - if(post_body != null){ - request.setHeader('Content-Type', 'application/json'); - request.setBody(post_body); - } - - request.setHeader('Authorization', 'OAuth ' + UserInfo.getSessionId()); - request.setTimeout(120000); - System.debug(url); - return h.send(request); - } - - private void loadRecords(String json_records){ - String error = null; - String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; - HttpResponse response = makeHTTPCall('POST', graph_url, json_records); - String response_body = response.getBody(); - if(response.getStatusCode()!=200){ - error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; - }else{ - error = parseResponse(response_body); - } - - if(error!=null){ - System.debug('Error: ' + error); - // System.debug('DOWNLOADED Data'); - // System.debug(response_body); - CalloutException e = new CalloutException( error); - throw e; - } - } - - private String parseResponse(String response) { - Map graph_parse = (Map)Json.deserializeUntyped(response); - return parseError(graph_parse); - } - - private String parseError(Map graph_parse){ - String rc = null; - List graphs = (List)graph_parse.get('graphs'); - for(Object graph: graphs){ - Map graphobj = (Map) graph; - boolean success = (boolean)graphobj.get('isSuccessful'); - if(success) continue; - Map graphResponse = (Map)graphobj.get('graphResponse'); - List compositeResponse = (List)graphResponse.get('compositeResponse'); - for(Object single_response: compositeResponse){ - Map single_response_obj = (Map)single_response; - Integer status = (Integer)single_response_obj.get('httpStatusCode'); - if(status!=200 && status!=201){ - List body = (List)single_response_obj.get('body'); - Map body_obj = (Map)body[0]; - if(rc==null && (String)body_obj.get('errorCode')!='PROCESSING_HALTED') { - System.debug('Error: ' + body.toString()); - rc = body_obj.toString(); - break; - } - } - } - } - - return rc; - } -} - -LoadCompositeAPIData loader = new LoadCompositeAPIData(); - -// load a single Composite Graph JSON Payload -loader.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); - -// load anoter single one slightly less efficiently -loader.loadJsonSet('https://gist.githubusercontent.com/prescod/ffa992a7218906ab0dcf160b5d755259/raw/f9d40587a2ba9b04275241723637ed571bd55617/Graph%2520Gist'); - -// load a set of 3 Composite Graph JSONs in a distributed set -loader.loadJsonSet('https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json'); - -// load a single-file bundle of 3 Composite Graph JSONs -loader.loadJsonSet('https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json'); - -System.debug('SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! SUCCESS! '); -// experimentally, much more than 15 hits a cumulative -// maximum time allotted for callout error -// -// for (Integer i = 0; i < 15; i++) { -// loader.loadSingleJsonGraphPayload('https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json'); -// System.debug(i); -// } diff --git a/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls b/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls deleted file mode 100644 index c380738c..00000000 --- a/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls +++ /dev/null @@ -1,153 +0,0 @@ -// This code can be run in Anonymous Apex to load data from Github Gists -// sfdx force:apex:execute -f src/classes/LoadCompositeAPIData.cls -f ./examples/salesforce/LoadCompositeAPIData.apex -u Snowfakery__qa -// or -// cci task run execute_anon --path examples/salesforce/LoadCompositeAPIData.apex --org qa -// -// Or called from other Apex, like the LoadSnowfakeryJSONData which exposes a -// an Invocable endpoint - -public class LoadCompositeAPIData { - - // Load one of three JSON formats. - // - // 1. One with a top-level key called "tables" which links to other - // composite graph payload jsons, like this: - // - // https://gist.githubusercontent.com/prescod/6f3aebafde63971d1093549c3bef4e41/raw/8885df2618ece474c196d5d94b594dd2b5961c71/csvw_metadata.json - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Folder - // - // 2. One with a top-level key called "data" which embeds compsite graph - // payloads as strings. Like this: - // - // https://gist.githubusercontent.com/prescod/6220fa27d8493be954be949c9f57f2b2/raw/b603a4d11ef20f0a30e79260322baa52f969068d/out.bundle.json' - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput.Bundle - // - // 3. One which is just a single composite graph payload like this: - // - // https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json - // - // Which is recognizable by its top-level "graphs" key. - // - // Create the files with snowfakery.experimental.SalesforceCompositeAPIOutput - - public static void loadJsonSet(String set_url){ - String json_record_sets = downloadJSON(set_url); - Map data = (Map)Json.deserializeUntyped(json_record_sets); - List tables = (List)data.get('tables'); - if(tables != null){ - loadDistributedJsonSet(tables); - return; - } - - List graph_jsons = (List)data.get('data'); - if(graph_jsons != null){ - loadBundledJsonSet(graph_jsons); - return; - } - - List graphs = (List)data.get('graphs'); - if(graphs != null){ - loadRecords(json_record_sets); - return; - } - - } - - // optimized method for a single composite graph (<500 records) - // This method doesn't parse the JSON to see what's in it. - public static void loadSingleJsonGraphPayload(String url) { - System.debug('Loading JSON ' + url); - String json_records = downloadJSON(url); - loadRecords(json_records); - System.debug('Loaded JSON ' + url); - } - - public static void loadDistributedJsonSet(List tables){ - for(Object table_url: tables){ - Map url_obj = (Map) table_url; - String url = (String)url_obj.get('url'); - loadSingleJsonGraphPayload(url); - } - } - - public static void loadBundledJsonSet(List graph_jsons){ - for(Object graph_json: graph_jsons){ - loadRecords((String)graph_json); - } - } - - private static String downloadJSON(String url){ - HttpResponse response = makeHTTPCall('GET', url, null); - return response.getBody(); - } - - private static HttpResponse makeHTTPCall(String method, String url, String post_body){ - Http h = new Http(); - HttpRequest request = new HttpRequest(); - request.setEndpoint(url); - request.setMethod(method); - if(post_body != null){ - request.setHeader('Content-Type', 'application/json'); - request.setBody(post_body); - } - - request.setHeader('Authorization', 'OAuth ' + UserInfo.getSessionId()); - request.setTimeout(120000); - System.debug(url); - return h.send(request); - } - - private static void loadRecords(String json_records){ - String error = null; - String graph_url = System.URL.getSalesforceBaseUrl().toExternalForm() + '/services/data/v54.0/composite/graph'; - HttpResponse response = makeHTTPCall('POST', graph_url, json_records); - String response_body = response.getBody(); - if(response.getStatusCode()!=200){ - error = 'Error creating objects! ' + response.getStatus() + ' ' + response_body; - }else{ - error = parseResponse(response_body); - } - - if(error!=null){ - System.debug('Error: ' + error); - // System.debug('DOWNLOADED Data'); - // System.debug(response_body); - CalloutException e = new CalloutException( error); - throw e; - } - } - - private static String parseResponse(String response) { - Map graph_parse = (Map)Json.deserializeUntyped(response); - return parseError(graph_parse); - } - - private static String parseError(Map graph_parse){ - String rc = null; - List graphs = (List)graph_parse.get('graphs'); - for(Object graph: graphs){ - Map graphobj = (Map) graph; - boolean success = (boolean)graphobj.get('isSuccessful'); - if(success) continue; - Map graphResponse = (Map)graphobj.get('graphResponse'); - List compositeResponse = (List)graphResponse.get('compositeResponse'); - for(Object single_response: compositeResponse){ - Map single_response_obj = (Map)single_response; - Integer status = (Integer)single_response_obj.get('httpStatusCode'); - if(status!=200 && status!=201){ - List body = (List)single_response_obj.get('body'); - Map body_obj = (Map)body[0]; - if(rc==null && (String)body_obj.get('errorCode')!='PROCESSING_HALTED') { - System.debug('Error: ' + body.toString()); - rc = body_obj.toString(); - break; - } - } - } - } - - return rc; - } -} \ No newline at end of file diff --git a/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml b/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml deleted file mode 100644 index 40d67933..00000000 --- a/examples/salesforce/force-app/main/default/classes/LoadCompositeAPIData.cls-meta.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - 54.0 - Active - diff --git a/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls b/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls deleted file mode 100644 index 54f506d5..00000000 --- a/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls +++ /dev/null @@ -1,9 +0,0 @@ -public class LoadSnowfakeryJSONData { - @InvocableMethod(label='Load Snowfakery Data Bundle' - description='Load a Snowfakery data bundle file into an Org by URL (JSON Graph API format)') - public static void loadJsonSet(List json_urls){ - for(String json_url: json_urls){ - LoadCompositeAPIData.loadJsonSet(json_url); - } - } -} \ No newline at end of file diff --git a/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml b/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml deleted file mode 100644 index 40d67933..00000000 --- a/examples/salesforce/force-app/main/default/classes/LoadSnowfakeryJSONData.cls-meta.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - 54.0 - Active - diff --git a/examples/salesforce/force-app/main/default/flows/Test.flow-meta.xml b/examples/salesforce/force-app/main/default/flows/Test.flow-meta.xml deleted file mode 100644 index 4225f6db..00000000 --- a/examples/salesforce/force-app/main/default/flows/Test.flow-meta.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - Load_Data - - 176 - 158 - LoadSnowfakeryJSONData - apex - - json_urls - - https://gist.githubusercontent.com/prescod/13302ecbd08fc3fe92db7d6ee4614d25/raw/c88949d2170c7c11f94958ec672ec8b972cc10d4/composite.json - - - - 54.0 - Test {!$Flow.CurrentDateTime} - - - BuilderType - - LightningFlowBuilder - - - - CanvasMode - - AUTO_LAYOUT_CANVAS - - - - OriginBuilderType - - LightningFlowBuilder - - - Flow - - 50 - 0 - - Load_Data - - - Draft - diff --git a/sfdx-project.json b/sfdx-project.json deleted file mode 100644 index 7ccd2382..00000000 --- a/sfdx-project.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "packageDirectories": [ - { - "path": "force-app", - "default": true - } - ], - "name": "Snowfakery", - "namespace": "", - "sfdcLoginUrl": "https://login.salesforce.com", - "sourceApiVersion": "54.0" -} diff --git a/unpackaged/site-settings/package.xml b/unpackaged/site-settings/package.xml deleted file mode 100644 index c735167c..00000000 --- a/unpackaged/site-settings/package.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - Github_Gists - RemoteSiteSetting - - 50.0 - \ No newline at end of file diff --git a/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite b/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite deleted file mode 100644 index b3322a68..00000000 --- a/unpackaged/site-settings/remoteSiteSettings/Github_Gists.remoteSite +++ /dev/null @@ -1,7 +0,0 @@ - - - Github Gists for loading Snowfakery Data Bundles - false - true - https://gist.githubusercontent.com - From 99dca32ea32c96f987b350053ab53e28498c1d51 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 14 Jun 2022 14:24:25 -0700 Subject: [PATCH 13/15] Delete files that moved to other repo --- cumulusci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/cumulusci.yml b/cumulusci.yml index ef842213..4f9f07c2 100644 --- a/cumulusci.yml +++ b/cumulusci.yml @@ -5,7 +5,6 @@ project: api_version: "50.0" dependencies: - github: https://github.com/SalesforceFoundation/NPSP - source_format: sfdx sources: npsp: From 481e15761bf1d3d664dc9ae11463d205020b68f5 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Tue, 14 Jun 2022 16:41:13 -0700 Subject: [PATCH 14/15] Fix referencing bug --- snowfakery/experimental/DataPack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snowfakery/experimental/DataPack.py b/snowfakery/experimental/DataPack.py index a98a8935..75fc8616 100644 --- a/snowfakery/experimental/DataPack.py +++ b/snowfakery/experimental/DataPack.py @@ -72,7 +72,7 @@ def flatten( target_object_row, ) -> T.Union[str, int]: target_reference = f"{target_object_row._tablename}_{target_object_row.id}" - return "@{%s}" % target_reference + return "@{%s.i}" % target_reference def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: # NOTE: Could improve loading performance by breaking graphs up From f94b60b7b765e426d5e585be09405800f9b97919 Mon Sep 17 00:00:00 2001 From: Paul Prescod Date: Wed, 15 Jun 2022 14:30:10 -0700 Subject: [PATCH 15/15] Fix inter-object references --- snowfakery/experimental/DataPack.py | 2 +- tests/test_DataPack.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/snowfakery/experimental/DataPack.py b/snowfakery/experimental/DataPack.py index 75fc8616..8bd9f4c6 100644 --- a/snowfakery/experimental/DataPack.py +++ b/snowfakery/experimental/DataPack.py @@ -72,7 +72,7 @@ def flatten( target_object_row, ) -> T.Union[str, int]: target_reference = f"{target_object_row._tablename}_{target_object_row.id}" - return "@{%s.i}" % target_reference + return "@{%s.id}" % target_reference def close(self, **kwargs) -> T.Optional[T.Sequence[str]]: # NOTE: Could improve loading performance by breaking graphs up diff --git a/tests/test_DataPack.py b/tests/test_DataPack.py index 85787921..a306a720 100644 --- a/tests/test_DataPack.py +++ b/tests/test_DataPack.py @@ -3,7 +3,11 @@ from snowfakery.data_generator import generate from snowfakery.data_generator_runtime import StoppingCriteria -from snowfakery.experimental.DataPack import DataPack, ApexDataPack +from snowfakery.experimental.DataPack import ( + DataPack, + ApexDataPack, + SalesforceCompositeAPIOutput, +) import json ## Fill this out when it isn't experimental anymore @@ -26,6 +30,19 @@ def test_composite(self): print(single_payload) assert single_payload["graphs"][0]["compositeRequest"][0]["method"] == "POST" + def test_reference(self): + out = StringIO() + output_stream = SalesforceCompositeAPIOutput(out) + with open("examples/basic-salesforce.recipe.yml") as f: + generate(f, {}, output_stream) + output_stream.close() + print(out.getvalue()) + data = json.loads(out.getvalue()) + assert ( + data["graphs"][0]["compositeRequest"][-1]["body"]["AccountId"] + == "@{Account_2.id}" + ) + @patch("snowfakery.experimental.DataPack.MAX_BATCH_SIZE", 50) def test_composite_upsert(self): out = StringIO()