Move PCF2-Lift Options to a shared location

fbpcs/emp_games/lift/calculator/test/CalculatorAppTest.cpp

@@ -24,8 +24,6 @@
 constexpr int32_t tsOffset = 10;
 
 DEFINE_bool(is_conversion_lift, true, "is conversion lift");
-DEFINE_int32(num_conversions_per_user, 4, "num of conversions per user");
-DEFINE_int64(epoch, 1546300800, "epoch");
 
 namespace private_lift {
 class CalculatorAppTest : public ::testing::Test {

fbpcs/emp_games/lift/common/CommonLiftOptions.cpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
+#include <gflags/gflags.h>
+
+DEFINE_bool(
+    compute_publisher_breakdowns,
+    true,
+    "To enable or disable computing publisher breakdown for result validation");
+DEFINE_bool(
+    log_cost,
+    false,
+    "Log cost info into cloud which will be used for dashboard");
+DEFINE_bool(
+    use_tls,
+    false,
+    "Whether to use TLS when communicating with other parties.");
+DEFINE_bool(
+    use_xor_encryption,
+    true,
+    "Reveal output with XOR secret shares instead of in the clear to both parties");
+DEFINE_uint32(concurrency, 1, "max number of games that will run concurrently");
+DEFINE_uint32(
+    file_start_index,
+    0,
+    "First file that will be read with base path");
+DEFINE_uint32(
+    num_conversions_per_user,
+    4,
+    "Cap and pad to this many conversions per user");
+DEFINE_uint32(num_files, 0, "Number of files that should be read");
+DEFINE_uint32(party, 1, "1 = publisher, 2 = partner");
+DEFINE_uint32(
+    port,
+    10000,
+    "Network port for establishing connection to other player");
+DEFINE_uint64(
+    epoch,
+    1546300800,
+    "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps");
+DEFINE_string(
+    ca_cert_path,
+    "",
+    "Relative file path where root CA cert is stored. It will be prefixed with $HOME.");
+DEFINE_string(
+    input_base_path,
+    "",
+    "Local or s3 base path for the sharded input files");
+DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name");
+DEFINE_string(
+    log_cost_s3_region,
+    ".s3.us-west-2.amazonaws.com/",
+    "s3 region name");
+DEFINE_string(
+    pc_feature_flags,
+    "",
+    "A String of PC Feature Flags passing from PCS, separated by comma");
+DEFINE_string(
+    private_key_path,
+    "",
+    "Relative file path where private key is stored. It will be prefixed with $HOME.");
+DEFINE_string(
+    run_name,
+    "",
+    "A user given run name that will be used in s3 filename");
+DEFINE_string(
+    server_cert_path,
+    "",
+    "Relative file path where server cert is stored. It will be prefixed with $HOME.");
+DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address");

fbpcs/emp_games/lift/common/CommonLiftOptions.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <gflags/gflags_declare.h>
+
+DECLARE_bool(compute_publisher_breakdowns);
+DECLARE_bool(log_cost);
+DECLARE_bool(use_tls);
+DECLARE_bool(use_xor_encryption);
+DECLARE_uint32(concurrency);
+DECLARE_uint32(file_start_index);
+DECLARE_uint32(num_conversions_per_user);
+DECLARE_uint32(num_files);
+DECLARE_uint32(party);
+DECLARE_uint32(port);
+DECLARE_uint64(epoch);
+DECLARE_string(ca_cert_path);
+DECLARE_string(input_base_path);
+DECLARE_string(log_cost_s3_bucket);
+DECLARE_string(log_cost_s3_region);
+DECLARE_string(pc_feature_flags);
+DECLARE_string(private_key_path);
+DECLARE_string(run_name);
+DECLARE_string(server_cert_path);
+DECLARE_string(server_ip);

fbpcs/emp_games/lift/metadata_compaction/MainUtil.h

@@ -8,13 +8,13 @@
 #pragma once
 
 #include <memory>
+#include <vector>
 #include "folly/Format.h"
 
 #include "fbpcf/engine/communication/SocketPartyCommunicationAgentFactory.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorApp.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGame.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactorGameFactory.h"
-
 namespace private_lift {
 
 struct LiftMetadataCompactionFilePaths {

fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.cpp

@@ -6,93 +6,89 @@
  */
 
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h"
+#include <gflags/gflags.h>
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
 
-DEFINE_int32(party, 1, "1 = publisher, 2 = partner");
-DEFINE_bool(
-    use_xor_encryption,
-    true,
-    "Reveal output with XOR secret shares instead of in the clear to both parties");
-DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address");
-DEFINE_int32(
-    port,
-    10000,
-    "Network port for establishing connection to other player");
-
-// Lift settings
 DEFINE_string(input_path, "", "Input file to run lift metadata compaction");
-DEFINE_string(
-    output_global_params_path,
-    "",
-    "Output file to write global params from input data.");
-DEFINE_string(
-    output_secret_shares_path,
-    "",
-    "Output file to write compacted metadata secret share results.");
-DEFINE_int32(
-    file_start_index,
-    0,
-    "First file that will be read with base path");
-DEFINE_int32(num_files, 0, "Number of files that should be read");
-DEFINE_string(
-    input_base_path,
-    "",
-    "Local or s3 base path for the sharded input files");
 DEFINE_string(
     output_global_params_base_path,
     "",
     "Local or s3 base path where output global param files are written to");
+DEFINE_string(
+    output_global_params_path,
+    "",
+    "Output file to write global params from input data.");
 DEFINE_string(
     output_secret_shares_base_path,
     "",
     "Local or s3 base path where output secret share files are written to");
-DEFINE_int32(concurrency, 1, "max number of games that will run concurrently");
-DEFINE_int32(
-    epoch,
-    1546300800,
-    "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps");
-DEFINE_int32(
-    num_conversions_per_user,
-    4,
-    "Cap and pad to this many conversions per user");
+DEFINE_string(
+    output_secret_shares_path,
+    "",
+    "Output file to write compacted metadata secret share results.");
+// Common lift options
 DEFINE_bool(
     compute_publisher_breakdowns,
     true,
     "To enable or disable computing publisher breakdown for result validation");
-
-// TLS Settings
+DEFINE_bool(
+    log_cost,
+    false,
+    "Log cost info into cloud which will be used for dashboard");
 DEFINE_bool(
     use_tls,
     false,
     "Whether to use TLS when communicating with other parties.");
+DEFINE_bool(
+    use_xor_encryption,
+    true,
+    "Reveal output with XOR secret shares instead of in the clear to both parties");
+DEFINE_uint32(concurrency, 1, "max number of games that will run concurrently");
+DEFINE_uint32(
+    file_start_index,
+    0,
+    "First file that will be read with base path");
+DEFINE_uint32(
+    num_conversions_per_user,
+    4,
+    "Cap and pad to this many conversions per user");
+DEFINE_uint32(num_files, 0, "Number of files that should be read");
+DEFINE_uint32(party, 1, "1 = publisher, 2 = partner");
+DEFINE_uint32(
+    port,
+    10000,
+    "Network port for establishing connection to other player");
+DEFINE_uint64(
+    epoch,
+    1546300800,
+    "Unixtime of 2019-01-01. Used as our 'new epoch' for timestamps");
 DEFINE_string(
     ca_cert_path,
     "",
     "Relative file path where root CA cert is stored. It will be prefixed with $HOME.");
 DEFINE_string(
-    server_cert_path,
+    input_base_path,
     "",
-    "Relative file path where server cert is stored. It will be prefixed with $HOME.");
+    "Local or s3 base path for the sharded input files");
+DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name");
+DEFINE_string(
+    log_cost_s3_region,
+    ".s3.us-west-2.amazonaws.com/",
+    "s3 region name");
+DEFINE_string(
+    pc_feature_flags,
+    "",
+    "A String of PC Feature Flags passing from PCS, separated by comma");
 DEFINE_string(
     private_key_path,
     "",
     "Relative file path where private key is stored. It will be prefixed with $HOME.");
-
-// Logging flags
 DEFINE_string(
     run_name,
     "",
     "A user given run name that will be used in s3 filename");
-DEFINE_bool(
-    log_cost,
-    false,
-    "Log cost info into cloud which will be used for dashboard");
-DEFINE_string(log_cost_s3_bucket, "", "s3 bucket name");
-DEFINE_string(
-    log_cost_s3_region,
-    ".s3.us-west-2.amazonaws.com/",
-    "s3 region name");
-
 DEFINE_string(
-    pc_feature_flags,
+    server_cert_path,
     "",
-    "A String of PC Feature Flags passing from PCS, separated by comma");
+    "Relative file path where server cert is stored. It will be prefixed with $HOME.");
+DEFINE_string(server_ip, "127.0.0.1", "Server's IP Address");

fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h

@@ -9,36 +9,9 @@
 
 #include <gflags/gflags.h>
 
-// MPC settings
-DECLARE_int32(party);
-DECLARE_bool(use_xor_encryption);
-DECLARE_string(server_ip);
-DECLARE_int32(port);
-
 // Lift settings
 DECLARE_string(input_path);
-DECLARE_string(output_global_params_path);
-DECLARE_string(output_secret_shares_path);
-DECLARE_int32(file_start_index);
-DECLARE_int32(num_files);
-DECLARE_string(input_base_path);
 DECLARE_string(output_global_params_base_path);
+DECLARE_string(output_global_params_path);
 DECLARE_string(output_secret_shares_base_path);
-DECLARE_int32(concurrency);
-DECLARE_int32(epoch);
-DECLARE_int32(num_conversions_per_user);
-DECLARE_bool(compute_publisher_breakdowns);
-
-// TLS Settings
-DECLARE_bool(use_tls);
-DECLARE_string(ca_cert_path);
-DECLARE_string(server_cert_path);
-DECLARE_string(private_key_path);
-
-// Logging flags
-DECLARE_string(run_name);
-DECLARE_bool(log_cost);
-DECLARE_string(log_cost_s3_bucket);
-DECLARE_string(log_cost_s3_region);
-
-DECLARE_string(pc_feature_flags);
+DECLARE_string(output_secret_shares_path);

fbpcs/emp_games/lift/metadata_compaction/main.cpp

@@ -13,8 +13,10 @@
 #include "fbpcf/aws/AwsSdk.h"
 #include "fbpcs/performance_tools/CostEstimation.h"
 
+#include <gflags/gflags.h>
 #include "fbpcs/emp_games/common/Constants.h"
 #include "fbpcs/emp_games/common/Util.h"
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MainUtil.h"
 #include "fbpcs/emp_games/lift/metadata_compaction/MetadataCompactionOptions.h"
 

fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include "fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h"
+#include <gflags/gflags.h>
+
+DEFINE_bool(
+    is_conversion_lift,
+    true,
+    "Use conversion_lift logic (as opposed to converter_lift logic)");
+DEFINE_string(
+    input_directory,
+    "",
+    "Data directory where input files are located");
+DEFINE_string(
+    input_expanded_key_path,
+    "out.csv_expanded_key_0",
+    "Input file name of the expanded key for UDP decryption. Used when decoupled UDP is enabled.");
+DEFINE_string(
+    input_filenames,
+    "in.csv_0[,in.csv_1,in.csv_2,...]",
+    "List of input file names that should be parsed (should have a header)");
+DEFINE_string(
+    input_global_params_path,
+    "out.csv_global_params_0",
+    "Input file name of global parameter setup. Used when reading inputs in secret share format rather than plaintext.");
+DEFINE_string(
+    output_base_path,
+    "",
+    "Local or s3 base path where output files are written to");
+DEFINE_string(
+    output_directory,
+    "",
+    "Local or s3 path where output files are written to");
+DEFINE_string(
+    output_filenames,
+    "out.csv_0[,out.csv_1,out.csv_2,...]",
+    "List of output file names that correspond to input filenames (positionally)");
+DEFINE_string(
+    run_id,
+    "",
+    "A run_id used to identify all the logs in a PL run.");

fbpcs/emp_games/lift/pcf2_calculator/LiftOptions.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <gflags/gflags_declare.h>
+#include "fbpcs/emp_games/lift/common/CommonLiftOptions.h"
+
+DECLARE_bool(is_conversion_lift);
+DECLARE_string(input_directory);
+DECLARE_string(input_expanded_key_path);
+DECLARE_string(input_filenames);
+DECLARE_string(input_global_params_path);
+DECLARE_string(output_base_path);
+DECLARE_string(output_directory);
+DECLARE_string(output_filenames);
+DECLARE_string(run_id);