Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a --stats flag to the doc task, which prints timing and memory stats #3515

Merged
merged 1 commit into from
Oct 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 33 additions & 19 deletions tool/src/subprocess_launcher.dart
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,13 @@ class SubprocessLauncher {

SubprocessLauncher(this.context, [this.defaultEnvironment = const {}]);

/// A wrapper around start/await process.exitCode that will display the
/// output of the executable continuously and fail on non-zero exit codes.
/// A wrapper around start/await `process.exitCode` that displays the output
/// of [executable] with [arguments] continuously and throws on non-zero exit
/// codes.
///
/// It will also parse any valid JSON objects (one per line) it encounters
/// on stdout/stderr, and return them. Returns null if no JSON objects
/// on stdout/stderr, and return them. Returns null if no JSON objects
/// were encountered, or if DRY_RUN is set to 1 in the execution environment.
///
/// Makes running programs in grinder similar to `set -ex` for bash, even on
/// Windows (though some of the bashisms will no longer make sense).
// TODO(jcollins-g): refactor to return a stream of stderr/stdout lines and
// their associated JSON objects.
Future<Iterable<Map<String, Object?>>> runStreamed(
Expand All @@ -52,6 +50,7 @@ class SubprocessLauncher {
String? workingDirectory,
Map<String, String>? environment,
bool includeParentEnvironment = true,
bool withStats = false,
}) async {
environment = {...defaultEnvironment, ...?environment};
var jsonObjects = <Map<String, Object?>>[];
Expand Down Expand Up @@ -111,22 +110,14 @@ class SubprocessLauncher {

if (Platform.environment.containsKey('DRY_RUN')) return {};

var realExecutable = executable;
var realArguments = <String>[];
if (withStats) {
(executable, arguments) = _wrapWithTime(executable, arguments);
}
if (Platform.isLinux) {
// Use GNU coreutils to force line buffering. This makes sure that
// subprocesses that die due to fatal signals do not chop off the
// last few lines of their output.
//
// Dart does not actually do this (seems to flush manually) unless
// the VM crashes.
realExecutable = 'stdbuf';
realArguments.addAll(['-o', 'L', '-e', 'L']);
realArguments.add(executable);
(executable, arguments) = _wrapWithStdbuf(executable, arguments);
}
realArguments.addAll(arguments);

var process = await Process.start(realExecutable, realArguments,
var process = await Process.start(executable, arguments,
workingDirectory: workingDirectory,
environment: environment,
includeParentEnvironment: includeParentEnvironment);
Expand All @@ -143,18 +134,41 @@ class SubprocessLauncher {
return jsonObjects;
}

/// Wraps [command] and [args] with a call to `stdbuf`.
(String, List<String>) _wrapWithStdbuf(String command, List<String> args) =>
// Use GNU coreutils to force line buffering. This makes sure that
// subprocesses that die due to fatal signals do not chop off the last few
// lines of their output.
//
// Dart does not actually do this (seems to flush manually) unless the VM
// crashes.
(
'stdbuf',
[
...['-o', 'L', '-e', 'L'],
command,
...args,
],
);

/// Wraps [command] and [args] with a command to `time`.
(String, List<String>) _wrapWithTime(String command, List<String> args) =>
('/usr/bin/time', ['-l', command, ...args]);

Future<Iterable<Map<String, Object?>>> runStreamedDartCommand(
List<String> arguments, {
String? workingDirectory,
Map<String, String>? environment,
bool includeParentEnvironment = true,
bool withStats = false,
}) async =>
await runStreamed(
Platform.executable,
arguments,
workingDirectory: workingDirectory,
environment: environment,
includeParentEnvironment: includeParentEnvironment,
withStats: withStats,
);

static final _quotables = RegExp(r'[ "\r\n\$]');
Expand Down
50 changes: 32 additions & 18 deletions tool/task.dart
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ void main(List<String> args) async {
..addCommand('validate');
parser.addCommand('doc')
..addOption('name')
..addOption('version');
..addOption('version')
..addFlag('stats');
parser.addCommand('serve')
..addOption('name')
..addOption('version');
Expand Down Expand Up @@ -284,23 +285,25 @@ Future<void> runDoc(ArgResults commandResults) async {
throw ArgumentError('"doc" command requires a single target.');
}
var target = commandResults.rest.single;
var stats = commandResults['stats'];
await switch (target) {
'flutter' => docFlutter(),
'package' => _docPackage(commandResults),
'flutter' => docFlutter(withStats: stats),
'package' => _docPackage(commandResults, withStats: stats),
'sdk' => docSdk(),
'testing-package' => docTestingPackage(),
_ => throw UnimplementedError('Unknown doc target: "$target"'),
};
}

Future<void> docFlutter() async {
Future<void> docFlutter({bool withStats = false}) async {
print('building flutter docs into: $flutterDir');
var env = createThrowawayPubCache();
await _docFlutter(
flutterPath: flutterDir.path,
cwd: Directory.current.path,
env: env,
label: 'docs',
withStats: withStats,
);
var indexContents =
File(path.join(flutterDir.path, 'dev', 'docs', 'doc', 'index.html'))
Expand All @@ -313,6 +316,7 @@ Future<Iterable<Map<String, Object?>>> _docFlutter({
required String cwd,
required Map<String, String> env,
String? label,
bool withStats = false,
}) async {
var flutterRepo = await FlutterRepo.copyFromExistingFlutterRepo(
await cleanFlutterRepo, flutterPath, env, label);
Expand Down Expand Up @@ -353,20 +357,27 @@ Future<Iterable<Map<String, Object?>>> _docFlutter({
'--json',
],
workingDirectory: flutterPath,
withStats: withStats,
);
}

final Directory flutterDir =
Directory.systemTemp.createTempSync('flutter').absolute;

Future<void> _docPackage(ArgResults commandResults) async {
Future<void> _docPackage(
ArgResults commandResults, {
bool withStats = false,
}) async {
var name = commandResults['name'] as String;
var version = commandResults['version'] as String?;
await docPackage(name: name, version: version);
await docPackage(name: name, version: version, withStats: withStats);
}

Future<String> docPackage(
{required String name, required String? version}) async {
Future<String> docPackage({
required String name,
required String? version,
bool withStats = false,
}) async {
var env = createThrowawayPubCache();
var versionContext = version == null ? '' : '-$version';
var launcher = SubprocessLauncher('build-$name$versionContext', env);
Expand All @@ -392,16 +403,18 @@ Future<String> docPackage(
environment: flutterRepo.env,
workingDirectory: pubPackageDir.absolute.path);
await launcher.runStreamed(
flutterRepo.cacheDart,
[
'--enable-asserts',
path.join(Directory.current.absolute.path, 'bin', 'dartdoc.dart'),
'--json',
'--link-to-remote',
'--show-progress',
],
environment: flutterRepo.env,
workingDirectory: pubPackageDir.absolute.path);
flutterRepo.cacheDart,
[
'--enable-asserts',
path.join(Directory.current.absolute.path, 'bin', 'dartdoc.dart'),
'--json',
'--link-to-remote',
'--show-progress',
],
environment: flutterRepo.env,
workingDirectory: pubPackageDir.absolute.path,
withStats: withStats,
);
} else {
await launcher.runStreamedDartCommand(['pub', 'get'],
workingDirectory: pubPackageDir.absolute.path);
Expand All @@ -414,6 +427,7 @@ Future<String> docPackage(
'--show-progress',
],
workingDirectory: pubPackageDir.absolute.path,
withStats: withStats,
);
}
return path.join(pubPackageDir.absolute.path, 'doc', 'api');
Expand Down