Skip to content
This repository was archived by the owner on Oct 24, 2025. It is now read-only.

Commit 937fa56

Browse files
bsamuel-uibsamuel-ui
committed
Clean up how the commands are built and run, and make sure strip is called correctly.
- Make sure we're consistently quoting arguments. - Add mergeCommands function to construct a script for docker to run when needed. - Add getStripMode to run strip correctly for the platform and docker.
1 parent 337cd46 commit 937fa56

File tree

3 files changed

+129
-101
lines changed

3 files changed

+129
-101
lines changed

‎lib/pip.js‎

Lines changed: 107 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,42 @@ const set = require('lodash.set');
66
const { spawnSync } = require('child_process');
77
const { quote } = require('shell-quote');
88
const { buildImage, getBindPath, getDockerUid } = require('./docker');
9-
const { getStripCommand, deleteFiles } = require('./slim');
9+
const { getStripCommand, getStripMode, deleteFiles } = require('./slim');
1010
const {
1111
checkForAndDeleteMaxCacheVersions,
1212
md5Path,
1313
getRequirementsWorkingPath,
1414
getUserCachePath
1515
} = require('./shared');
1616

17-
function quote_single(quoteme) {
18-
return quote([quoteme]);
17+
/**
18+
* Omit empty commands.
19+
* In this context, a "command" is a list of arguments. An empty list or falsy value is ommitted.
20+
* @param {string[][]} many commands to merge.
21+
* @return {string[][]} a list of valid commands.
22+
*/
23+
function filterCommands(commands) {
24+
return commands.filter((cmd) => Boolean(cmd) && cmd.length > 0);
25+
}
26+
27+
/**
28+
* Render zero or more commands as a single command for a Unix environment.
29+
* In this context, a "command" is a list of arguments. An empty list or falsy value is ommitted.
30+
*
31+
* @param {string[][]} many commands to merge.
32+
* @return {string[]} a single list of words.
33+
*/
34+
function mergeCommands(commands) {
35+
const cmds = filterCommands(commands);
36+
if (cmds.length === 0) {
37+
throw new Error('Expected at least one non-empty command')
38+
} else if (cmds.length === 1) {
39+
return cmds[0];
40+
} else {
41+
// Quote the arguments in each command and join them all using &&.
42+
const script = cmds.map(quote).join(' && ');
43+
return ["/bin/sh", "-c", script];
44+
}
1945
}
2046

2147
/**
@@ -51,6 +77,25 @@ function installRequirementsFile(
5177
}
5278
}
5379

80+
function pipAcceptsSystem(pythonBin) {
81+
// Check if pip has Debian's --system option and set it if so
82+
const pipTestRes = spawnSync(pythonBin, [
83+
'-m',
84+
'pip',
85+
'help',
86+
'install'
87+
]);
88+
if (pipTestRes.error) {
89+
if (pipTestRes.error.code === 'ENOENT') {
90+
throw new Error(
91+
`${pythonBin} not found! Try the pythonBin option.`
92+
);
93+
}
94+
throw pipTestRes.error;
95+
}
96+
return pipTestRes.stdout.toString().indexOf('--system') >= 0;
97+
}
98+
5499
/**
55100
* Install requirements described from requirements in the targetFolder into that same targetFolder
56101
* @param {string} targetFolder
@@ -65,15 +110,16 @@ function installRequirements(targetFolder, serverless, options) {
65110
`Installing requirements from ${targetRequirementsTxt} ...`
66111
);
67112

68-
let cmd;
69-
let cmdOptions;
70-
let pipCmd = [
113+
const dockerCmd = [];
114+
const pipCmd = [
71115
options.pythonBin,
72116
'-m',
73117
'pip',
74118
'install',
75119
...options.pipCmdExtraArgs
76120
];
121+
const pipCmds = [pipCmd];
122+
const postCmds = [];
77123
// Check if we're using the legacy --cache-dir command...
78124
if (options.pipCmdExtraArgs.indexOf('--cache-dir') > -1) {
79125
if (options.dockerizePip) {
@@ -94,8 +140,8 @@ function installRequirements(targetFolder, serverless, options) {
94140

95141
if (!options.dockerizePip) {
96142
// Push our local OS-specific paths for requirements and target directory
97-
pipCmd.push('-t', dockerPathForWin(options, targetFolder));
98-
pipCmd.push('-r', dockerPathForWin(options, targetRequirementsTxt));
143+
pipCmd.push('-t', dockerPathForWin(targetFolder),
144+
'-r', dockerPathForWin(targetRequirementsTxt));
99145
// If we want a download cache...
100146
if (options.useDownloadCache) {
101147
const downloadCacheDir = path.join(
@@ -104,35 +150,17 @@ function installRequirements(targetFolder, serverless, options) {
104150
);
105151
serverless.cli.log(`Using download cache directory ${downloadCacheDir}`);
106152
fse.ensureDirSync(downloadCacheDir);
107-
pipCmd.push('--cache-dir', quote_single(downloadCacheDir));
153+
pipCmd.push('--cache-dir', downloadCacheDir);
108154
}
109155

110-
// Check if pip has Debian's --system option and set it if so
111-
const pipTestRes = spawnSync(options.pythonBin, [
112-
'-m',
113-
'pip',
114-
'help',
115-
'install'
116-
]);
117-
if (pipTestRes.error) {
118-
if (pipTestRes.error.code === 'ENOENT') {
119-
throw new Error(
120-
`${options.pythonBin} not found! ` + 'Try the pythonBin option.'
121-
);
122-
}
123-
throw pipTestRes.error;
124-
}
125-
if (pipTestRes.stdout.toString().indexOf('--system') >= 0) {
156+
if (pipAcceptsSystem(options.pythonBin)) {
126157
pipCmd.push('--system');
127158
}
128159
}
129160
// If we are dockerizing pip
130161
if (options.dockerizePip) {
131-
cmd = 'docker';
132-
133162
// Push docker-specific paths for requirements and target directory
134-
pipCmd.push('-t', '/var/task/');
135-
pipCmd.push('-r', '/var/task/requirements.txt');
163+
pipCmd.push('-t', '/var/task/', '-r', '/var/task/requirements.txt');
136164

137165
// Build docker image if required
138166
let dockerImage;
@@ -148,28 +176,18 @@ function installRequirements(targetFolder, serverless, options) {
148176

149177
// Prepare bind path depending on os platform
150178
const bindPath = dockerPathForWin(
151-
options,
152179
getBindPath(serverless, targetFolder)
153180
);
154181

155-
cmdOptions = ['run', '--rm', '-v', `${bindPath}:/var/task:z`];
182+
dockerCmd.push('docker', 'run', '--rm', '-v', `${bindPath}:/var/task:z`);
156183
if (options.dockerSsh) {
157184
// Mount necessary ssh files to work with private repos
158-
cmdOptions.push(
159-
'-v',
160-
quote_single(`${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`)
161-
);
162-
cmdOptions.push(
163-
'-v',
164-
quote_single(
165-
`${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`
166-
)
167-
);
168-
cmdOptions.push(
169-
'-v',
170-
quote_single(`${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`)
185+
dockerCmd.push(
186+
'-v', `${process.env.HOME}/.ssh/id_rsa:/root/.ssh/id_rsa:z`,
187+
'-v', `${process.env.HOME}/.ssh/known_hosts:/root/.ssh/known_hosts:z`,
188+
'-v', `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`,
189+
'-e', 'SSH_AUTH_SOCK=/tmp/ssh_sock'
171190
);
172-
cmdOptions.push('-e', 'SSH_AUTH_SOCK=/tmp/ssh_sock');
173191
}
174192

175193
// If we want a download cache...
@@ -189,104 +207,100 @@ function installRequirements(targetFolder, serverless, options) {
189207
);
190208
const windowsized = getBindPath(serverless, downloadCacheDir);
191209
// And now push it to a volume mount and to pip...
192-
cmdOptions.push(
210+
dockerCmd.push(
193211
'-v',
194-
quote_single(`${windowsized}:${dockerDownloadCacheDir}:z`)
212+
`${windowsized}:${dockerDownloadCacheDir}:z`
195213
);
196-
pipCmd.push('--cache-dir', quote_single(dockerDownloadCacheDir));
214+
pipCmd.push('--cache-dir', dockerDownloadCacheDir);
197215
}
198216

199217
if (options.dockerEnv) {
200218
// Add environment variables to docker run cmd
201219
options.dockerEnv.forEach(function(item) {
202-
cmdOptions.push('-e', item);
220+
dockerCmd.push('-e', item);
203221
});
204222
}
205223

206224
if (process.platform === 'linux') {
207225
// Use same user so requirements folder is not root and so --cache-dir works
208-
var commands = [];
209226
if (options.useDownloadCache) {
210227
// Set the ownership of the download cache dir to root
211-
commands.push(quote(['chown', '-R', '0:0', dockerDownloadCacheDir]));
228+
pipCmds.unshift(['chown', '-R', '0:0', dockerDownloadCacheDir]);
212229
}
213230
// Install requirements with pip
214-
commands.push(pipCmd.join(' '));
215231
// Set the ownership of the current folder to user
216-
commands.push(
217-
quote([
218-
'chown',
219-
'-R',
220-
`${process.getuid()}:${process.getgid()}`,
221-
'/var/task'
222-
])
223-
);
232+
pipCmds.push(['chown', '-R', `${process.getuid()}:${process.getgid()}`, '/var/task']);
224233
if (options.useDownloadCache) {
225234
// Set the ownership of the download cache dir back to user
226-
commands.push(
227-
quote([
235+
pipCmds.push(
236+
[
228237
'chown',
229238
'-R',
230239
`${process.getuid()}:${process.getgid()}`,
231240
dockerDownloadCacheDir
232-
])
241+
]
233242
);
234243
}
235-
pipCmd = ['/bin/bash', '-c', '"' + commands.join(' && ') + '"'];
236244
} else {
237245
// Use same user so --cache-dir works
238-
cmdOptions.push('-u', quote_single(getDockerUid(bindPath)));
246+
dockerCmd.push('-u', getDockerUid(bindPath));
239247
}
240-
cmdOptions.push(dockerImage);
241-
cmdOptions.push(...pipCmd);
242-
} else {
243-
cmd = pipCmd[0];
244-
cmdOptions = pipCmd.slice(1);
248+
dockerCmd.push(dockerImage);
245249
}
246250

247251
// If enabled slimming, strip so files
248-
if (options.slim === true || options.slim === 'true') {
249-
const preparedPath = dockerPathForWin(options, targetFolder);
250-
cmdOptions.push(getStripCommand(options, preparedPath));
252+
switch (getStripMode(options)) {
253+
case 'docker':
254+
pipCmds.push(getStripCommand(options, '/var/task'));
255+
case 'direct':
256+
postCmds.push(getStripCommand(options, dockerPathForWin(targetFolder)));
251257
}
258+
252259
let spawnArgs = { shell: true };
253260
if (process.env.SLS_DEBUG) {
254261
spawnArgs.stdio = 'inherit';
255262
}
256-
const res = spawnSync(cmd, cmdOptions, spawnArgs);
257-
if (res.error) {
258-
if (res.error.code === 'ENOENT') {
259-
if (options.dockerizePip) {
260-
throw new Error('docker not found! Please install it.');
261-
}
262-
throw new Error(
263-
`${options.pythonBin} not found! Try the pythonBin option.`
264-
);
265-
}
266-
throw res.error;
267-
}
268-
if (res.status !== 0) {
269-
throw new Error(res.stderr);
263+
let mainCmds = [];
264+
if (dockerCmd.length) {
265+
dockerCmd.push(...mergeCommands(pipCmds));
266+
mainCmds = [dockerCmd];
267+
} else {
268+
mainCmds = pipCmds;
270269
}
270+
mainCmds.push(...postCmds);
271+
272+
serverless.cli.log(`Running ${quote(dockerCmd)}...`);
273+
274+
filterCommands(mainCmds).forEach(([cmd, ...args]) => {
275+
const res = spawnSync(cmd, args);
276+
if (res.error) {
277+
if (res.error.code === 'ENOENT') {
278+
const advice = cmd.indexOf('python') > -1 ? 'Try the pythonBin option' : 'Please install it';
279+
throw new Error(`${cmd} not found! ${advice}`);
280+
}
281+
throw res.error;
282+
}
283+
if (res.status !== 0) {
284+
throw new Error(res.stderr);
285+
}
286+
});
271287
// If enabled slimming, delete files in slimPatterns
272288
if (options.slim === true || options.slim === 'true') {
273289
deleteFiles(options, targetFolder);
274290
}
275291
}
276292

277293
/**
278-
* convert path from Windows style to Linux style, if needed
279-
* @param {Object} options
294+
* Convert path from Windows style to Linux style, if needed.
280295
* @param {string} path
281296
* @return {string}
282297
*/
283-
function dockerPathForWin(options, path) {
298+
function dockerPathForWin(path) {
284299
if (process.platform === 'win32') {
285-
return `"${path.replace(/\\/g, '/')}"`;
286-
} else if (process.platform === 'win32' && !options.dockerizePip) {
300+
return path.replace(/\\/g, '/');
301+
} else {
287302
return path;
288303
}
289-
return quote_single(path);
290304
}
291305

292306
/** create a filtered requirements.txt without anything from noDeploy

‎lib/slim.js‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,19 @@ const isWsl = require('is-wsl');
22
const glob = require('glob-all');
33
const fse = require('fs-extra');
44

5-
const getStripCommand = (options, folderPath) =>
6-
process.platform !== 'win32' || isWsl || options.dockerizePip
7-
? ` && find ${folderPath} -name "*.so" -exec strip {} ';'`
8-
: '';
5+
const getStripMode = (options) => {
6+
if (options.slim === false || options.slim === 'false') {
7+
return 'skip';
8+
} else if (options.dockerizePip) {
9+
return 'docker';
10+
} else if (!isWsl && process.platform === 'win32' || process.platform === 'darwin') {
11+
return 'skip';
12+
} else {
13+
return 'direct';
14+
}
15+
}
16+
17+
const getStripCommand = (options, folderPath) => (['find', folderPath, '-name', '*.so', '-exec', 'strip', '{}', '+']);
918

1019
const deleteFiles = (options, folderPath) => {
1120
let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*'];
@@ -27,6 +36,7 @@ const deleteFiles = (options, folderPath) => {
2736
};
2837

2938
module.exports = {
39+
getStripMode,
3040
getStripCommand,
3141
deleteFiles
3242
};

‎test.js‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ const deasync = require('deasync-promise');
33
const glob = require('glob-all');
44
const JSZip = require('jszip');
55
const tape = require('tape');
6+
const { quote } = require('shell-quote');
67
const { removeSync, readFileSync, copySync } = require('fs-extra');
78
const { sep } = require('path');
89

@@ -25,11 +26,14 @@ const mkCommand = cmd => (args, options = {}) => {
2526
options
2627
)
2728
);
28-
if (error) throw error;
29+
if (error) {
30+
console.error(`Error running: ${quote([cmd, ...args])}`);
31+
throw error;
32+
}
2933
if (status) {
30-
console.error(stdout.toString()); // eslint-disable-line no-console
31-
console.error(stderr.toString()); // eslint-disable-line no-console
32-
throw new Error(`${cmd} failed with status code ${status}`);
34+
console.error('STDOUT: ', stdout.toString()); // eslint-disable-line no-console
35+
console.error('STDERR: ', stderr.toString()); // eslint-disable-line no-console
36+
throw new Error(`${quote([cmd, ...args])} failed with status code ${status}`);
3337
}
3438
return stdout && stdout.toString().trim();
3539
};

0 commit comments

Comments
 (0)