-
Notifications
You must be signed in to change notification settings - Fork 0
/
gatsby-node.js
100 lines (90 loc) · 2.95 KB
/
gatsby-node.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
const glob = require(`tiny-glob`);
const fs = require(`fs-extra`);
const { createReadStream, readFile } = require(`fs`);
const fetch = require(`node-fetch`);
const BATCH_RUN_ID = Math.random().toString(36).slice(2);
const BATCH_MAX_RPS = 6;
const BATCH_MAX_PARALLEL_API_REQUESTS = 3;
const DATAUNLOCKER_ENDPOINT = ({ propertyId, scriptVersion }) =>
`https://api.dataunlocker.com/properties/${propertyId}/scripts/${scriptVersion}/inject?batchRunId=${encodeURIComponent(
BATCH_RUN_ID
)}`;
// Ensure that this plugin gets correct options.
exports.pluginOptionsSchema = ({ Joi }) => {
return Joi.object({
propertyId: Joi.string()
.pattern(/^[a-zA-Z0-9]{24}$/)
.required(),
scriptVersion: Joi.string().pattern(
/^latest|(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
),
});
};
const processHtmlFile = async (filePath, { propertyId, scriptVersion }) => {
let apiResult;
let processedBody = '';
const fileContents = await fs.readFile(filePath);
if (!/<head>/i.test(fileContents)) {
console.log(
`DataUnlocker: skipping ${filePath} as it doesn't look like valid HTML.`
);
return;
}
try {
console.log(`DataUnlocker: patching ${filePath}`);
apiResult = await fetch(
DATAUNLOCKER_ENDPOINT({ propertyId, scriptVersion }),
{
method: 'post',
headers: {
'Content-Type': 'text/html',
},
body: createReadStream(filePath),
}
);
} catch (e) {
console.error(
`DataUnlocker: unable to patch ${filePath}. DataUnlocker's API endpoint errors when processing: ${e}`
);
return;
}
try {
processedBody = await apiResult.text();
} catch (e) {
/* Keep processedBody empty, as the status should not be 200. */
}
if (apiResult.status !== 200) {
console.error(
`DataUnlocker: unable to patch ${filePath}. DataUnlocker's API endpoint returned status ${apiResult.status}: ${processedBody}`
);
return;
}
return await fs.writeFile(filePath, processedBody);
};
exports.onPostBuild = async (
_,
{ propertyId = '', scriptVersion = 'latest' }
) => {
const files = await glob(`public/**/*.html`, {
filesOnly: true,
});
const promises = files.map((fileName) => async () => {
const start = Date.now();
await processHtmlFile(fileName, { propertyId, scriptVersion });
const extra =
(1000 / BATCH_MAX_RPS) * BATCH_MAX_PARALLEL_API_REQUESTS -
(Date.now() - start);
if (extra > 0) {
await new Promise((r) => setTimeout(r, extra));
}
});
// Run tasks in parallel with the max concurrency limited to BATCH_MAX_PARALLEL_API_REQUESTS.
const workers = new Array(BATCH_MAX_PARALLEL_API_REQUESTS)
.fill(promises.entries())
.map(async (iterator) => {
for (let [_, f] of iterator) {
await f();
}
});
await Promise.allSettled(workers);
};