diff --git a/contenttype-data.js b/contenttype-data.js index 7662c7c..529dd5a 100644 --- a/contenttype-data.js +++ b/contenttype-data.js @@ -16,13 +16,17 @@ var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/cl var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass")); +var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty")); + function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2["default"])(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2["default"])(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2["default"])(this, result); }; } function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } } var FetchContentTypes = /*#__PURE__*/function () { - function FetchContentTypes() { + function FetchContentTypes(query) { (0, _classCallCheck2["default"])(this, FetchContentTypes); + (0, _defineProperty2["default"])(this, "query", void 0); + this.query = query; } (0, _createClass2["default"])(FetchContentTypes, [{ @@ -55,25 +59,27 @@ var FetchDefaultContentTypes = /*#__PURE__*/function (_FetchContentTypes) { var _super = _createSuper(FetchDefaultContentTypes); - function FetchDefaultContentTypes() { + function FetchDefaultContentTypes(query) { + var _this; + (0, _classCallCheck2["default"])(this, FetchDefaultContentTypes); - return _super.apply(this, arguments); + _this = _super.call(this, query); + _this.query = query; + return _this; } (0, _createClass2["default"])(FetchDefaultContentTypes, [{ key: "getPagedData", value: function () { var _getPagedData2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(url, config, responseKey, fn) { - var query, result; + var result; return _regenerator["default"].wrap(function _callee2$(_context2) { while (1) { switch (_context2.prev = _context2.next) { case 0: - query = { - include_global_field_schema: true - }; + this.query.query = JSON.stringify(this.query.query); _context2.next = 3; - return fn.apply(null, [url, config, responseKey, query]); + return fn.apply(null, [url, config, responseKey, this.query]); case 3: result = _context2.sent; @@ -84,7 +90,7 @@ var FetchDefaultContentTypes = /*#__PURE__*/function (_FetchContentTypes) { return _context2.stop(); } } - }, _callee2); + }, _callee2, this); })); function getPagedData(_x, _x2, _x3, _x4) { @@ -102,63 +108,65 @@ var FetchSpecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes2) { var _super2 = _createSuper(FetchSpecifiedContentTypes); - function FetchSpecifiedContentTypes() { + function FetchSpecifiedContentTypes(query) { + var _this2; + (0, _classCallCheck2["default"])(this, FetchSpecifiedContentTypes); - return _super2.apply(this, arguments); + _this2 = _super2.call(this, query); // We don't want to restrict the specified content-types download by last fetch time, as it can ignore updates from referred content-types. + + delete query.query; + _this2.query = query; + return _this2; } (0, _createClass2["default"])(FetchSpecifiedContentTypes, [{ key: "getPagedData", value: function () { var _getPagedData3 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee3(url, config, responseKey, fn) { - var query, contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result; + var contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result; return _regenerator["default"].wrap(function _callee3$(_context3) { while (1) { switch (_context3.prev = _context3.next) { case 0: - query = { - query: JSON.stringify({ - uid: { - $in: config.contentTypes - } - }), - include_global_field_schema: true + this.query.query.uid = { + $in: config.contentTypes }; - _context3.next = 3; - return fn.apply(null, [url, config, responseKey, query]); + this.query.query = JSON.stringify(this.query.query); + _context3.next = 4; + return fn.apply(null, [url, config, responseKey, this.query]); - case 3: + case 4: contentTypes = _context3.sent; referredContentTypes = new ReferredContentTypes(); referredContentTypesList = referredContentTypes.getReferredContentTypes(contentTypes); referredContentTypesData = []; if (!referredContentTypesList.length) { - _context3.next = 12; + _context3.next = 13; break; } - query.query = JSON.stringify({ + this.query.query = JSON.stringify({ uid: { $in: referredContentTypesList } }); - _context3.next = 11; - return fn.apply(null, [url, config, responseKey, query]); + _context3.next = 12; + return fn.apply(null, [url, config, responseKey, this.query]); - case 11: + case 12: referredContentTypesData = _context3.sent; - case 12: + case 13: result = contentTypes.concat(referredContentTypesData); return _context3.abrupt("return", result); - case 14: + case 15: case "end": return _context3.stop(); } } - }, _callee3); + }, _callee3, this); })); function getPagedData(_x5, _x6, _x7, _x8) { @@ -176,63 +184,65 @@ var FetchUnspecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes3) { var _super3 = _createSuper(FetchUnspecifiedContentTypes); - function FetchUnspecifiedContentTypes() { + function FetchUnspecifiedContentTypes(query) { + var _this3; + (0, _classCallCheck2["default"])(this, FetchUnspecifiedContentTypes); - return _super3.apply(this, arguments); + _this3 = _super3.call(this, query); // We don't want to restrict the specified content-types download by last fetch time, as it can ignore updates from referred content-types. + + delete query.query; + _this3.query = query; + return _this3; } (0, _createClass2["default"])(FetchUnspecifiedContentTypes, [{ key: "getPagedData", value: function () { var _getPagedData4 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee4(url, config, responseKey, fn) { - var query, contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result; + var contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result; return _regenerator["default"].wrap(function _callee4$(_context4) { while (1) { switch (_context4.prev = _context4.next) { case 0: - query = { - query: JSON.stringify({ - uid: { - $nin: config.excludeContentTypes - } - }), - include_global_field_schema: true + this.query.query.uid = { + $nin: config.excludeContentTypes }; - _context4.next = 3; - return fn.apply(null, [url, config, responseKey, query]); + this.query.query = JSON.stringify(this.query.query); + _context4.next = 4; + return fn.apply(null, [url, config, responseKey, this.query]); - case 3: + case 4: contentTypes = _context4.sent; referredContentTypes = new ReferredContentTypes(); referredContentTypesList = referredContentTypes.getReferredContentTypes(contentTypes); referredContentTypesData = []; if (!referredContentTypesList.length) { - _context4.next = 12; + _context4.next = 13; break; } - query.query = JSON.stringify({ + this.query.query = JSON.stringify({ uid: { $in: referredContentTypesList } }); - _context4.next = 11; - return fn.apply(null, [url, config, responseKey, query]); + _context4.next = 12; + return fn.apply(null, [url, config, responseKey, this.query]); - case 11: + case 12: referredContentTypesData = _context4.sent; - case 12: + case 13: result = contentTypes.concat(referredContentTypesData); return _context4.abrupt("return", result); - case 14: + case 15: case "end": return _context4.stop(); } } - }, _callee4); + }, _callee4, this); })); function getPagedData(_x9, _x10, _x11, _x12) { diff --git a/create-schema-customization.js b/create-schema-customization.js index 98073d3..8ea6af3 100644 --- a/create-schema-customization.js +++ b/create-schema-customization.js @@ -50,7 +50,7 @@ exports.createSchemaCustomization = /*#__PURE__*/function () { _context2.prev = 3; contentTypeOption = getContentTypeOption(configOptions); _context2.next = 7; - return fetchContentTypes(configOptions, contentTypeOption); + return fetchContentTypes(configOptions, contentTypeOption, cache); case 7: contentTypes = _context2.sent; diff --git a/fetch.js b/fetch.js index 49be7e0..c6ffec7 100644 --- a/fetch.js +++ b/fetch.js @@ -5,8 +5,14 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator")); +var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty")); + var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator")); +function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; } + +function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2["default"])(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; } + var queryString = require('query-string'); var fetch = require('node-fetch'); // eslint-disable-next-line import/no-unresolved @@ -27,7 +33,9 @@ var _require3 = require('./entry-data'), FetchSpecifiedLocalesAndContentTypesEntries = _require3.FetchSpecifiedLocalesAndContentTypesEntries; var _require4 = require('./utils'), - CODES = _require4.CODES; + CODES = _require4.CODES, + LAST_CONTENT_TYPE_FETCH_TIME = _require4.LAST_CONTENT_TYPE_FETCH_TIME, + DEFAULT_CONTENT_TYPE_FETCH_TIME = _require4.DEFAULT_CONTENT_TYPE_FETCH_TIME; var OPTION_CLASS_MAPPING = { '': FetchDefaultContentTypes, @@ -98,26 +106,43 @@ exports.fetchData = /*#__PURE__*/function () { }(); exports.fetchContentTypes = /*#__PURE__*/function () { - var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(config, contentTypeOption) { - var url, responseKey, contentType, allContentTypes; + var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(config, contentTypeOption, cache) { + var typePrefix, lastFetchTimeKey, contentTypeFetchTimeQuery, currentTime, query, url, responseKey, contentType, allContentTypes; return _regenerator["default"].wrap(function _callee2$(_context2) { while (1) { switch (_context2.prev = _context2.next) { case 0: _context2.prev = 0; config.cdn = config.cdn ? config.cdn : 'https://cdn.contentstack.io/v3'; + typePrefix = config.type_prefix || 'Contentstack'; + lastFetchTimeKey = "".concat(typePrefix, "_").concat(config.api_key, "_").concat(LAST_CONTENT_TYPE_FETCH_TIME); + _context2.next = 6; + return getLastContentTypeFetchTime(lastFetchTimeKey, cache); + + case 6: + contentTypeFetchTimeQuery = _context2.sent; + currentTime = new Date(); + currentTime = currentTime.toISOString(); + query = { + include_global_field_schema: true, + query: _objectSpread({}, contentTypeFetchTimeQuery) + }; url = 'content_types'; responseKey = 'content_types'; - contentType = new OPTION_CLASS_MAPPING[contentTypeOption](); - _context2.next = 7; + contentType = new OPTION_CLASS_MAPPING[contentTypeOption](query); + _context2.next = 15; return contentType.getPagedData(url, config, responseKey, getPagedData); - case 7: + case 15: allContentTypes = _context2.sent; + _context2.next = 18; + return cache.set(lastFetchTimeKey, currentTime); + + case 18: return _context2.abrupt("return", allContentTypes); - case 11: - _context2.prev = 11; + case 21: + _context2.prev = 21; _context2.t0 = _context2["catch"](0); reporter.panic({ id: CODES.SyncError, @@ -127,15 +152,15 @@ exports.fetchContentTypes = /*#__PURE__*/function () { error: _context2.t0 }); - case 14: + case 24: case "end": return _context2.stop(); } } - }, _callee2, null, [[0, 11]]); + }, _callee2, null, [[0, 21]]); })); - return function (_x5, _x6) { + return function (_x5, _x6, _x7) { return _ref2.apply(this, arguments); }; }(); @@ -163,7 +188,7 @@ var fetchSyncData = /*#__PURE__*/function () { }, _callee3); })); - return function fetchSyncData(_x7, _x8) { + return function fetchSyncData(_x8, _x9) { return _ref3.apply(this, arguments); }; }(); @@ -211,7 +236,7 @@ var fetchCsData = /*#__PURE__*/function () { }, _callee4); })); - return function fetchCsData(_x9, _x10, _x11) { + return function fetchCsData(_x10, _x11, _x12) { return _ref4.apply(this, arguments); }; }(); @@ -265,7 +290,7 @@ var getPagedData = /*#__PURE__*/function () { }, _callee5); })); - return function getPagedData(_x12, _x13, _x14) { + return function getPagedData(_x13, _x14, _x15) { return _ref5.apply(this, arguments); }; }(); @@ -317,7 +342,46 @@ var getSyncData = /*#__PURE__*/function () { }, _callee6); })); - return function getSyncData(_x15, _x16, _x17, _x18) { + return function getSyncData(_x16, _x17, _x18, _x19) { return _ref6.apply(this, arguments); }; +}(); + +var getLastContentTypeFetchTime = /*#__PURE__*/function () { + var _ref7 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee7(lastFetchTimeKey, cache) { + var lastFetch, fetchTimeQuery; + return _regenerator["default"].wrap(function _callee7$(_context7) { + while (1) { + switch (_context7.prev = _context7.next) { + case 0: + _context7.next = 2; + return cache.get(lastFetchTimeKey); + + case 2: + lastFetch = _context7.sent; + fetchTimeQuery = {}; + + if (lastFetch) { + fetchTimeQuery.updated_at = { + $gte: lastFetch + }; + } else { + fetchTimeQuery.created_at = { + $gte: DEFAULT_CONTENT_TYPE_FETCH_TIME + }; + } + + return _context7.abrupt("return", fetchTimeQuery); + + case 6: + case "end": + return _context7.stop(); + } + } + }, _callee7); + })); + + return function getLastContentTypeFetchTime(_x20, _x21) { + return _ref7.apply(this, arguments); + }; }(); \ No newline at end of file diff --git a/package.json b/package.json index e209164..119e8c8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gatsby-source-contentstack", - "version": "4.0.1", + "version": "4.0.2", "description": "Gatsby source plugin for building websites using Contentstack as a data source", "scripts": { "prepublish": "npm run build", diff --git a/src/contenttype-data.js b/src/contenttype-data.js index dd1936e..49a997a 100644 --- a/src/contenttype-data.js +++ b/src/contenttype-data.js @@ -1,36 +1,46 @@ 'use strict'; class FetchContentTypes { + query; + constructor(query) { + this.query = query; + } async getPagedData() { } } class FetchDefaultContentTypes extends FetchContentTypes { + constructor(query) { + super(query); + this.query = query; + } + async getPagedData(url, config, responseKey, fn) { - const query = { - include_global_field_schema: true - }; - const result = await fn.apply(null, [url, config, responseKey, query]); + this.query.query = JSON.stringify(this.query.query); + const result = await fn.apply(null, [url, config, responseKey, this.query]); return result; } } class FetchSpecifiedContentTypes extends FetchContentTypes { + constructor(query) { + super(query); + // We don't want to restrict the specified content-types download by last fetch time, as it can ignore updates from referred content-types. + delete query.query; + this.query = query; + } + async getPagedData(url, config, responseKey, fn) { - const query = { - query: JSON.stringify({ - uid: { $in: config.contentTypes } - }), - include_global_field_schema: true - }; - const contentTypes = await fn.apply(null, [url, config, responseKey, query]); + this.query.query.uid = { $in: config.contentTypes }; + this.query.query = JSON.stringify(this.query.query); + const contentTypes = await fn.apply(null, [url, config, responseKey, this.query]); const referredContentTypes = new ReferredContentTypes(); const referredContentTypesList = referredContentTypes.getReferredContentTypes(contentTypes); let referredContentTypesData = []; if (referredContentTypesList.length) { - query.query = JSON.stringify({ uid: { $in: referredContentTypesList } }); - referredContentTypesData = await fn.apply(null, [url, config, responseKey, query]); + this.query.query = JSON.stringify({ uid: { $in: referredContentTypesList } }); + referredContentTypesData = await fn.apply(null, [url, config, responseKey, this.query]); } const result = contentTypes.concat(referredContentTypesData); @@ -39,22 +49,25 @@ class FetchSpecifiedContentTypes extends FetchContentTypes { } class FetchUnspecifiedContentTypes extends FetchContentTypes { + constructor(query) { + super(query); + // We don't want to restrict the specified content-types download by last fetch time, as it can ignore updates from referred content-types. + delete query.query; + this.query = query; + } + async getPagedData(url, config, responseKey, fn) { - const query = { - query: JSON.stringify({ - uid: { $nin: config.excludeContentTypes } - }), - include_global_field_schema: true - }; - const contentTypes = await fn.apply(null, [url, config, responseKey, query]); + this.query.query.uid = { $nin: config.excludeContentTypes }; + this.query.query = JSON.stringify(this.query.query); + const contentTypes = await fn.apply(null, [url, config, responseKey, this.query]); const referredContentTypes = new ReferredContentTypes(); const referredContentTypesList = referredContentTypes.getReferredContentTypes(contentTypes); let referredContentTypesData = []; if (referredContentTypesList.length) { - query.query = JSON.stringify({ uid: { $in: referredContentTypesList } }); - referredContentTypesData = await fn.apply(null, [url, config, responseKey, query]); + this.query.query = JSON.stringify({ uid: { $in: referredContentTypesList } }); + referredContentTypesData = await fn.apply(null, [url, config, responseKey, this.query]); } const result = contentTypes.concat(referredContentTypesData); diff --git a/src/create-schema-customization.js b/src/create-schema-customization.js index 6eea4fc..64a1989 100644 --- a/src/create-schema-customization.js +++ b/src/create-schema-customization.js @@ -14,7 +14,7 @@ exports.createSchemaCustomization = async ({ cache, actions, schema, reporter }, const disableMandatoryFields = configOptions.disableMandatoryFields || false; try { const contentTypeOption = getContentTypeOption(configOptions); - contentTypes = await fetchContentTypes(configOptions, contentTypeOption); + contentTypes = await fetchContentTypes(configOptions, contentTypeOption, cache); // Caching content-types because we need to be able to support multiple stacks. await cache.set(typePrefix, contentTypes); } catch (error) { diff --git a/src/fetch.js b/src/fetch.js index 6d0c208..bd028ea 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -7,7 +7,7 @@ const fetch = require('node-fetch'); const { version } = require('./package.json'); const { FetchDefaultContentTypes, FetchSpecifiedContentTypes, FetchUnspecifiedContentTypes } = require('./contenttype-data'); const { FetchDefaultEntries, FetchSpecifiedContentTypesEntries, FetchSpecifiedLocalesEntries, FetchSpecifiedLocalesAndContentTypesEntries } = require('./entry-data'); -const { CODES } = require('./utils'); +const { CODES, LAST_CONTENT_TYPE_FETCH_TIME, DEFAULT_CONTENT_TYPE_FETCH_TIME } = require('./utils'); const OPTION_CLASS_MAPPING = { '': FetchDefaultContentTypes, @@ -50,15 +50,29 @@ exports.fetchData = async (configOptions, reporter, cache, contentTypeOption) => } }; - -exports.fetchContentTypes = async (config, contentTypeOption) => { +exports.fetchContentTypes = async (config, contentTypeOption, cache) => { try { config.cdn = config.cdn ? config.cdn : 'https://cdn.contentstack.io/v3'; + const typePrefix = config.type_prefix || 'Contentstack'; + const lastFetchTimeKey = `${typePrefix}_${config.api_key}_${LAST_CONTENT_TYPE_FETCH_TIME}`; + const contentTypeFetchTimeQuery = await getLastContentTypeFetchTime(lastFetchTimeKey, cache); + + let currentTime = new Date(); + currentTime = currentTime.toISOString(); + const query = { + include_global_field_schema: true, + query: { ...contentTypeFetchTimeQuery }, + }; + const url = 'content_types'; const responseKey = 'content_types'; - const contentType = new OPTION_CLASS_MAPPING[contentTypeOption](); + const contentType = new OPTION_CLASS_MAPPING[contentTypeOption](query); const allContentTypes = await contentType.getPagedData(url, config, responseKey, getPagedData); + + // Set last fetch time here. + await cache.set(lastFetchTimeKey, currentTime); + return allContentTypes; } catch (error) { reporter.panic({ @@ -139,3 +153,14 @@ const getSyncData = async (url, config, query, responseKey, aggregatedResponse = } return aggregatedResponse; }; + +const getLastContentTypeFetchTime = async (lastFetchTimeKey, cache) => { + const lastFetch = await cache.get(lastFetchTimeKey); + const fetchTimeQuery = {}; + if (lastFetch) { + fetchTimeQuery.updated_at = { $gte: lastFetch }; + } else { + fetchTimeQuery.created_at = { $gte: DEFAULT_CONTENT_TYPE_FETCH_TIME }; + } + return fetchTimeQuery; +}; diff --git a/src/utils.js b/src/utils.js index 67971f3..c383be2 100644 --- a/src/utils.js +++ b/src/utils.js @@ -42,6 +42,8 @@ exports.checkIfUnsupportedFormat = data => { exports.SUPPORTED_FILES_COUNT = 'SUPPORTED_FILES_COUNT'; exports.IMAGE_REGEXP = new RegExp('https://(stag-images|(eu-)?images).(blz-)?contentstack.(io|com)/v3/assets/'); exports.ASSET_NODE_UIDS = 'ASSET_NODE_UIDS'; +exports.DEFAULT_CONTENT_TYPE_FETCH_TIME = '1970-01-01T00:00:00.000Z'; +exports.LAST_CONTENT_TYPE_FETCH_TIME = 'content-type-fetch-time'; exports.CODES = { SyncError: '10001', diff --git a/utils.js b/utils.js index 455a749..a7b8e12 100644 --- a/utils.js +++ b/utils.js @@ -45,6 +45,8 @@ exports.checkIfUnsupportedFormat = function (data) { exports.SUPPORTED_FILES_COUNT = 'SUPPORTED_FILES_COUNT'; exports.IMAGE_REGEXP = new RegExp('https://(stag-images|(eu-)?images).(blz-)?contentstack.(io|com)/v3/assets/'); exports.ASSET_NODE_UIDS = 'ASSET_NODE_UIDS'; +exports.DEFAULT_CONTENT_TYPE_FETCH_TIME = '1970-01-01T00:00:00.000Z'; +exports.LAST_CONTENT_TYPE_FETCH_TIME = 'content-type-fetch-time'; exports.CODES = { SyncError: '10001', APIError: '10002',