From c4d7e7c451f4b5a97a928c76d7c1a71a78579b59 Mon Sep 17 00:00:00 2001 From: Dani Gellis Date: Tue, 19 Nov 2019 15:28:33 -0800 Subject: [PATCH 1/3] when users are queued, crawl users' events, along with their repos. 3 tests failing currently --- lib/visitorMap.js | 3 ++- providers/fetcher/githubFetcher.js | 2 +- providers/fetcher/githubProcessor.js | 1 + routes/requests.js | 9 +++++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/visitorMap.js b/lib/visitorMap.js index 7af952c..b6e6548 100644 --- a/lib/visitorMap.js +++ b/lib/visitorMap.js @@ -248,7 +248,8 @@ const repo = { const user = { _type: 'user', - repos: collection(repo) + repos: collection(repo), + events: collection(event) }; const org = { diff --git a/providers/fetcher/githubFetcher.js b/providers/fetcher/githubFetcher.js index 7d39fe6..c6e30fe 100644 --- a/providers/fetcher/githubFetcher.js +++ b/providers/fetcher/githubFetcher.js @@ -272,7 +272,7 @@ class GitHubFetcher { teams: { tokenTraits: ['admin'] }, team: { tokenTraits: ['admin'] }, members: { tokenTraits: ['admin'] }, - events: { tokenTraits: ['admin'] }, + events: { tokenTraits: [['admin'], ['public']] }, collaborators: { tokenTraits: ['admin'], headers: { Accept: 'application/vnd.github.korra-preview' } }, reviews: { headers: { Accept: 'application/vnd.github.black-cat-preview+json' } }, review: { headers: { Accept: 'application/vnd.github.black-cat-preview+json' } }, diff --git a/providers/fetcher/githubProcessor.js b/providers/fetcher/githubProcessor.js index 57dc791..70fc910 100644 --- a/providers/fetcher/githubProcessor.js +++ b/providers/fetcher/githubProcessor.js @@ -151,6 +151,7 @@ class GitHubProcessor { request.linkSiblings('urn:users'); this._addCollection(request, 'repos', 'repo'); + this._addCollection(request, 'events', null, document.events_url.replace('{/privacy}', '')); return document; } diff --git a/routes/requests.js b/routes/requests.js index df81108..5cf6216 100644 --- a/routes/requests.js +++ b/routes/requests.js @@ -66,8 +66,13 @@ function buildRequestFromSpec(spec) { let crawlType = null; let crawlUrl = 'https://api.github.com/'; if (spec.indexOf('/') > -1) { - crawlType = 'repo'; - crawlUrl += 'repos/' + spec; + if(spec.slice(0, 6)==="users/") { + crawlType = 'user'; + crawlUrl += spec; + } else { + crawlType = 'repo'; + crawlUrl += 'repos/' + spec; + } } else { crawlType = 'org'; crawlUrl += 'orgs/' + spec; From b09b9ff909725acc75ce29f85885d0c815fde935 Mon Sep 17 00:00:00 2001 From: Dani Gellis Date: Thu, 21 Nov 2019 13:18:11 -0800 Subject: [PATCH 2/3] two of three tests passing --- test/unit/crawlerTests.js | 2 +- test/unit/gitHubProcessorTests.js | 5 ++++- test/unit/processingTests.js | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/unit/crawlerTests.js b/test/unit/crawlerTests.js index 66c14c8..6c312f9 100644 --- a/test/unit/crawlerTests.js +++ b/test/unit/crawlerTests.js @@ -759,7 +759,7 @@ describe('Crawler whole meal deal', () => { const request = new Request('user', 'http://test.com/users/user1'); request.policy = TraversalPolicy.reload('user'); normal.requests = [request]; - crawler.fetcher.responses = [createResponse({ id: 42, repos_url: 'http://test.com/users/user1/repos' })]; + crawler.fetcher.responses = [createResponse({ id: 42, repos_url: 'http://test.com/users/user1/repos', events_url: 'http://test.com/users/user1/events' })]; return Q.try(() => { return crawler.processOne({ name: 'test' }); }).then( () => { expect(normal.pop.callCount).to.be.equal(1); diff --git a/test/unit/gitHubProcessorTests.js b/test/unit/gitHubProcessorTests.js index d226d57..437e724 100644 --- a/test/unit/gitHubProcessorTests.js +++ b/test/unit/gitHubProcessorTests.js @@ -179,6 +179,7 @@ describe('User processing', () => { _metadata: { links: {} }, id: 9, repos_url: 'http://repos', + events_url: 'https://api.github.com/users/testuser/events', }; const processor = new GitHubProcessor(); @@ -187,12 +188,14 @@ describe('User processing', () => { const links = { self: { href: 'urn:user:9', type: 'resource' }, siblings: { href: 'urn:users', type: 'collection' }, - repos: { href: 'urn:user:9:repos', type: 'collection' } + repos: { href: 'urn:user:9:repos', type: 'collection' }, + events: {href: 'urn:user:9:events', type: 'collection'} } expectLinks(document._metadata.links, links); const expected = [ { type: 'repos', url: 'http://repos', qualifier: 'urn:user:9', path: '/repos' }, + { type: 'events', url: 'https://api.github.com/users/testuser/events', qualifier: 'urn:user:9', path: '/events'}, ]; expectQueued(queue, expected); }); diff --git a/test/unit/processingTests.js b/test/unit/processingTests.js index f75f0c5..362bc75 100644 --- a/test/unit/processingTests.js +++ b/test/unit/processingTests.js @@ -19,7 +19,7 @@ describe('Simple processing', () => { .then(processOne) .then(checkDoc.bind(null, 'org', 'urn:org:1', 4)) .then(processOne) - .then(checkDoc.bind(null, 'user', 'urn:user:1', 1)) + .then(checkDoc.bind(null, 'user', 'urn:user:1', 2)) .then(processOne) .then(checkDoc.bind(null, 'repos', 'urn:org:1:repos:page:1', 0)) .then(processOne) @@ -29,7 +29,7 @@ describe('Simple processing', () => { .then(processOne) .then(checkDoc.bind(null, 'repos', 'urn:org:1:repos:page:1', 0)) .then(processOne) - .then(checkDoc.bind(null, 'user', 'urn:user:2', 1)) // queued as a member of the org + .then(checkDoc.bind(null, 'user', 'urn:user:2', 2)) // queued as a member of the org .then(processOne) .then(checkDoc.bind(null, 'team', 'urn:team:20', 2)) .then(processOne) @@ -133,6 +133,7 @@ const resources = { "id": 1, "url": "https://api.github.com/users/test", "repos_url": "https://api.github.com/users/test/repos", + "events_url": "https://api.github.com/users/test/events", } }, 'https://api.github.com/users/user2': { @@ -141,6 +142,7 @@ const resources = { "id": 2, "url": "https://api.github.com/users/user2", "repos_url": "https://api.github.com/users/user2/repos", + "events_url": "https://api.github.com/users/user2/events", } }, 'https://api.github.com/users/user2/repos': { From 0c3428905a4699014633856d4a65ec8478356e9a Mon Sep 17 00:00:00 2001 From: Dani Gellis Date: Wed, 4 Dec 2019 11:20:17 -0800 Subject: [PATCH 3/3] change events test url to match repos test url --- test/unit/gitHubProcessorTests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/gitHubProcessorTests.js b/test/unit/gitHubProcessorTests.js index 437e724..57e8366 100644 --- a/test/unit/gitHubProcessorTests.js +++ b/test/unit/gitHubProcessorTests.js @@ -179,7 +179,7 @@ describe('User processing', () => { _metadata: { links: {} }, id: 9, repos_url: 'http://repos', - events_url: 'https://api.github.com/users/testuser/events', + events_url: 'http://events', }; const processor = new GitHubProcessor(); @@ -195,7 +195,7 @@ describe('User processing', () => { const expected = [ { type: 'repos', url: 'http://repos', qualifier: 'urn:user:9', path: '/repos' }, - { type: 'events', url: 'https://api.github.com/users/testuser/events', qualifier: 'urn:user:9', path: '/events'}, + { type: 'events', url: 'http://events', qualifier: 'urn:user:9', path: '/events'}, ]; expectQueued(queue, expected); });