Skip to content

Commit 0aab9a7

Browse files
author
Uroš Marolt
committed
Merge branch 'main' into crowd-linux
2 parents 4a42c89 + e414c05 commit 0aab9a7

File tree

83 files changed

+10797
-3438
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+10797
-3438
lines changed

‎.github/workflows/lf-production-deploy-new.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ on:
1111
description: Deploy integration-sync-worker service?
1212
required: true
1313
type: boolean
14+
deploy_webhook_api:
15+
description: Deploy webhook-api service?
16+
required: true
17+
type: boolean
1418

1519
env:
1620
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -66,6 +70,27 @@ jobs:
6670
id: image
6771
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
6872

73+
build-and-push-webhook-api:
74+
runs-on: ubuntu-latest
75+
if: ${{ inputs.deploy_webhook_api }}
76+
outputs:
77+
image: ${{ steps.image.outputs.IMAGE }}
78+
defaults:
79+
run:
80+
shell: bash
81+
steps:
82+
- name: Check out repository code
83+
uses: actions/checkout@v2
84+
85+
- uses: ./.github/actions/build-docker-image
86+
id: image-builder
87+
with:
88+
image: webhook-api
89+
90+
- name: Set docker image output
91+
id: image
92+
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
93+
6994
deploy-search-sync-worker:
7095
needs: build-and-push-search-sync-worker
7196
runs-on: ubuntu-latest
@@ -101,3 +126,21 @@ jobs:
101126
service: integration-sync-worker
102127
image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
103128
cluster: ${{ env.CROWD_CLUSTER }}
129+
130+
deploy-webhook-api:
131+
needs: build-and-push-webhook-api
132+
runs-on: ubuntu-latest
133+
if: ${{ inputs.deploy_webhook_api }}
134+
defaults:
135+
run:
136+
shell: bash
137+
138+
steps:
139+
- name: Check out repository code
140+
uses: actions/checkout@v2
141+
142+
- uses: ./.github/actions/deploy-service
143+
with:
144+
service: webhook-api
145+
image: ${{ needs.build-and-push-webhook-api.outputs.image }}
146+
cluster: ${{ env.CROWD_CLUSTER }}

‎.github/workflows/lf-staging-deploy-webhook-api.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: LF Staging Deploy Webhook API
2+
3+
on:
4+
push:
5+
branches:
6+
- 'lf-staging/**'
7+
- 'lf-staging-**'
8+
paths:
9+
- 'services/libs/**'
10+
- 'services/apps/webhook_api/**'
11+
12+
env:
13+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
14+
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
15+
CROWD_CLUSTER: ${{ secrets.LF_STAGING_CLUSTER_NAME }}
16+
CROWD_ROLE_ARN: ${{ secrets.LF_STAGING_CLUSTER_ROLE_ARN }}
17+
AWS_ACCESS_KEY_ID: ${{ secrets.LF_AWS_ACCESS_KEY_ID }}
18+
AWS_SECRET_ACCESS_KEY: ${{ secrets.LF_AWS_SECRET_ACCESS_KEY }}
19+
AWS_REGION: ${{ secrets.LF_AWS_REGION }}
20+
SLACK_CHANNEL: deploys-lf-staging
21+
SLACK_WEBHOOK: ${{ secrets.LF_STAGING_SLACK_CHANNEL_HOOK }}
22+
23+
jobs:
24+
build-and-push:
25+
runs-on: ubuntu-latest
26+
outputs:
27+
image: ${{ steps.image.outputs.IMAGE }}
28+
defaults:
29+
run:
30+
shell: bash
31+
32+
steps:
33+
- name: Check out repository code
34+
uses: actions/checkout@v2
35+
36+
- uses: ./.github/actions/build-docker-image
37+
id: image-builder
38+
with:
39+
image: webhook-api
40+
41+
- name: Set docker image output
42+
id: image
43+
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
44+
45+
deploy-webhook-api:
46+
needs: build-and-push
47+
runs-on: ubuntu-latest
48+
defaults:
49+
run:
50+
shell: bash
51+
52+
steps:
53+
- name: Check out repository code
54+
uses: actions/checkout@v2
55+
56+
- uses: ./.github/actions/deploy-service
57+
with:
58+
service: webhook-api
59+
image: ${{ needs.build-and-push.outputs.image }}
60+
cluster: ${{ env.CROWD_CLUSTER }}

‎.github/workflows/production-deploy-new.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ on:
1111
description: Deploy integration-sync-worker service?
1212
required: true
1313
type: boolean
14+
deploy_webhook_api:
15+
description: Deploy webhook-api service?
16+
required: true
17+
type: boolean
1418

1519
env:
1620
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -66,6 +70,27 @@ jobs:
6670
id: image
6771
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
6872

73+
build-and-push-webhook-api:
74+
runs-on: ubuntu-latest
75+
if: ${{ inputs.deploy_webhook_api }}
76+
outputs:
77+
image: ${{ steps.image.outputs.IMAGE }}
78+
defaults:
79+
run:
80+
shell: bash
81+
steps:
82+
- name: Check out repository code
83+
uses: actions/checkout@v2
84+
85+
- uses: ./.github/actions/build-docker-image
86+
id: image-builder
87+
with:
88+
image: webhook-api
89+
90+
- name: Set docker image output
91+
id: image
92+
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
93+
6994
deploy-search-sync-worker:
7095
needs: build-and-push-search-sync-worker
7196
runs-on: ubuntu-latest
@@ -101,3 +126,21 @@ jobs:
101126
service: integration-sync-worker
102127
image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
103128
cluster: ${{ env.CROWD_CLUSTER }}
129+
130+
deploy-webhook-api:
131+
needs: build-and-push-webhook-api
132+
runs-on: ubuntu-latest
133+
if: ${{ inputs.deploy_webhook-api }}
134+
defaults:
135+
run:
136+
shell: bash
137+
138+
steps:
139+
- name: Check out repository code
140+
uses: actions/checkout@v2
141+
142+
- uses: ./.github/actions/deploy-service
143+
with:
144+
service: webhook-api
145+
image: ${{ needs.build-and-push-webhook-api.outputs.image }}
146+
cluster: ${{ env.CROWD_CLUSTER }}

‎.github/workflows/staging-deploy-webhook-api.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Staging Deploy Webhook API
2+
3+
on:
4+
push:
5+
branches:
6+
- 'staging/**'
7+
- 'staging-**'
8+
paths:
9+
- 'services/libs/**'
10+
- 'services/apps/webhook_api/**'
11+
12+
env:
13+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
14+
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
15+
CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
16+
CROWD_ROLE_ARN: ${{ secrets.STAGING_CLUSTER_ROLE_ARN }}
17+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
18+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
19+
AWS_REGION: ${{ secrets.AWS_REGION }}
20+
SLACK_CHANNEL: deploys-staging
21+
SLACK_WEBHOOK: ${{ secrets.STAGING_SLACK_CHANNEL_HOOK }}
22+
23+
jobs:
24+
build-and-push:
25+
runs-on: ubuntu-latest
26+
outputs:
27+
image: ${{ steps.image.outputs.IMAGE }}
28+
defaults:
29+
run:
30+
shell: bash
31+
32+
steps:
33+
- name: Check out repository code
34+
uses: actions/checkout@v2
35+
36+
- uses: ./.github/actions/build-docker-image
37+
id: image-builder
38+
with:
39+
image: webhook-api
40+
41+
- name: Set docker image output
42+
id: image
43+
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
44+
45+
deploy-webhook-api:
46+
needs: build-and-push
47+
runs-on: ubuntu-latest
48+
defaults:
49+
run:
50+
shell: bash
51+
52+
steps:
53+
- name: Check out repository code
54+
uses: actions/checkout@v2
55+
56+
- uses: ./.github/actions/deploy-service
57+
with:
58+
service: webhook-api
59+
image: ${{ needs.build-and-push.outputs.image }}
60+
cluster: ${{ env.CROWD_CLUSTER }}

‎backend/src/api/webhooks/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { safeWrap } from '../../middlewares/errorMiddleware'
22

33
export default (app) => {
4-
app.post(`/github`, safeWrap(require('./github').default))
4+
// app.post(`/github`, safeWrap(require('./github').default))
55
app.post(`/stripe`, safeWrap(require('./stripe').default))
66
app.post(`/sendgrid`, safeWrap(require('./sendgrid').default))
77
app.post(`/discourse/:tenantId`, safeWrap(require('./discourse').default))

‎backend/src/serverless/integrations/services/integrations/discourseIntegrationService.ts‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ export class DiscourseIntegrationService extends IntegrationServiceBase {
130130
}
131131

132132
const newStreams: IPendingStream[] = []
133-
const nextPageStream: IPendingStream | undefined = undefined
133+
let nextPageStream: IPendingStream | undefined
134134
const activities: AddActivitiesSingle[] = []
135135

136136
// another switch statement to handle the different types of results, helps with type safety
@@ -150,23 +150,35 @@ export class DiscourseIntegrationService extends IntegrationServiceBase {
150150
break
151151
case DiscourseStreamType.TOPICS_FROM_CATEGORY:
152152
const data2 = result.data as DiscourseTopicResponse
153-
data2.topic_list.topics.forEach((topic) => {
154-
newStreams.push({
155-
value: DiscourseStreamType.POSTS_FROM_TOPIC,
153+
if (data2?.topic_list?.topics?.length > 0) {
154+
data2.topic_list.topics.forEach((topic) => {
155+
newStreams.push({
156+
value: DiscourseStreamType.POSTS_FROM_TOPIC,
157+
metadata: {
158+
topicId: topic.id,
159+
topic_slug: topic.slug,
160+
page: 0,
161+
},
162+
})
163+
})
164+
165+
// we aslo need to trigger nextPageStream
166+
nextPageStream = {
167+
value: DiscourseStreamType.TOPICS_FROM_CATEGORY,
156168
metadata: {
157-
topicId: topic.id,
158-
topic_slug: topic.slug,
159-
page: 0,
169+
category_id: stream.metadata.category_id,
170+
category_slug: stream.metadata.category_slug,
171+
page: stream.metadata.page + 1,
160172
},
161-
})
162-
})
173+
}
174+
}
163175
break
164176
case DiscourseStreamType.POSTS_FROM_TOPIC:
165177
const data3 = result.data as DiscoursePostsFromTopicResponse
166-
const batchSize = 100
178+
const batchSize = 30
167179
const postBatches: number[][] = []
168180

169-
data3.post_stream.stream.forEach((postId, index) => {
181+
data3?.post_stream?.stream?.forEach((postId, index) => {
170182
if (index % batchSize === 0) {
171183
postBatches.push([])
172184
}
@@ -193,7 +205,7 @@ export class DiscourseIntegrationService extends IntegrationServiceBase {
193205
// just add the activities
194206
const data4 = result.data as DiscoursePostsByIdsResponse
195207
const { topicId, lastIdInPreviousBatch } = stream.metadata
196-
const posts = data4.post_stream.posts
208+
const posts = data4?.post_stream?.posts
197209
for (const post of posts) {
198210
if (usernameIsBot(post.username)) {
199211
/* eslint-disable no-continue */

‎backend/src/serverless/integrations/usecases/discourse/getCategories.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import axios, { AxiosRequestConfig } from 'axios'
22
import { Logger } from '@crowd/logging'
3+
import { RateLimitError } from '@crowd/types'
34
import type { DiscourseConnectionParams } from '../../types/discourseTypes'
45
import { DiscourseCategoryResponse } from '../../types/discourseTypes'
56

@@ -24,6 +25,10 @@ export const getDiscourseCategories = async (
2425
const response = await axios(config)
2526
return response.data
2627
} catch (err) {
28+
if (err.response && err.response.status === 429) {
29+
// wait 5 mins
30+
throw new RateLimitError(5 * 60, 'discourse/getcategories')
31+
}
2732
logger.error({ err, params }, 'Error while getting Discourse categories')
2833
throw err
2934
}

‎backend/src/serverless/integrations/usecases/discourse/getPostsByIds.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import axios, { AxiosRequestConfig } from 'axios'
22
import { Logger } from '@crowd/logging'
3+
import { RateLimitError } from '@crowd/types'
34
import type { DiscourseConnectionParams } from '../../types/discourseTypes'
45
import { DiscoursePostsByIdsResponse, DiscoursePostsByIdsInput } from '../../types/discourseTypes'
56

@@ -47,6 +48,10 @@ export const getDiscoursePostsByIds = async (
4748
const response = await axios(config)
4849
return response.data
4950
} catch (err) {
51+
if (err.response && err.response.status === 429) {
52+
// wait 5 mins
53+
throw new RateLimitError(5 * 60, 'discourse/getpostsbyids')
54+
}
5055
logger.error({ err, params, input }, 'Error while getting posts by ids from Discourse ')
5156
throw err
5257
}

‎backend/src/serverless/integrations/usecases/discourse/getPostsFromTopic.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import axios, { AxiosRequestConfig } from 'axios'
22
import { Logger } from '@crowd/logging'
3+
import { RateLimitError } from '@crowd/types'
34
import type { DiscourseConnectionParams } from '../../types/discourseTypes'
45
import { DiscoursePostsFromTopicResponse, DiscoursePostsInput } from '../../types/discourseTypes'
56

@@ -28,6 +29,10 @@ export const getDiscoursePostsFromTopic = async (
2829
const response = await axios(config)
2930
return response.data
3031
} catch (err) {
32+
if (err.response && err.response.status === 429) {
33+
// wait 5 mins
34+
throw new RateLimitError(5 * 60, 'discourse/getpostsfromtopic')
35+
}
3136
logger.error({ err, params, input }, 'Error while getting posts from topic from Discourse ')
3237
throw err
3338
}

‎backend/src/serverless/integrations/usecases/discourse/getTopics.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import axios, { AxiosRequestConfig } from 'axios'
22
import { Logger } from '@crowd/logging'
3+
import { RateLimitError } from '@crowd/types'
34
import type { DiscourseConnectionParams } from '../../types/discourseTypes'
45
import { DiscourseCategoryResponse, DiscourseTopicsInput } from '../../types/discourseTypes'
56

@@ -28,6 +29,10 @@ export const getDiscourseTopics = async (
2829
const response = await axios(config)
2930
return response.data
3031
} catch (err) {
32+
if (err.response && err.response.status === 429) {
33+
// wait 5 mins
34+
throw new RateLimitError(5 * 60, 'discourse/gettopics')
35+
}
3136
logger.error({ err, params }, 'Error while getting Discourse categories')
3237
throw err
3338
}

0 commit comments

Comments
 (0)