Migrate SQS to prisma and move queue messages to sqlite

This commit is contained in:
Matthew Bessette 2024-12-18 21:09:01 -05:00
parent 22da8d73d3
commit 095ecbd643
19 changed files with 221 additions and 189 deletions

29
package-lock.json generated
View File

@ -19,8 +19,7 @@
"joi": "^17.9.0", "joi": "^17.9.0",
"js2xmlparser": "^5.0.0", "js2xmlparser": "^5.0.0",
"rxjs": "^7.8.0", "rxjs": "^7.8.0",
"sqlite3": "^5.1.6", "sqlite3": "^5.1.6"
"uuidv4": "^6.2.13"
}, },
"devDependencies": { "devDependencies": {
"@types/express": "^4.17.17", "@types/express": "^4.17.17",
@ -690,12 +689,6 @@
"@types/send": "*" "@types/send": "*"
} }
}, },
"node_modules/@types/uuid": {
"version": "8.3.4",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz",
"integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==",
"license": "MIT"
},
"node_modules/@ungap/structured-clone": { "node_modules/@ungap/structured-clone": {
"version": "1.2.1", "version": "1.2.1",
"resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.1.tgz", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.1.tgz",
@ -4413,26 +4406,6 @@
"node": ">= 0.4.0" "node": ">= 0.4.0"
} }
}, },
"node_modules/uuidv4": {
"version": "6.2.13",
"resolved": "https://registry.npmjs.org/uuidv4/-/uuidv4-6.2.13.tgz",
"integrity": "sha512-AXyzMjazYB3ovL3q051VLH06Ixj//Knx7QnUSi1T//Ie3io6CpsPu9nVMOx5MoLWh6xV0B9J0hIaxungxXUbPQ==",
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
"license": "MIT",
"dependencies": {
"@types/uuid": "8.3.4",
"uuid": "8.3.2"
}
},
"node_modules/uuidv4/node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"license": "MIT",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/vary": { "node_modules/vary": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",

View File

@ -20,8 +20,7 @@
"joi": "^17.9.0", "joi": "^17.9.0",
"js2xmlparser": "^5.0.0", "js2xmlparser": "^5.0.0",
"rxjs": "^7.8.0", "rxjs": "^7.8.0",
"sqlite3": "^5.1.6", "sqlite3": "^5.1.6"
"uuidv4": "^6.2.13"
}, },
"devDependencies": { "devDependencies": {
"@types/express": "^4.17.17", "@types/express": "^4.17.17",

View File

@ -13,7 +13,7 @@ model Attribute {
name String name String
value String value String
@@index([arn]) @@unique([arn, name])
} }
model Audit { model Audit {
@ -25,22 +25,25 @@ model Audit {
} }
model Secret { model Secret {
versionId String @id versionId String @id
name String name String
description String? description String?
secretString String secretString String
accountId String accountId String
region String region String
createdAt DateTime @default(now()) createdAt DateTime @default(now())
deletionDate DateTime? deletionDate DateTime?
@@index([name]) @@index([name])
} }
model SnsTopic { model SnsTopic {
name String @id id Int @id @default(autoincrement())
name String
accountId String accountId String
region String region String
@@unique([accountId, region, name])
} }
model SnsTopicSubscription { model SnsTopicSubscription {
@ -52,11 +55,37 @@ model SnsTopicSubscription {
region String region String
} }
model SqsQueue {
id Int @id @default(autoincrement())
name String
accountId String
region String
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
messages SqsQueueMessage[]
@@unique([accountId, region, name])
}
model SqsQueueMessage {
id String @id
queueId Int
senderId String
message String
inFlightRelease DateTime
createdAt DateTime @default(now())
queue SqsQueue @relation(fields: [queueId], references: [id])
@@index([queueId])
}
model Tag { model Tag {
id Int @id @default(autoincrement()) id Int @id @default(autoincrement())
arn String arn String
name String name String
value String value String
@@index([arn]) @@unique([arn, name])
} }

View File

@ -1,5 +1,5 @@
import { randomUUID } from 'crypto';
import { Action } from './action.enum'; import { Action } from './action.enum';
import * as uuid from 'uuid';
import * as Joi from 'joi'; import * as Joi from 'joi';
export type AwsProperties = { export type AwsProperties = {
@ -34,7 +34,7 @@ export abstract class AbstractActionHandler<T = Record<string, string | number |
xmlns: "https://sns.amazonaws.com/doc/2010-03-31/" xmlns: "https://sns.amazonaws.com/doc/2010-03-31/"
}, },
ResponseMetadata: { ResponseMetadata: {
RequestId: uuid.v4(), RequestId: randomUUID(),
} }
} }

View File

@ -66,7 +66,7 @@ export class AttributesService {
const components = breakdownAwsQueryParam(param); const components = breakdownAwsQueryParam(param);
if (!components) { if (!components) {
return []; continue;
} }
const [type, _, idx, slot] = components; const [type, _, idx, slot] = components;

View File

@ -1,12 +1,11 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Attribute } from './attributes.entity';
import { AttributesService } from './attributes.service'; import { AttributesService } from './attributes.service';
import { Tag } from './tags.entity';
import { TagsService } from './tags.service'; import { TagsService } from './tags.service';
import { PrismaModule } from '../_prisma/prisma.module';
@Module({ @Module({
imports: [TypeOrmModule.forFeature([Attribute, Tag])], imports: [PrismaModule],
providers: [AttributesService, TagsService], providers: [AttributesService, TagsService],
exports: [AttributesService, TagsService], exports: [AttributesService, TagsService],
}) })

View File

@ -1,5 +0,0 @@
export interface CreateTagDto {
arn: string;
name: string;
value: string;
}

View File

@ -1,10 +1,10 @@
import { Provider } from '@nestjs/common'; import { InjectionToken, Provider } from '@nestjs/common';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import { ExistingActionHandlers } from './default-action-handler.constants'; import { ExistingActionHandlers } from './default-action-handler.constants';
import * as Joi from 'joi'; import * as Joi from 'joi';
import { AbstractActionHandler, Format } from '../abstract-action.handler'; import { AbstractActionHandler, Format } from '../abstract-action.handler';
export const DefaultActionHandlerProvider = (symbol, format: Format, actions: Action[]): Provider => ({ export const DefaultActionHandlerProvider = (symbol: InjectionToken, format: Format, actions: Action[]): Provider => ({
provide: symbol, provide: symbol,
useFactory: (existingActionHandlers: ExistingActionHandlers) => { useFactory: (existingActionHandlers: ExistingActionHandlers) => {
const cloned = { ...existingActionHandlers }; const cloned = { ...existingActionHandlers };

View File

@ -1,12 +1,14 @@
import { Provider } from '@nestjs/common'; import { InjectionToken, OptionalFactoryDependency, Provider } from '@nestjs/common';
import { AbstractActionHandler } from '../abstract-action.handler'; import { AbstractActionHandler } from '../abstract-action.handler';
import { Action } from '../action.enum';
import { ExistingActionHandlers } from './default-action-handler.constants'; import { ExistingActionHandlers } from './default-action-handler.constants';
export const ExistingActionHandlersProvider = (inject): Provider => ({ export const ExistingActionHandlersProvider = (inject: Array<InjectionToken | OptionalFactoryDependency>): Provider => ({
provide: ExistingActionHandlers, provide: ExistingActionHandlers,
useFactory: (...args: AbstractActionHandler[]) => args.reduce((m, h) => { useFactory: (...args: AbstractActionHandler[]) => args.reduce((m, h) => {
m[h.action] = h; m[h.action] = h;
return m; return m;
}, {}), }, {} as Record<Action, AbstractActionHandler>),
inject, inject,
}); });

View File

@ -1,6 +1,6 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import * as Joi from 'joi'; import * as Joi from 'joi';
import * as uuid from 'uuid'; import { randomUUID } from 'crypto';
import { PrismaService } from '../_prisma/prisma.service'; import { PrismaService } from '../_prisma/prisma.service';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
@ -44,7 +44,7 @@ export class PublishHandler extends AbstractActionHandler<QueryParams> {
throw new BadRequestException(); throw new BadRequestException();
} }
const MessageId = uuid.v4(); const MessageId = randomUUID();
const subscriptions = await this.prismaService.snsTopicSubscription.findMany({ where: { topicArn: arn } }); const subscriptions = await this.prismaService.snsTopicSubscription.findMany({ where: { topicArn: arn } });
const topicAttributes = await this.attributeService.getByArn(arn); const topicAttributes = await this.attributeService.getByArn(arn);

View File

@ -1,12 +1,12 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import * as Joi from 'joi';
import { Repository } from 'typeorm';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import * as Joi from 'joi';
import { TagsService } from '../aws-shared-entities/tags.service';
import { SqsQueue } from './sqs-queue.entity';
import { AttributesService } from '../aws-shared-entities/attributes.service'; import { AttributesService } from '../aws-shared-entities/attributes.service';
import { TagsService } from '../aws-shared-entities/tags.service';
import { SqsQueueEntryService } from './sqs-queue-entry.service';
import { SqsQueue } from './sqs-queue.entity';
type QueryParams = { type QueryParams = {
QueueName: string; QueueName: string;
@ -16,8 +16,7 @@ type QueryParams = {
export class CreateQueueHandler extends AbstractActionHandler<QueryParams> { export class CreateQueueHandler extends AbstractActionHandler<QueryParams> {
constructor( constructor(
@InjectRepository(SqsQueue) private readonly sqsQueueEntryService: SqsQueueEntryService,
private readonly sqsQueueRepo: Repository<SqsQueue>,
private readonly tagsService: TagsService, private readonly tagsService: TagsService,
private readonly attributeService: AttributesService, private readonly attributeService: AttributesService,
) { ) {
@ -32,11 +31,11 @@ export class CreateQueueHandler extends AbstractActionHandler<QueryParams> {
const { QueueName: name } = params; const { QueueName: name } = params;
const queue = await this.sqsQueueRepo.create({ const queue = await this.sqsQueueEntryService.createQueue({
name, name,
accountId: awsProperties.accountId, accountId: awsProperties.accountId,
region: awsProperties.region, region: awsProperties.region,
}).save(); });
const tags = TagsService.tagPairs(params); const tags = TagsService.tagPairs(params);
await this.tagsService.createMany(queue.arn, tags); await this.tagsService.createMany(queue.arn, tags);

View File

@ -1,9 +1,10 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import * as Joi from 'joi';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import * as Joi from 'joi';
import { SqsQueue } from './sqs-queue.entity';
import { SqsQueueEntryService } from './sqs-queue-entry.service'; import { SqsQueueEntryService } from './sqs-queue-entry.service';
import { SqsQueue } from './sqs-queue.entity';
type QueryParams = { type QueryParams = {
QueueUrl: string; QueueUrl: string;

View File

@ -1,9 +1,10 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import * as Joi from 'joi';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import * as Joi from 'joi';
import { SqsQueue } from './sqs-queue.entity';
import { SqsQueueEntryService } from './sqs-queue-entry.service'; import { SqsQueueEntryService } from './sqs-queue-entry.service';
import { SqsQueue } from './sqs-queue.entity';
type QueryParams = { type QueryParams = {
QueueUrl: string; QueueUrl: string;

View File

@ -1,13 +1,12 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import * as Joi from 'joi';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import * as Joi from 'joi';
import { AttributesService } from '../aws-shared-entities/attributes.service'; import { AttributesService } from '../aws-shared-entities/attributes.service';
import { InjectRepository } from '@nestjs/typeorm';
import { SqsQueue } from './sqs-queue.entity';
import { Repository } from 'typeorm';
import { SqsQueueEntryService } from './sqs-queue-entry.service';
import { TagsService } from '../aws-shared-entities/tags.service'; import { TagsService } from '../aws-shared-entities/tags.service';
import { SqsQueueEntryService } from './sqs-queue-entry.service';
import { SqsQueue } from './sqs-queue.entity';
type QueryParams = { type QueryParams = {
QueueUrl?: string, QueueUrl?: string,
@ -18,8 +17,6 @@ type QueryParams = {
export class DeleteQueueHandler extends AbstractActionHandler<QueryParams> { export class DeleteQueueHandler extends AbstractActionHandler<QueryParams> {
constructor( constructor(
@InjectRepository(SqsQueue)
private readonly sqsQueueRepo: Repository<SqsQueue>,
private readonly tagsService: TagsService, private readonly tagsService: TagsService,
private readonly attributeService: AttributesService, private readonly attributeService: AttributesService,
private readonly sqsQueueEntryService: SqsQueueEntryService, private readonly sqsQueueEntryService: SqsQueueEntryService,
@ -37,22 +34,15 @@ export class DeleteQueueHandler extends AbstractActionHandler<QueryParams> {
protected async handle(params: QueryParams, awsProperties: AwsProperties) { protected async handle(params: QueryParams, awsProperties: AwsProperties) {
const [accountId, name] = SqsQueue.tryGetAccountIdAndNameFromPathOrArn(params.QueueUrl ?? params.__path); const [accountId, name] = SqsQueue.tryGetAccountIdAndNameFromPathOrArn(params.QueueUrl ?? params.__path);
const queue = await this.sqsQueueRepo.findOne({ where: { accountId , name } }); const queue = await this.sqsQueueEntryService.findQueueByAccountIdAndName(accountId, name);
if(!queue) { if (!queue) {
throw new BadRequestException('ResourceNotFoundException'); throw new BadRequestException('ResourceNotFoundException');
} }
await this.sqsQueueEntryService.purge(accountId, name); await this.sqsQueueEntryService.purge(accountId, name);
await this.tagsService.deleteByArn(queue.arn); await this.tagsService.deleteByArn(queue.arn);
await this.attributeService.deleteByArn(queue.arn); await this.attributeService.deleteByArn(queue.arn);
await queue.remove(); await this.sqsQueueEntryService.deleteQueue(queue.id);
}
private async getAttributes(attributeNames: string[], queueArn: string) {
if (attributeNames.length === 0 || attributeNames.length === 1 && attributeNames[0] === 'All') {
return await this.attributeService.getByArn(queueArn);
}
return await this.attributeService.getByArnAndNames(queueArn, attributeNames);
} }
} }

View File

@ -1,25 +1,22 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import * as Joi from 'joi';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import * as Joi from 'joi';
import { AttributesService } from '../aws-shared-entities/attributes.service'; import { AttributesService } from '../aws-shared-entities/attributes.service';
import { InjectRepository } from '@nestjs/typeorm';
import { SqsQueue } from './sqs-queue.entity';
import { Repository } from 'typeorm';
import { SqsQueueEntryService } from './sqs-queue-entry.service'; import { SqsQueueEntryService } from './sqs-queue-entry.service';
import { SqsQueue } from './sqs-queue.entity';
type QueryParams = { type QueryParams = {
QueueUrl?: string, QueueUrl?: string,
'AttributeName.1'?: string; 'AttributeName.1'?: string;
__path: string; __path: string;
} } & Record<string, string>;
@Injectable() @Injectable()
export class GetQueueAttributesHandler extends AbstractActionHandler<QueryParams> { export class GetQueueAttributesHandler extends AbstractActionHandler<QueryParams> {
constructor( constructor(
@InjectRepository(SqsQueue)
private readonly sqsQueueRepo: Repository<SqsQueue>,
private readonly attributeService: AttributesService, private readonly attributeService: AttributesService,
private readonly sqsQueueEntryService: SqsQueueEntryService, private readonly sqsQueueEntryService: SqsQueueEntryService,
) { ) {
@ -42,23 +39,23 @@ export class GetQueueAttributesHandler extends AbstractActionHandler<QueryParams
l.push(params[k]); l.push(params[k]);
} }
return l; return l;
}, []); }, [] as string[]);
const [accountId, name] = SqsQueue.tryGetAccountIdAndNameFromPathOrArn(params.QueueUrl ?? params.__path); const [accountId, name] = SqsQueue.tryGetAccountIdAndNameFromPathOrArn(params.QueueUrl ?? params.__path);
const queue = await this.sqsQueueRepo.findOne({ where: { accountId , name } }); const queue = await this.sqsQueueEntryService.findQueueByAccountIdAndName(accountId, name);
if(!queue) { if(!queue) {
return; return;
} }
const queueMetrics = this.sqsQueueEntryService.metrics(queue.arn); const queueMetrics = await this.sqsQueueEntryService.metrics(queue.id);
const attributes = await this.getAttributes(attributeNames, queue.arn); const attributes = await this.getAttributes(attributeNames, queue.arn);
const attributeMap = attributes.reduce((m, a) => { const attributeMap = attributes.reduce((m, a) => {
m[a.name] = a.value; m[a.name] = a.value;
return m; return m;
}, {}); }, {} as Record<string, string>);
const response = { const response: Record<string, string> = {
...attributeMap, ...attributeMap,
ApproximateNumberOfMessages: `${queueMetrics.total}`, ApproximateNumberOfMessages: `${queueMetrics.total}`,
ApproximateNumberOfMessagesNotVisible: `${queueMetrics.inFlight}`, ApproximateNumberOfMessagesNotVisible: `${queueMetrics.inFlight}`,
@ -66,7 +63,8 @@ export class GetQueueAttributesHandler extends AbstractActionHandler<QueryParams
LastModifiedTimestamp: `${new Date(queue.updatedAt).getTime()}`, LastModifiedTimestamp: `${new Date(queue.updatedAt).getTime()}`,
QueueArn: queue.arn, QueueArn: queue.arn,
} }
return { Attribute: Object.keys(response).map(k => ({ return {
Attribute: Object.keys(response).map(k => ({
Name: k, Name: k,
Value: response[k], Value: response[k],
})) }))

View File

@ -1,20 +1,18 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import * as Joi from 'joi';
import { Repository } from 'typeorm';
import { PrismaService } from '../_prisma/prisma.service';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import * as Joi from 'joi';
import { SqsQueue } from './sqs-queue.entity'; import { SqsQueue } from './sqs-queue.entity';
type QueryParams = { type QueryParams = {}
}
@Injectable() @Injectable()
export class ListQueuesHandler extends AbstractActionHandler<QueryParams> { export class ListQueuesHandler extends AbstractActionHandler<QueryParams> {
constructor( constructor(
@InjectRepository(SqsQueue) private readonly prismaService: PrismaService,
private readonly sqsQueueRepo: Repository<SqsQueue>,
) { ) {
super(); super();
} }
@ -25,7 +23,14 @@ export class ListQueuesHandler extends AbstractActionHandler<QueryParams> {
protected async handle(params: QueryParams, awsProperties: AwsProperties) { protected async handle(params: QueryParams, awsProperties: AwsProperties) {
const queues = await this.sqsQueueRepo.find({ where: { accountId: awsProperties.accountId }}); const rawQueues = await this.prismaService.sqsQueue.findMany({
where: {
accountId: awsProperties.accountId,
region: awsProperties.region,
}
});
const queues = rawQueues.map(q => new SqsQueue(q));
return { return {
QueueUrl: queues.map((q) => q.getUrl(awsProperties.host)) QueueUrl: queues.map((q) => q.getUrl(awsProperties.host))

View File

@ -1,11 +1,11 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import * as Joi from 'joi';
import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler'; import { AbstractActionHandler, AwsProperties, Format } from '../abstract-action.handler';
import { Action } from '../action.enum'; import { Action } from '../action.enum';
import * as Joi from 'joi';
import { AttributesService } from '../aws-shared-entities/attributes.service'; import { AttributesService } from '../aws-shared-entities/attributes.service';
import { InjectRepository } from '@nestjs/typeorm'; import { SqsQueueEntryService } from './sqs-queue-entry.service';
import { SqsQueue } from './sqs-queue.entity'; import { SqsQueue } from './sqs-queue.entity';
import { Repository } from 'typeorm';
type QueryParams = { type QueryParams = {
'Attribute.Name': string; 'Attribute.Name': string;
@ -17,9 +17,8 @@ type QueryParams = {
export class SetQueueAttributesHandler extends AbstractActionHandler<QueryParams> { export class SetQueueAttributesHandler extends AbstractActionHandler<QueryParams> {
constructor( constructor(
@InjectRepository(SqsQueue)
private readonly sqsQueueRepo: Repository<SqsQueue>,
private readonly attributeService: AttributesService, private readonly attributeService: AttributesService,
private readonly sqsQueueEntryService: SqsQueueEntryService,
) { ) {
super(); super();
} }
@ -34,7 +33,7 @@ export class SetQueueAttributesHandler extends AbstractActionHandler<QueryParams
protected async handle(params: QueryParams, awsProperties: AwsProperties) { protected async handle(params: QueryParams, awsProperties: AwsProperties) {
const [accountId, name] = SqsQueue.getAccountIdAndNameFromPath(params.__path); const [accountId, name] = SqsQueue.getAccountIdAndNameFromPath(params.__path);
const queue = await this.sqsQueueRepo.findOne({ where: { accountId , name } }); const queue = await this.sqsQueueEntryService.findQueueByAccountIdAndName(accountId, name);
const attributes = SqsQueue.attributePairs(params); const attributes = SqsQueue.attributePairs(params);
if (params['Attribute.Name'] && params['Attribute.Value']) { if (params['Attribute.Name'] && params['Attribute.Value']) {

View File

@ -1,8 +1,9 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import { Prisma, SqsQueueMessage } from '@prisma/client';
import { Repository } from 'typeorm'; import { randomUUID } from 'crypto';
import { PrismaService } from '../_prisma/prisma.service';
import { SqsQueue } from './sqs-queue.entity'; import { SqsQueue } from './sqs-queue.entity';
import * as uuid from 'uuid';
type QueueEntry = { type QueueEntry = {
id: string; id: string;
@ -20,94 +21,115 @@ const FIFTEEN_SECONDS = 15 * 1000;
@Injectable() @Injectable()
export class SqsQueueEntryService { export class SqsQueueEntryService {
// Heavy use may require event-driven locking implementation
private queues: Record<string, QueueEntry[]> = {};
private queueObjectCache: Record<string, [Date, SqsQueue]> = {}; private queueObjectCache: Record<string, [Date, SqsQueue]> = {};
constructor( constructor(
@InjectRepository(SqsQueue) private readonly prismaService: PrismaService,
private readonly sqsQueueRepo: Repository<SqsQueue>,
) {} ) {}
async findQueueByAccountIdAndName(accountId: string, name: string): Promise<SqsQueue> { async findQueueByAccountIdAndName(accountId: string, name: string): Promise<SqsQueue | null> {
return await this.sqsQueueRepo.findOne({ where: { accountId, name } }); const prisma = await this.prismaService.sqsQueue.findFirst({ where: { accountId, name } });
return prisma ? new SqsQueue(prisma) : null;
} }
metrics(queueArn: string): Metrics { async createQueue(data: Prisma.SqsQueueCreateInput): Promise<SqsQueue> {
const prisma = await this.prismaService.sqsQueue.create({ data });
return new SqsQueue(prisma);
}
async deleteQueue(id: number): Promise<void> {
await this.prismaService.sqsQueue.delete({ where: { id }});
}
async metrics(queueId: number): Promise<Metrics> {
const now = new Date(); const now = new Date();
return this.getQueueList(queueArn).reduce<Metrics>((acc, e) => { const [total, inFlight] = await Promise.all([
acc.total += 1; this.prismaService.sqsQueueMessage.count({ where: { queueId }}),
acc.inFlight += e.inFlightReleaseDate > now ? 1 : 0; this.prismaService.sqsQueueMessage.count({ where: { queueId, inFlightRelease: { gt: now } }}),
return acc; ]);
}, { total: 0, inFlight: 0 });
return { total, inFlight }
} }
async publish(accountId: string, queueName: string, message: string) { async publish(accountId: string, queueName: string, message: string) {
const queue = await this.sqsQueueRepo.findOne({ where: { accountId, name: queueName }}); const prisma = await this.prismaService.sqsQueue.findFirst({ where: { accountId, name: queueName }});
if (!queue) { if (!prisma) {
console.warn(`Warning bad subscription to ${queueName}`); console.warn(`Warning bad subscription to ${queueName}`);
return; return;
} }
this.getQueueList(queue.arn).push({ const queue = new SqsQueue(prisma);
id: uuid.v4(),
queueArn: queue.arn, await this.prismaService.sqsQueueMessage.create({
senderId: accountId, data: {
message, id: randomUUID(),
inFlightReleaseDate: new Date(), queueId: queue.id,
createdAt: new Date(), senderId: accountId,
message,
inFlightRelease: new Date(),
}
}); });
} }
async receiveMessages(accountId: string, queueName: string, maxNumberOfMessages = 10, visabilityTimeout = 0): Promise<QueueEntry[]> { async receiveMessages(accountId: string, queueName: string, maxNumberOfMessages = 10, visabilityTimeout = 0): Promise<SqsQueueMessage[]> {
const queue = await this.getQueueHelper(accountId, queueName); const queue = await this.getQueueHelper(accountId, queueName);
const accessDate = new Date(); const accessDate = new Date();
const newInFlightReleaseDate = new Date(accessDate); const newInFlightReleaseDate = new Date(accessDate);
newInFlightReleaseDate.setSeconds(accessDate.getSeconds() + visabilityTimeout); newInFlightReleaseDate.setSeconds(accessDate.getSeconds() + visabilityTimeout);
const records = this.getQueueList(queue.arn).filter(e => e.inFlightReleaseDate <= accessDate).slice(0, maxNumberOfMessages - 1); const records = await this.prismaService.sqsQueueMessage.findMany({
records.forEach(e => e.inFlightReleaseDate = newInFlightReleaseDate); where: {
return records; queueId: queue.id,
inFlightRelease: {
lte: accessDate,
}
},
take: maxNumberOfMessages,
});
await this.prismaService.sqsQueueMessage.updateMany({
data: {
inFlightRelease: newInFlightReleaseDate
},
where: {
id: {
in: records.map(r => r.id)
}
}
});
return records.map(r => ({ ...r, inFlightRelease: newInFlightReleaseDate }));
} }
async deleteMessage(accountId: string, queueName: string, id: string): Promise<void> { async deleteMessage(id: string): Promise<void> {
await this.prismaService.sqsQueueMessage.delete({ where: { id }});
const queue = await this.getQueueHelper(accountId, queueName);
const records = this.getQueueList(queue.arn);
const loc = records.findIndex(r => r.id === id);
records.splice(loc, 1);
} }
async purge(accountId: string, queueName: string) { async purge(accountId: string, queueName: string) {
const queue = await this.sqsQueueRepo.findOne({ where: { accountId, name: queueName }}); const queue = await this.findQueueByAccountIdAndName(accountId, queueName);
this.queues[queue.arn] = [];
if (!queue) {
return;
}
await this.prismaService.sqsQueueMessage.deleteMany({ where: { queueId: queue.id }});
} }
private async getQueueHelper(accountId: string, queueName: string): Promise<SqsQueue> { private async getQueueHelper(accountId: string, queueName: string): Promise<SqsQueue> {
if (!this.queueObjectCache[`${accountId}/${queueName}`] || this.queueObjectCache[`${accountId}/${queueName}`][0] < new Date()) { if (!this.queueObjectCache[`${accountId}/${queueName}`] || this.queueObjectCache[`${accountId}/${queueName}`][0] < new Date()) {
this.queueObjectCache[`${accountId}/${queueName}`] = [new Date(Date.now() + FIFTEEN_SECONDS), await this.sqsQueueRepo.findOne({ where: { accountId, name: queueName }})]; const queue = await this.findQueueByAccountIdAndName(accountId, queueName);
if (!queue) {
throw new BadRequestException('Queue not found');
}
this.queueObjectCache[`${accountId}/${queueName}`] = [new Date(Date.now() + FIFTEEN_SECONDS), queue];
} }
const [_, queue] = this.queueObjectCache[`${accountId}/${queueName}`]; const [_, queue] = this.queueObjectCache[`${accountId}/${queueName}`];
if (!queue) {
throw new BadRequestException('Queue not found');
}
return queue; return queue;
} }
private getQueueList(arn: string): QueueEntry[] {
if (!this.queues[arn]) {
this.queues[arn] = [];
}
return this.queues[arn];
}
} }

View File

@ -1,4 +1,5 @@
import { BaseEntity, Column, CreateDateColumn, Entity, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import { SqsQueue as PrismaSqsQueue } from '@prisma/client';
import { getPathFromUrl } from '../util/get-path-from-url'; import { getPathFromUrl } from '../util/get-path-from-url';
const attributeSlotMap = { const attributeSlotMap = {
@ -6,23 +7,24 @@ const attributeSlotMap = {
'Value': 'value', 'Value': 'value',
} }
@Entity('sqs_queue') export class SqsQueue implements PrismaSqsQueue {
export class SqsQueue extends BaseEntity {
@PrimaryColumn({ name: 'name' }) id: number;
name: string; name: string;
@Column({ name: 'account_id', nullable: false })
accountId: string; accountId: string;
@Column({ name: 'region', nullable: false })
region: string; region: string;
createdAt: Date;
updatedAt: Date;
@CreateDateColumn() constructor(p: PrismaSqsQueue) {
createdAt: string; this.id = p.id;
this.name = p.name;
this.accountId = p.accountId;
this.region = p.region;
this.createdAt = p.createdAt;
this.updatedAt = p.updatedAt;
}
@UpdateDateColumn()
updatedAt: string;
get arn(): string { get arn(): string {
return `arn:aws:sqs:${this.region}:${this.accountId}:${this.name}`; return `arn:aws:sqs:${this.region}:${this.accountId}:${this.name}`;
@ -39,8 +41,8 @@ export class SqsQueue extends BaseEntity {
static getAccountIdAndNameFromArn(arn: string): [string, string] { static getAccountIdAndNameFromArn(arn: string): [string, string] {
const parts = arn.split(':'); const parts = arn.split(':');
const name = parts.pop(); const name = parts.pop() as string;
const accountId = parts.pop(); const accountId = parts.pop() as string;
return [accountId, name]; return [accountId, name];
} }
@ -53,18 +55,36 @@ export class SqsQueue extends BaseEntity {
} }
static attributePairs(queryParams: Record<string, string>): { key: string, value: string }[] { static attributePairs(queryParams: Record<string, string>): { key: string, value: string }[] {
const pairs = [null]; const pairs: { key: string, value: string }[] = [];
for (const param of Object.keys(queryParams)) { for (const param of Object.keys(queryParams)) {
const [type, idx, slot] = param.split('.'); const components = this.breakdownAwsQueryParam(param);
if (!components) {
continue;
}
const [type, idx, slot] = components;
if (type === 'Attribute') { if (type === 'Attribute') {
if (!pairs[+idx]) { if (!pairs[idx]) {
pairs[+idx] = { key: '', value: ''}; pairs[idx] = { key: '', value: ''};
} }
pairs[+idx][attributeSlotMap[slot]] = queryParams[param]; pairs[+idx][slot] = queryParams[param];
} }
} }
pairs.shift();
return pairs; return pairs;
} }
private static breakdownAwsQueryParam(paramKey: string): [string, number, 'key' | 'value'] | null {
const parts = paramKey.split('.');
if (parts.length !== 3) {
return null;
}
const [type, idx, slot] = parts;
return [type, +idx, attributeSlotMap[slot as 'Name' | 'Value'] as 'key' | 'value'];
}
} }