From 861b1e3dbba8b64f75473883ff0cbe6cd210c17f Mon Sep 17 00:00:00 2001 From: Gwendal Date: Thu, 22 Jul 2021 18:36:07 +0200 Subject: [PATCH 1/3] feat: hash code at submission --- package-lock.json | 20 +++++++++++++++++-- package.json | 2 ++ src/hash/hash.module.ts | 8 ++++++++ src/hash/hash.service.ts | 9 +++++++++ .../1626968543120-AddSubmissionHashCode.ts | 14 +++++++++++++ src/submissions/submission.entity.ts | 5 +++++ src/submissions/submissions.service.ts | 6 ++++++ 7 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 src/hash/hash.module.ts create mode 100644 src/hash/hash.service.ts create mode 100644 src/migrations/1626968543120-AddSubmissionHashCode.ts diff --git a/package-lock.json b/package-lock.json index 7458556..7f83e76 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1788,6 +1788,14 @@ "@babel/types": "^7.3.0" } }, + "@types/bcrypt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/bcrypt/-/bcrypt-5.0.0.tgz", + "integrity": "sha512-agtcFKaruL8TmcvqbndlqHPSJgsolhf/qPWchFlgnW1gECTN/nKbFcoFnvKAQRFfKbh+BO6A3SWdJu9t+xF3Lw==", + "requires": { + "@types/node": "*" + } + }, "@types/body-parser": { "version": "1.19.1", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.1.tgz", @@ -1941,8 +1949,7 @@ "@types/node": { "version": "16.3.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.3.tgz", - "integrity": "sha512-8h7k1YgQKxKXWckzFCMfsIwn0Y61UK6tlD6y2lOb3hTOIMlK3t9/QwHOhc81TwU+RMf0As5fj7NPjroERCnejQ==", - "dev": true + "integrity": "sha512-8h7k1YgQKxKXWckzFCMfsIwn0Y61UK6tlD6y2lOb3hTOIMlK3t9/QwHOhc81TwU+RMf0As5fj7NPjroERCnejQ==" }, "@types/parse-json": { "version": "4.0.0", @@ -2980,6 +2987,15 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "bcrypt": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-5.0.1.tgz", + "integrity": "sha512-9BTgmrhZM2t1bNuDtrtIMVSmmxZBrJ71n8Wg+YgdjHuIWYF7SjjmCPZFB+/5i/o/PIeRpwVJR3P+NrpIItUjqw==", + "requires": { + "@mapbox/node-pre-gyp": "^1.0.0", + "node-addon-api": "^3.1.0" + } + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", diff --git a/package.json b/package.json index b3949e2..8df9122 100644 --- a/package.json +++ b/package.json @@ -44,10 +44,12 @@ "@nestjs/swagger": "^5.0.4", "@nestjs/terminus": "^7.2.0", "@nestjs/typeorm": "^8.0.1", + "@types/bcrypt": "^5.0.0", "amqp-connection-manager": "^3.2.2", "amqplib": "^0.8.0", "antlr4ts": "^0.5.0-alpha.4", "argon2": "^0.28.2", + "bcrypt": "^5.0.1", "cache-manager": "^3.4.4", "class-transformer": "^0.4.0", "class-validator": "^0.13.1", diff --git a/src/hash/hash.module.ts b/src/hash/hash.module.ts new file mode 100644 index 0000000..b411f91 --- /dev/null +++ b/src/hash/hash.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { HashService } from './hash.service'; + +@Module({ + providers: [HashService], + exports: [HashService], +}) +export class HashModule {} diff --git a/src/hash/hash.service.ts b/src/hash/hash.service.ts new file mode 100644 index 0000000..5c8802f --- /dev/null +++ b/src/hash/hash.service.ts @@ -0,0 +1,9 @@ +import { Injectable } from '@nestjs/common'; +import bcrypt from 'bcrypt'; + +@Injectable() +export class HashService { + public async hashCode(source: string): Promise { + return bcrypt.hash(source, 8); + } +} diff --git a/src/migrations/1626968543120-AddSubmissionHashCode.ts b/src/migrations/1626968543120-AddSubmissionHashCode.ts new file mode 100644 index 0000000..3f6bf5f --- /dev/null +++ b/src/migrations/1626968543120-AddSubmissionHashCode.ts @@ -0,0 +1,14 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class AddSubmissionHashCode1626968543120 implements MigrationInterface { + name = 'AddSubmissionHashCode1626968543120' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "submissions" ADD "codeHash" character varying`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "submissions" DROP COLUMN "codeHash"`); + } + +} diff --git a/src/submissions/submission.entity.ts b/src/submissions/submission.entity.ts index 348fee7..105d243 100644 --- a/src/submissions/submission.entity.ts +++ b/src/submissions/submission.entity.ts @@ -36,6 +36,11 @@ export class Submission extends BaseEntity { @Column() code: string; + @ApiProperty() + @jsonMember + @Column({ nullable: true }) + codeHash: string; + @jsonMember @ApiProperty() @Column() diff --git a/src/submissions/submissions.service.ts b/src/submissions/submissions.service.ts index a2e803e..7af0040 100644 --- a/src/submissions/submissions.service.ts +++ b/src/submissions/submissions.service.ts @@ -16,6 +16,7 @@ import { InsertSubmissionDTO } from './dto/insert-submission-dto'; import { JobStatusDTO } from './dto/job-status.dto'; import { Submission } from './submission.entity'; import { BenchmarkIdDto } from '../benchmarks/dto/benchmarkId.dto'; +import { HashService } from '../hash/hash.service'; @Injectable() export class SubmissionsService { @@ -27,12 +28,17 @@ export class SubmissionsService { private benchmarkService: BenchmarkService, ) {} + private hashService = new HashService(); + async create( insertSubmissionDTO: InsertSubmissionDTO, lintScore: number, qualityScore: number, ): Promise { const submission = new Submission(insertSubmissionDTO); + submission.codeHash = await this.hashService.hashCode( + insertSubmissionDTO.code, + ); submission.status = 'waiting'; submission.lintScore = lintScore; submission.qualityScore = qualityScore; From 6d87bc6e68a34feb9edea466347a405bfb1d922b Mon Sep 17 00:00:00 2001 From: Gwendal Date: Thu, 22 Jul 2021 23:38:07 +0200 Subject: [PATCH 2/3] feat: find duplicated submission --- src/hash/hash.service.ts | 7 +++ src/submissions/submission.entity.ts | 13 +++++- src/submissions/submissions.controller.ts | 3 ++ src/submissions/submissions.service.ts | 53 ++++++++++++++++++++--- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/hash/hash.service.ts b/src/hash/hash.service.ts index 5c8802f..06f22f6 100644 --- a/src/hash/hash.service.ts +++ b/src/hash/hash.service.ts @@ -6,4 +6,11 @@ export class HashService { public async hashCode(source: string): Promise { return bcrypt.hash(source, 8); } + + public async compareSourceToHash( + source: string, + hash: string, + ): Promise { + return bcrypt.compare(source, hash); + } } diff --git a/src/submissions/submission.entity.ts b/src/submissions/submission.entity.ts index 105d243..210993a 100644 --- a/src/submissions/submission.entity.ts +++ b/src/submissions/submission.entity.ts @@ -1,12 +1,14 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { ApiProperty } from '@nestjs/swagger'; import { User } from 'src/users/user.entity'; -import { jsonMember, jsonObject } from 'typedjson'; +import { jsonArrayMember, jsonMember, jsonObject } from 'typedjson'; import { BaseEntity, Column, CreateDateColumn, Entity, + JoinTable, + ManyToMany, ManyToOne, PrimaryGeneratedColumn, UpdateDateColumn, @@ -108,4 +110,13 @@ export class Submission extends BaseEntity { }) @ApiProperty({ type: () => Benchmark }) benchmark: Benchmark; + + @ApiProperty() + @JoinTable() + @jsonArrayMember(() => Submission) + @ManyToMany( + () => Submission, + (submission) => submission.duplicatedSubmissions, + ) + duplicatedSubmissions: Submission[]; } diff --git a/src/submissions/submissions.controller.ts b/src/submissions/submissions.controller.ts index e14acb7..9149ae6 100644 --- a/src/submissions/submissions.controller.ts +++ b/src/submissions/submissions.controller.ts @@ -82,6 +82,9 @@ export class SubmissionsController { // Since this endpoint is used for polling, the service will fetch from cache first and fallback to DB const submission = await this.submissionsService.findOne(findSubmissionDTO); + // @ts-ignore + console.log(submission.duplicatedSubmissions); + if (!submission) { throw new NotFoundException(); } diff --git a/src/submissions/submissions.service.ts b/src/submissions/submissions.service.ts index 7af0040..60dd611 100644 --- a/src/submissions/submissions.service.ts +++ b/src/submissions/submissions.service.ts @@ -17,6 +17,7 @@ import { JobStatusDTO } from './dto/job-status.dto'; import { Submission } from './submission.entity'; import { BenchmarkIdDto } from '../benchmarks/dto/benchmarkId.dto'; import { HashService } from '../hash/hash.service'; +import { Benchmark } from '../benchmarks/benchmark.entity'; @Injectable() export class SubmissionsService { @@ -36,13 +37,6 @@ export class SubmissionsService { qualityScore: number, ): Promise { const submission = new Submission(insertSubmissionDTO); - submission.codeHash = await this.hashService.hashCode( - insertSubmissionDTO.code, - ); - submission.status = 'waiting'; - submission.lintScore = lintScore; - submission.qualityScore = qualityScore; - const benchmark = await this.benchmarkService.findOne( insertSubmissionDTO.benchmarkId, ); @@ -51,6 +45,18 @@ export class SubmissionsService { submission.benchmark = benchmark; } + submission.codeHash = await this.hashService.hashCode( + insertSubmissionDTO.code, + ); + submission.status = 'waiting'; + submission.lintScore = lintScore; + submission.qualityScore = qualityScore; + submission.duplicatedSubmissions = await this.findSameSubmissions( + insertSubmissionDTO.code, + submission.benchmark, + insertSubmissionDTO.language, + ); + return submission.save(); } @@ -188,6 +194,39 @@ export class SubmissionsService { }); } + async findSameSubmissions( + source: string, + benchmark: Benchmark, + language: string, + ): Promise { + // Get all submissions in bench for the languages + const submissions: Submission[] = await this.submissionsRepository.find({ + where: [ + { + benchmark, + language, + }, + ], + order: { qualityScore: 'DESC' }, + }); + + const sameSubmissions: Submission[] = []; + // for each check with compare function + + submissions.forEach((submission) => { + this.hashService + .compareSourceToHash(source, submission.codeHash) + .then((value) => { + if (value) { + sameSubmissions.push(submission); + } + }) + .catch((reason) => console.log(reason)); + }); + + return sameSubmissions; + } + async languageMatcher(language: string): Promise { switch (language) { case 'python': From 710a211f660ea3035777e95c8695b5579dc0c4b4 Mon Sep 17 00:00:00 2001 From: Gwendal Date: Fri, 23 Jul 2021 02:47:14 +0200 Subject: [PATCH 3/3] fix: apply self reference for duplicated submissions --- .../1626996924860-AddDuplicatedSubmission.ts | 16 ++++++++++++++++ src/submissions/submission.entity.ts | 15 +++++++-------- src/submissions/submissions.service.ts | 13 ++++++++++++- 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 src/migrations/1626996924860-AddDuplicatedSubmission.ts diff --git a/src/migrations/1626996924860-AddDuplicatedSubmission.ts b/src/migrations/1626996924860-AddDuplicatedSubmission.ts new file mode 100644 index 0000000..911eb84 --- /dev/null +++ b/src/migrations/1626996924860-AddDuplicatedSubmission.ts @@ -0,0 +1,16 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class AddDuplicatedSubmission1626996924860 implements MigrationInterface { + name = 'AddDuplicatedSubmission1626996924860' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "submissions" ADD "selfId" uuid`); + await queryRunner.query(`ALTER TABLE "submissions" ADD CONSTRAINT "FK_8cc0cfc3583dbe9e530311136f6" FOREIGN KEY ("selfId") REFERENCES "submissions"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "submissions" DROP CONSTRAINT "FK_8cc0cfc3583dbe9e530311136f6"`); + await queryRunner.query(`ALTER TABLE "submissions" DROP COLUMN "selfId"`); + } + +} diff --git a/src/submissions/submission.entity.ts b/src/submissions/submission.entity.ts index 210993a..8181225 100644 --- a/src/submissions/submission.entity.ts +++ b/src/submissions/submission.entity.ts @@ -1,15 +1,14 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { ApiProperty } from '@nestjs/swagger'; import { User } from 'src/users/user.entity'; -import { jsonArrayMember, jsonMember, jsonObject } from 'typedjson'; +import { jsonMember, jsonObject } from 'typedjson'; import { BaseEntity, Column, CreateDateColumn, Entity, - JoinTable, - ManyToMany, ManyToOne, + OneToMany, PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm'; @@ -111,12 +110,12 @@ export class Submission extends BaseEntity { @ApiProperty({ type: () => Benchmark }) benchmark: Benchmark; - @ApiProperty() - @JoinTable() - @jsonArrayMember(() => Submission) - @ManyToMany( - () => Submission, + @ManyToOne( + (type) => Submission, (submission) => submission.duplicatedSubmissions, ) + self: Submission; + + @OneToMany((type) => Submission, (submission) => submission.self) duplicatedSubmissions: Submission[]; } diff --git a/src/submissions/submissions.service.ts b/src/submissions/submissions.service.ts index 60dd611..3607438 100644 --- a/src/submissions/submissions.service.ts +++ b/src/submissions/submissions.service.ts @@ -56,6 +56,7 @@ export class SubmissionsService { submission.benchmark, insertSubmissionDTO.language, ); + // submission.self = submission; return submission.save(); } @@ -115,7 +116,16 @@ export class SubmissionsService { // } // Fallback to DB - return this.submissionsRepository.findOne({ id: queriedSubmission.id }); + return this.submissionsRepository.findOne({ + where: [ + { + id: queriedSubmission.id, + }, + ], + relations: ['duplicatedSubmissions'], + order: { createdAt: 'DESC' }, + }); + // return this.submissionsRepository.findOne({ id: queriedSubmission.id }); } @RabbitSubscribe({ @@ -190,6 +200,7 @@ export class SubmissionsService { language: matchedLanguage, }, ], + relations: ['duplicatedSubmissions'], order: { createdAt: 'DESC' }, }); }