Fix artwork upload pipeline

This commit is contained in:
diyaa
2026-05-31 01:20:56 +02:00
parent 7b1952794c
commit 18ed79e3c4
24 changed files with 1652 additions and 56 deletions
+73
View File
@@ -0,0 +1,73 @@
import { ValidationPipe } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import type { NestExpressApplication } from '@nestjs/platform-express';
const setTitle = jest.fn().mockReturnThis();
const setDescription = jest.fn().mockReturnThis();
const setVersion = jest.fn().mockReturnThis();
const build = jest.fn().mockReturnValue({});
const createDocument = jest.fn().mockReturnValue({});
const setup = jest.fn();
jest.mock('@nestjs/core', () => ({
NestFactory: {
create: jest.fn(),
},
}));
jest.mock('@nestjs/swagger', () => {
const actual = jest.requireActual('@nestjs/swagger');
return {
...actual,
DocumentBuilder: jest.fn().mockImplementation(() => ({
setTitle,
setDescription,
setVersion,
build,
})),
SwaggerModule: {
...actual.SwaggerModule,
createDocument,
setup,
},
};
});
import { API_JSON_BODY_LIMIT, createApp } from './app.factory';
describe('createApp', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('registers a larger JSON body parser for artwork finalize payloads', async () => {
const app = {
useBodyParser: jest.fn(),
setGlobalPrefix: jest.fn(),
enableVersioning: jest.fn(),
useGlobalPipes: jest.fn(),
} as unknown as NestExpressApplication;
(NestFactory.create as jest.Mock).mockResolvedValue(app);
await createApp();
expect(NestFactory.create).toHaveBeenCalledWith(
expect.any(Function),
expect.objectContaining({
bufferLogs: true,
bodyParser: false,
}),
);
expect((app as any).useBodyParser).toHaveBeenNthCalledWith(1, 'json', {
limit: API_JSON_BODY_LIMIT,
});
expect((app as any).useBodyParser).toHaveBeenNthCalledWith(2, 'urlencoded', {
extended: true,
limit: API_JSON_BODY_LIMIT,
});
expect((app as any).useGlobalPipes).toHaveBeenCalledWith(
expect.any(ValidationPipe),
);
});
});
+15 -2
View File
@@ -1,10 +1,23 @@
import { ValidationPipe, VersioningType } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import type { NestExpressApplication } from '@nestjs/platform-express';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import { AppModule } from './app.module';
export async function createApp() {
const app = await NestFactory.create(AppModule, { bufferLogs: true });
export const API_JSON_BODY_LIMIT = '2mb';
export async function createApp(): Promise<NestExpressApplication> {
const app = await NestFactory.create<NestExpressApplication>(AppModule, {
bufferLogs: true,
bodyParser: false,
});
// Upload finalize requests can include base64 artwork payloads.
app.useBodyParser('json', { limit: API_JSON_BODY_LIMIT });
app.useBodyParser('urlencoded', {
extended: true,
limit: API_JSON_BODY_LIMIT,
});
app.setGlobalPrefix('api');
app.enableVersioning({
@@ -63,7 +63,7 @@ describe('ArtworkService', () => {
const userId = randomUUID();
const deviceId = randomUUID();
const artworkId = randomUUID();
const storageKey = join('library', 'artwork', `${artworkId}.png`);
const storageKey = join('users', userId, 'artwork', `${artworkId}.png`);
const bytes = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIW2P8z8DwHwAFgwJ/lBi4NwAAAABJRU5ErkJggg==',
'base64',
@@ -71,11 +71,10 @@ describe('ArtworkService', () => {
state.devices.set(deviceId, { id: deviceId, userId });
state.artworkAssets.set(artworkId, {
userId,
storageKey,
mimeType: 'image/png',
track: {
userId,
},
tracks: [{ userId }],
});
const filePath = storageService.resolve(storageKey);
@@ -97,11 +96,10 @@ describe('ArtworkService', () => {
state.devices.set(ownerDeviceId, { id: ownerDeviceId, userId: otherUserId });
state.artworkAssets.set(artworkId, {
storageKey: join('library', 'artwork', `${artworkId}.jpg`),
userId: ownerId,
storageKey: join('users', ownerId, 'artwork', `${artworkId}.jpg`),
mimeType: 'image/jpeg',
track: {
userId: ownerId,
},
tracks: [{ userId: ownerId }],
});
await expect(
@@ -116,11 +114,10 @@ describe('ArtworkService', () => {
state.devices.set(deviceId, { id: deviceId, userId });
state.artworkAssets.set(artworkId, {
storageKey: join('library', 'artwork', `${artworkId}.png`),
userId,
storageKey: join('users', userId, 'artwork', `${artworkId}.png`),
mimeType: 'image/png',
track: {
userId,
},
tracks: [{ userId }],
});
await expect(
+10 -3
View File
@@ -38,9 +38,11 @@ export class ArtworkService {
const artwork = await this.prismaService.artworkAsset.findUnique({
where: { id: artworkId },
select: {
userId: true,
storageKey: true,
mimeType: true,
track: {
tracks: {
take: 1,
select: {
userId: true,
},
@@ -48,11 +50,16 @@ export class ArtworkService {
},
});
if (!artwork || !artwork.track) {
if (!artwork) {
throw new NotFoundException('Artwork not found');
}
if (artwork.track.userId !== device.userId) {
const ownerUserId = artwork.userId ?? artwork.tracks[0]?.userId;
if (!ownerUserId) {
throw new NotFoundException('Artwork not found');
}
if (ownerUserId !== device.userId) {
throw new ForbiddenException('Artwork does not belong to this device user.');
}
@@ -29,6 +29,24 @@ export class LocalFilesystemStorageService {
return this.resolve(this.userAudioAssetStorageKey(userId, sha256));
}
userArtworkAssetStorageKey(
userId: string,
sha256: string,
fileExtension: string,
): string {
return join('users', userId, 'artwork', `${sha256}.${fileExtension}`);
}
userArtworkAssetPath(
userId: string,
sha256: string,
fileExtension: string,
): string {
return this.resolve(
this.userArtworkAssetStorageKey(userId, sha256, fileExtension),
);
}
tempUploadStorageKey(uploadId: string): string {
return join('temp', 'uploads', `${uploadId}.part`);
}
@@ -1,5 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
import { UploadSessionStatus } from '@prisma/client';
import { Type } from 'class-transformer';
import {
IsInt,
IsOptional,
@@ -9,6 +10,7 @@ import {
Max,
Min,
MinLength,
ValidateNested,
} from 'class-validator';
export class UploadPrepareRequestDto {
@@ -72,7 +74,41 @@ export class UploadSessionStatusResponseDto {
finalizedAt?: string;
}
export class UploadFinalizeArtworkDto {
@ApiProperty({ example: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ...' })
@IsString()
@MinLength(1)
dataBase64!: string;
@ApiProperty({
example:
'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
})
@Matches(/^[a-f0-9]{64}$/)
sha256!: string;
@ApiProperty({ enum: ['image/jpeg', 'image/png'] })
@Matches(/^image\/(jpeg|png)$/)
mimeType!: string;
@ApiProperty({ required: false, example: 512 })
@IsOptional()
@IsInt()
@Min(1)
@Max(Number.MAX_SAFE_INTEGER)
width?: number;
@ApiProperty({ required: false, example: 512 })
@IsOptional()
@IsInt()
@Min(1)
@Max(Number.MAX_SAFE_INTEGER)
height?: number;
}
export class UploadFinalizeRequestDto {
static readonly artworkMimeTypes = ['image/jpeg', 'image/png'] as const;
@ApiProperty({ example: 'Track Title' })
@IsString()
@MinLength(1)
@@ -94,6 +130,15 @@ export class UploadFinalizeRequestDto {
@Min(1)
@Max(Number.MAX_SAFE_INTEGER)
durationMs?: number;
@ApiProperty({
required: false,
type: () => UploadFinalizeArtworkDto,
})
@IsOptional()
@ValidateNested()
@Type(() => UploadFinalizeArtworkDto)
artwork?: UploadFinalizeArtworkDto;
}
export class UploadFinalizeResponseDto {
@@ -16,6 +16,7 @@ function createPrismaMock() {
const devices = new Map<string, any>();
const tracks = new Map<string, any>();
const audioAssets = new Map<string, any>();
const artworkAssets = new Map<string, any>();
const uploadSessions = new Map<string, any>();
const libraryEvents = new Map<bigint, any>();
let nextLibraryEventId = 1n;
@@ -145,6 +146,35 @@ function createPrismaMock() {
return updated;
}),
},
artworkAsset: {
findFirst: jest.fn().mockImplementation(async ({ where }) => {
return (
[...artworkAssets.values()].find(
(asset) =>
asset.userId === where.userId &&
asset.sha256 === where.sha256,
) ?? null
);
}),
create: jest.fn().mockImplementation(async ({ data }) => {
const record = {
id: randomUUID(),
createdAt: new Date(),
...data,
};
artworkAssets.set(record.id, record);
return record;
}),
update: jest.fn().mockImplementation(async ({ where, data }) => {
const current = artworkAssets.get(where.id);
const updated = {
...current,
...data,
};
artworkAssets.set(where.id, updated);
return updated;
}),
},
uploadSession: {
create: jest.fn().mockImplementation(async ({ data }) => {
const now = new Date();
@@ -199,6 +229,7 @@ function createPrismaMock() {
devices,
tracks,
audioAssets,
artworkAssets,
uploadSessions,
libraryEvents,
},
@@ -238,6 +269,13 @@ function sha256Hex(data: Buffer): string {
return createHash('sha256').update(data).digest('hex');
}
function sampleArtworkBytes(): Buffer {
return Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIW2P8z8DwHwAFgwJ/lBi4NwAAAABJRU5ErkJggg==',
'base64',
);
}
describe('UploadsService', () => {
let prismaMock: any;
let state: MockState;
@@ -369,6 +407,50 @@ describe('UploadsService', () => {
expect(session.audioAssetId).toBe(finalizeResponse.assetId);
});
it('finalize with embedded artwork creates and links an artwork asset', async () => {
const device = seedDevice();
const uploadedBytes = sampleMp3Bytes('finalize-with-artwork');
const artworkBytes = sampleArtworkBytes();
const sha256 = sha256Hex(uploadedBytes);
const artworkSHA256 = sha256Hex(artworkBytes);
const response = await service.prepare({
deviceId: device.id,
sha256,
originalFilename: 'finalize-with-artwork.mp3',
sizeBytes: uploadedBytes.length,
});
await service.uploadFile(response.uploadId!, createUploadRequest(uploadedBytes));
const finalizeResponse = await service.finalize(response.uploadId!, {
title: 'Artwork Track',
artist: 'Velody',
album: 'Milestone 8.1',
durationMs: 245000,
artwork: {
dataBase64: artworkBytes.toString('base64'),
sha256: artworkSHA256,
mimeType: 'image/png',
width: 1,
height: 1,
},
});
expect(finalizeResponse.trackId).toBeDefined();
expect(state.artworkAssets.size).toBe(1);
const track = [...state.tracks.values()][0];
const artworkAsset = [...state.artworkAssets.values()][0];
expect(track.artworkAssetId).toBe(artworkAsset.id);
expect(artworkAsset.userId).toBe(state.defaultUser.id);
expect(artworkAsset.sha256).toBe(artworkSHA256);
expect(artworkAsset.mimeType).toBe('image/png');
const storedBytes = await readFile(
join(storageRoot, 'users', state.defaultUser.id, 'artwork', `${artworkSHA256}.png`),
);
expect(storedBytes.equals(artworkBytes)).toBe(true);
});
it('returns exists from prepare after a successful upload and finalize', async () => {
const device = seedDevice();
const uploadedBytes = sampleMp3Bytes('duplicate-handling');
@@ -399,7 +481,73 @@ describe('UploadsService', () => {
});
expect(secondPrepare.status).toBe('exists');
expect(secondPrepare.uploadId).toBeDefined();
expect(secondPrepare.trackId).toBe(finalizeResponse.trackId);
expect(secondPrepare.assetId).toBe(finalizeResponse.assetId);
});
it('finalize on an exists prepare attaches artwork to an existing audio asset without re-uploading bytes', async () => {
const device = seedDevice();
const uploadedBytes = sampleMp3Bytes('duplicate-artwork-update');
const artworkBytes = sampleArtworkBytes();
const sha256 = sha256Hex(uploadedBytes);
const artworkSHA256 = sha256Hex(artworkBytes);
const firstPrepare = await service.prepare({
deviceId: device.id,
sha256,
originalFilename: 'duplicate-artwork-update.mp3',
sizeBytes: uploadedBytes.length,
});
await service.uploadFile(
firstPrepare.uploadId!,
createUploadRequest(uploadedBytes),
);
const firstFinalize = await service.finalize(firstPrepare.uploadId!, {
title: 'Duplicate Artwork Track',
artist: 'Velody',
album: 'Milestone 8.1',
durationMs: 123000,
});
const secondPrepare = await service.prepare({
deviceId: device.id,
sha256,
originalFilename: 'duplicate-artwork-update.mp3',
sizeBytes: uploadedBytes.length,
});
expect(secondPrepare.status).toBe('exists');
expect(secondPrepare.uploadId).toBeDefined();
const secondFinalize = await service.finalize(secondPrepare.uploadId!, {
title: 'Duplicate Artwork Track',
artist: 'Velody',
album: 'Milestone 8.1',
durationMs: 123000,
artwork: {
dataBase64: artworkBytes.toString('base64'),
sha256: artworkSHA256,
mimeType: 'image/png',
width: 1,
height: 1,
},
});
expect(secondFinalize.trackId).toBe(firstFinalize.trackId);
expect(secondFinalize.assetId).toBe(firstFinalize.assetId);
expect(state.audioAssets.size).toBe(1);
expect(state.artworkAssets.size).toBe(1);
const track = [...state.tracks.values()][0];
const artworkAsset = [...state.artworkAssets.values()][0];
expect(track.artworkAssetId).toBe(artworkAsset.id);
expect(artworkAsset.sha256).toBe(artworkSHA256);
const storedArtworkBytes = await readFile(
join(storageRoot, 'users', state.defaultUser.id, 'artwork', `${artworkSHA256}.png`),
);
expect(storedArtworkBytes.equals(artworkBytes)).toBe(true);
});
});
+199 -1
View File
@@ -12,12 +12,13 @@ import {
import type { Request } from 'express';
import { createHash, randomUUID } from 'node:crypto';
import { constants } from 'node:fs';
import { access, open, rename, unlink } from 'node:fs/promises';
import { access, open, rename, unlink, writeFile } from 'node:fs/promises';
import { extname } from 'node:path';
import { PrismaService } from '../../infrastructure/database/prisma.service';
import { AppConfigService } from '../config/config.service';
import { LocalFilesystemStorageService } from '../storage/storage.service';
import {
UploadFinalizeArtworkDto,
UploadFinalizeRequestDto,
UploadFinalizeResponseDto,
UploadPrepareRequestDto,
@@ -25,6 +26,15 @@ import {
UploadSessionStatusResponseDto,
} from './uploads.dto';
interface PreparedArtworkAssetInput {
sha256: string;
mimeType: string;
width?: number;
height?: number;
storageKey: string;
fileSizeBytes: bigint;
}
@Injectable()
export class UploadsService {
constructor(
@@ -57,8 +67,28 @@ export class UploadsService {
});
if (existingAsset) {
const uploadId = randomUUID();
const uploadSession = await this.prismaService.uploadSession.create({
data: {
id: uploadId,
userId: device.userId,
deviceId: body.deviceId,
trackId: existingAsset.trackId,
audioAssetId: existingAsset.id,
expectedSha256: body.sha256,
originalFilename: body.originalFilename,
expectedSizeBytes: BigInt(body.sizeBytes),
receivedBytes: BigInt(body.sizeBytes),
tempStoragePath: this.storageService.tempUploadStorageKey(uploadId),
status: UploadSessionStatus.COMPLETED,
completedAt: new Date(),
expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
},
});
return {
status: 'exists',
uploadId: uploadSession.id,
trackId: existingAsset.trackId ?? undefined,
assetId: existingAsset.id,
};
@@ -276,6 +306,13 @@ export class UploadsService {
);
}
const preparedArtwork = body.artwork
? await this.prepareArtworkAssetInput(
currentSession.userId,
body.artwork,
)
: null;
let audioAsset = await tx.audioAsset.findUnique({
where: {
userId_sha256: {
@@ -361,6 +398,25 @@ export class UploadsService {
});
}
const artworkAssetId = preparedArtwork
? (
await this.findOrCreateArtworkAsset(
tx,
currentSession.userId,
preparedArtwork,
)
).id
: null;
if ((track.artworkAssetId ?? null) !== artworkAssetId) {
track = await tx.track.update({
where: { id: track.id },
data: {
artworkAssetId,
},
});
}
await tx.libraryEvent.create({
data: {
userId: currentSession.userId,
@@ -423,6 +479,94 @@ export class UploadsService {
}
}
private async prepareArtworkAssetInput(
userId: string,
artwork: UploadFinalizeArtworkDto,
): Promise<PreparedArtworkAssetInput> {
const mimeType = this.normalizeArtworkMimeType(artwork.mimeType);
const artworkBytes = this.decodeArtworkData(artwork.dataBase64);
const actualSha256 = createHash('sha256').update(artworkBytes).digest('hex');
if (actualSha256 !== artwork.sha256) {
throw new UnprocessableEntityException(
'Artwork hash does not match the provided artwork sha256.',
);
}
const fileExtension = this.artworkFileExtension(mimeType);
const storageKey = this.storageService.userArtworkAssetStorageKey(
userId,
artwork.sha256,
fileExtension,
);
const filePath = this.storageService.resolve(storageKey);
await this.storageService.ensureParentDirectory(filePath);
if (!(await this.fileExists(filePath))) {
await this.writeFileAtomically(filePath, artworkBytes);
}
return {
sha256: artwork.sha256,
mimeType,
width: artwork.width,
height: artwork.height,
storageKey,
fileSizeBytes: BigInt(artworkBytes.length),
};
}
private async findOrCreateArtworkAsset(
tx: Pick<PrismaService, 'artworkAsset'>,
userId: string,
artwork: PreparedArtworkAssetInput,
) {
let artworkAsset = await tx.artworkAsset.findFirst({
where: {
userId,
sha256: artwork.sha256,
},
});
if (artworkAsset) {
const shouldUpdateArtworkAsset =
artworkAsset.mimeType !== artwork.mimeType ||
artworkAsset.width !== artwork.width ||
artworkAsset.height !== artwork.height ||
artworkAsset.storageKey !== artwork.storageKey ||
artworkAsset.fileSizeBytes !== artwork.fileSizeBytes ||
artworkAsset.userId !== userId;
if (shouldUpdateArtworkAsset) {
artworkAsset = await tx.artworkAsset.update({
where: { id: artworkAsset.id },
data: {
userId,
mimeType: artwork.mimeType,
width: artwork.width,
height: artwork.height,
storageKey: artwork.storageKey,
fileSizeBytes: artwork.fileSizeBytes,
},
});
}
return artworkAsset;
}
return tx.artworkAsset.create({
data: {
userId,
sha256: artwork.sha256,
mimeType: artwork.mimeType,
width: artwork.width,
height: artwork.height,
storageKey: artwork.storageKey,
fileSizeBytes: artwork.fileSizeBytes,
},
});
}
private assertMp3Filename(filename: string): void {
if (extname(filename).toLowerCase() !== '.mp3') {
throw new UnprocessableEntityException('Only MP3 uploads are supported.');
@@ -507,6 +651,45 @@ export class UploadsService {
);
}
private decodeArtworkData(dataBase64: string): Buffer {
const artworkBytes = Buffer.from(dataBase64, 'base64');
if (artworkBytes.length === 0) {
throw new UnprocessableEntityException(
'Artwork data must contain a non-empty base64 image payload.',
);
}
return artworkBytes;
}
private normalizeArtworkMimeType(mimeType: string): string {
switch (mimeType.trim().toLowerCase()) {
case 'image/jpeg':
case 'image/jpg':
return 'image/jpeg';
case 'image/png':
return 'image/png';
default:
throw new UnprocessableEntityException(
'Only embedded JPEG and PNG artwork are supported.',
);
}
}
private artworkFileExtension(mimeType: string): string {
switch (mimeType) {
case 'image/jpeg':
return 'jpg';
case 'image/png':
return 'png';
default:
throw new UnprocessableEntityException(
'Only embedded JPEG and PNG artwork are supported.',
);
}
}
private async markUploadFailed(
uploadId: string,
receivedBytes: number,
@@ -537,6 +720,21 @@ export class UploadsService {
}
}
private async writeFileAtomically(
path: string,
data: Buffer,
): Promise<void> {
const tempPath = `${path}.${randomUUID()}.tmp`;
try {
await writeFile(tempPath, data);
await rename(tempPath, path);
} catch (error) {
await this.safeUnlink(tempPath);
throw error;
}
}
private trimOptional(value?: string): string | undefined {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;