try to launch thousands of request in parallel

2023-04-17 10:48:09 +02:00 · 2023-04-17 10:48:09 +02:00 · 10a9b94588
parent ca03d1769a
commit 10a9b94588
12 changed files with 392 additions and 38 deletions
--- a/package-lock.json
+++ b/package-lock.json
@ -16,6 +16,7 @@
        "@grpc/grpc-js": "^1.8.13",
        "@grpc/proto-loader": "^0.7.6",
        "@liaoliaots/nestjs-redis": "^9.0.5",
+        "@nestjs/axios": "^2.0.0",
        "@nestjs/cache-manager": "^1.0.0",
        "@nestjs/common": "^9.0.0",
        "@nestjs/config": "^2.3.1",
@ -25,6 +26,7 @@
        "@nestjs/platform-express": "^9.0.0",
        "@nestjs/terminus": "^9.2.2",
        "@prisma/client": "^4.12.0",
+        "axios": "^1.3.5",
        "cache-manager": "^5.2.0",
        "cache-manager-ioredis-yet": "^1.1.0",
        "class-transformer": "^0.5.1",
@ -1572,6 +1574,17 @@
        "node": ">=8"
      }
    },
+    "node_modules/@nestjs/axios": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-2.0.0.tgz",
+      "integrity": "sha512-F6oceoQLEn031uun8NiommeMkRIojQqVryxQy/mK7fx0CI0KbgkJL3SloCQcsOD+agoEnqKJKXZpEvL6FNswJg==",
+      "peerDependencies": {
+        "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0",
+        "axios": "^1.3.1",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^6.0.0 || ^7.0.0"
+      }
+    },
    "node_modules/@nestjs/cache-manager": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/@nestjs/cache-manager/-/cache-manager-1.0.0.tgz",
@ -3060,8 +3073,17 @@
    "node_modules/asynckit": {
      "version": "0.4.0",
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/axios": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.5.tgz",
+      "integrity": "sha512-glL/PvG/E+xCWwV8S6nCHcrfg1exGx7vxyUIivIA1iL7BIh6bePylCfVHwp6k13ao7SATxB6imau2kqY+I67kw==",
+      "dependencies": {
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
+      }
    },
    "node_modules/babel-jest": {
      "version": "29.5.0",
@ -3757,7 +3779,6 @@
      "version": "1.0.8",
      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
      "dependencies": {
        "delayed-stream": "~1.0.0"
      },
@ -3955,7 +3976,6 @@
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true,
      "engines": {
        "node": ">=0.4.0"
      }
@ -4795,6 +4815,25 @@
      "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
      "dev": true
    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/fork-ts-checker-webpack-plugin": {
      "version": "8.0.0",
      "resolved": "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-8.0.0.tgz",
@ -4827,7 +4866,6 @@
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
      "dependencies": {
        "asynckit": "^0.4.0",
        "combined-stream": "^1.0.8",
@ -7232,6 +7270,11 @@
        "node": ">= 0.10"
      }
    },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
    "node_modules/pump": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
--- a/package.json
+++ b/package.json
@ -38,6 +38,7 @@
    "@grpc/grpc-js": "^1.8.13",
    "@grpc/proto-loader": "^0.7.6",
    "@liaoliaots/nestjs-redis": "^9.0.5",
+    "@nestjs/axios": "^2.0.0",
    "@nestjs/cache-manager": "^1.0.0",
    "@nestjs/common": "^9.0.0",
    "@nestjs/config": "^2.3.1",
@ -47,6 +48,7 @@
    "@nestjs/platform-express": "^9.0.0",
    "@nestjs/terminus": "^9.2.2",
    "@prisma/client": "^4.12.0",
+    "axios": "^1.3.5",
    "cache-manager": "^5.2.0",
    "cache-manager-ioredis-yet": "^1.1.0",
    "class-transformer": "^0.5.1",
--- a/src/modules/matcher/adapters/secondaries/georouter-creator.ts
+++ b/src/modules/matcher/adapters/secondaries/georouter-creator.ts
@ -2,13 +2,16 @@ import { Injectable } from '@nestjs/common';
 import { ICreateGeorouter } from '../../domain/interfaces/georouter-creator.interface';
 import { IGeorouter } from '../../domain/interfaces/georouter.interface';
 import { GraphhopperGeorouter } from './graphhopper-georouter';
+import { HttpService } from '@nestjs/axios';

@Injectable()
 export class GeorouterCreator implements ICreateGeorouter {
+  constructor(private readonly httpService: HttpService) {}
+
  create(type: string, url: string): IGeorouter {
    switch (type) {
      case 'graphhopper':
-        return new GraphhopperGeorouter(url);
+        return new GraphhopperGeorouter(url, this.httpService);
      default:
        throw new Error('Unknown geocoder');
    }
--- a/src/modules/matcher/adapters/secondaries/graphhopper-georouter.ts
+++ b/src/modules/matcher/adapters/secondaries/graphhopper-georouter.ts
@ -1,15 +1,133 @@
-import { Route } from '../../domain/entities/route';
+import { HttpService } from '@nestjs/axios';
+import { NamedRoute } from '../../domain/entities/named-route';
 import { IGeorouter } from '../../domain/interfaces/georouter.interface';
 import { GeorouterSettings } from '../../domain/types/georouter-settings.type';
+import { Path } from '../../domain/types/path.type';
+import { Injectable } from '@nestjs/common';
+import {
+  catchError,
+  defer,
+  forkJoin,
+  from,
+  lastValueFrom,
+  map,
+  mergeAll,
+  mergeMap,
+  toArray,
+} from 'rxjs';
+import { AxiosError } from 'axios';

+@Injectable()
 export class GraphhopperGeorouter implements IGeorouter {
  _url: string;
+  _urlArgs: Array<string>;
+  _withTime: boolean;
+  _withPoints: boolean;
+  _withDistance: boolean;
+  _paths: Array<Path>;
+  _httpService: HttpService;

-  constructor(url: string) {
+  constructor(url: string, httpService: HttpService) {
    this._url = url + '/route?';
+    this._httpService = httpService;
  }

-  route(routesRequested: [], settings: GeorouterSettings): Route[] {
-    throw new Error('Method not implemented.');
+  async route(
+    paths: Array<Path>,
+    settings: GeorouterSettings,
+  ): Promise<Array<NamedRoute>> {
+    this._setDefaultUrlArgs();
+    this._setWithTime(settings.withTime);
+    this._setWithPoints(settings.withPoints);
+    this._setWithDistance(settings.withDistance);
+    this._paths = paths;
+    const routes = await this._getRoutes();
+    console.log(routes.length);
+    return routes;
+  }
+
+  _setDefaultUrlArgs(): void {
+    this._urlArgs = [
+      'vehicle=car',
+      'weighting=fastest',
+      'points_encoded=false',
+    ];
+  }
+
+  _setWithTime(withTime: boolean): void {
+    this._withTime = withTime;
+    if (withTime) {
+      this._urlArgs.push('details=time');
+    }
+  }
+
+  _setWithPoints(withPoints: boolean): void {
+    this._withPoints = withPoints;
+    if (withPoints) {
+      this._urlArgs.push('calc_points=false');
+    }
+  }
+
+  _setWithDistance(withDistance: boolean): void {
+    this._withDistance = withDistance;
+    if (withDistance) {
+      this._urlArgs.push('instructions=true');
+    } else {
+      this._urlArgs.push('instructions=false');
+    }
+  }
+
+  async _getRoutes(): Promise<Array<NamedRoute>> {
+    const routes = Promise.all(
+      this._paths.map(async (path) => {
+        const url: string = [
+          this._getUrl(),
+          '&point=',
+          path.points
+            .map((point) => [point.lat, point.lon].join())
+            .join('&point='),
+        ].join('');
+        const res = await lastValueFrom(
+          this._httpService.get(url).pipe(
+            map((res) => res.data.paths[0].distance),
+            catchError((error: AxiosError) => {
+              throw new Error(error.message);
+            }),
+          ),
+        );
+        return <NamedRoute>{
+          key: path.key,
+          route: res,
+        };
+      }),
+    );
+    return routes;
+    // const date1 = new Date();
+    // const urls = this._paths.map((path) =>
+    //   defer(() =>
+    //     this._httpService
+    //       .get(
+    //         [
+    //           this._getUrl(),
+    //           '&point=',
+    //           path.points
+    //             .map((point) => [point.lat, point.lon].join())
+    //             .join('&point='),
+    //         ].join(''),
+    //       )
+    //       .pipe(map((res) => res.data.paths[0].distance)),
+    //   ),
+    // );
+    // const observables = from(urls);
+    // const routes = observables.pipe(mergeAll(7), toArray());
+    // routes.subscribe(() => {
+    //   const date2 = new Date();
+    //   console.log(date2.getTime() - date1.getTime());
+    // });
+    // return [];
+  }
+
+  _getUrl(): string {
+    return [this._url, this._urlArgs.join('&')].join('');
  }
 }
--- a/src/modules/matcher/domain/entities/geography.ts
+++ b/src/modules/matcher/domain/entities/geography.ts
@ -2,9 +2,9 @@ import { MatcherException } from '../../exceptions/matcher.exception';
 import { IRequestGeography } from '../interfaces/geography-request.interface';
 import { PointType } from '../types/geography.enum';
 import { Point } from '../types/point.type';
-import { Route } from './route';
 import { find } from 'geo-tz';
 import { Waypoint } from '../types/waypoint';
+import { Route } from './route';

 export class Geography {
  _geographyRequest: IRequestGeography;
--- a/src/modules/matcher/domain/entities/named-route.ts
+++ b/src/modules/matcher/domain/entities/named-route.ts
@ -0,0 +1,6 @@
+import { Route } from './route';
+
+export class NamedRoute {
+  key: string;
+  route: Route;
+}
--- a/src/modules/matcher/domain/interfaces/georouter.interface.ts
+++ b/src/modules/matcher/domain/interfaces/georouter.interface.ts
@ -1,6 +1,10 @@
-import { Route } from '../entities/route';
+import { NamedRoute } from '../entities/named-route';
 import { GeorouterSettings } from '../types/georouter-settings.type';
+import { Path } from '../types/path.type';

 export interface IGeorouter {
-  route(routesRequested: [], settings: GeorouterSettings): Array<Route>;
+  route(
+    paths: Array<Path>,
+    settings: GeorouterSettings,
+  ): Promise<Array<NamedRoute>>;
 }
--- a/src/modules/matcher/domain/types/path.type.ts
+++ b/src/modules/matcher/domain/types/path.type.ts
@ -0,0 +1,6 @@
+import { Point } from './point.type';
+
+export type Path = {
+  key: string;
+  points: Array<Point>;
+};
--- a/src/modules/matcher/domain/usecases/match.usecase.ts
+++ b/src/modules/matcher/domain/usecases/match.usecase.ts
@ -17,21 +17,54 @@ export class MatchUseCase {

  async execute(matchQuery: MatchQuery): Promise<ICollection<Match>> {
    try {
+      const paths = [];
+      for (let i = 0; i < 2000; i++) {
+        paths.push({
+          key: 'route' + i,
+          points: [
+            {
+              lat: 48.110899,
+              lon: -1.68365,
+            },
+            {
+              lat: 48.131105,
+              lon: -1.690067,
+            },
+            {
+              lat: 48.56516,
+              lon: -1.923553,
+            },
+            {
+              lat: 48.622813,
+              lon: -1.997177,
+            },
+            {
+              lat: 48.67846,
+              lon: -1.8554,
+            },
+          ],
+        });
+      }
+      const routes = await matchQuery.algorithmSettings.georouter.route(paths, {
+        withDistance: true,
+        withPoints: true,
+        withTime: false,
+      });
      const match = new Match();
      match.uuid = 'e23f9725-2c19-49a0-9ef6-17d8b9a5ec85';
-      this._messager.publish('matcher.match', 'match !');
+      // this._messager.publish('matcher.match', 'match !');
      return {
        data: [match],
        total: 1,
      };
    } catch (error) {
-      this._messager.publish(
-        'logging.matcher.match.crit',
-        JSON.stringify({
-          matchQuery,
-          error,
-        }),
-      );
+      // this._messager.publish(
+      //   'logging.matcher.match.crit',
+      //   JSON.stringify({
+      //     matchQuery,
+      //     error,
+      //   }),
+      // );
      throw error;
    }
  }
--- a/src/modules/matcher/matcher.module.ts
+++ b/src/modules/matcher/matcher.module.ts
@ -13,11 +13,13 @@ import { RedisClientOptions } from '@liaoliaots/nestjs-redis';
 import { redisStore } from 'cache-manager-ioredis-yet';
 import { DefaultParamsProvider } from './adapters/secondaries/default-params.provider';
 import { GeorouterCreator } from './adapters/secondaries/georouter-creator';
+import { HttpModule } from '@nestjs/axios';

@Module({
  imports: [
    DatabaseModule,
    CqrsModule,
+    HttpModule,
    RabbitMQModule.forRootAsync(RabbitMQModule, {
      imports: [ConfigModule],
      useFactory: async (configService: ConfigService) => ({
--- a/src/modules/matcher/tests/unit/georouter-creator.spec.ts
+++ b/src/modules/matcher/tests/unit/georouter-creator.spec.ts
@ -1,13 +1,32 @@
+import { Test, TestingModule } from '@nestjs/testing';
 import { GeorouterCreator } from '../../adapters/secondaries/georouter-creator';
 import { GraphhopperGeorouter } from '../../adapters/secondaries/graphhopper-georouter';
+import { HttpService } from '@nestjs/axios';
+
+const mockHttpService = jest.fn();

 describe('Georouter creator', () => {
+  let georouterCreator: GeorouterCreator;
+
+  beforeAll(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      imports: [],
+      providers: [
+        GeorouterCreator,
+        {
+          provide: HttpService,
+          useValue: mockHttpService,
+        },
+      ],
+    }).compile();
+
+    georouterCreator = module.get<GeorouterCreator>(GeorouterCreator);
+  });
+
  it('should be defined', () => {
-    const georouterCreator: GeorouterCreator = new GeorouterCreator();
    expect(georouterCreator).toBeDefined();
  });
  it('should create a graphhopper georouter', () => {
-    const georouterCreator: GeorouterCreator = new GeorouterCreator();
    const georouter = georouterCreator.create(
      'graphhopper',
      'http://localhost',
@ -15,7 +34,6 @@ describe('Georouter creator', () => {
    expect(georouter).toBeInstanceOf(GraphhopperGeorouter);
  });
  it('should throw an exception if georouter type is unknown', () => {
-    const georouterCreator: GeorouterCreator = new GeorouterCreator();
    expect(() =>
      georouterCreator.create('unknown', 'http://localhost'),
    ).toThrow();
--- a/src/modules/matcher/tests/unit/graphhopper-georouter.spec.ts
+++ b/src/modules/matcher/tests/unit/graphhopper-georouter.spec.ts
@ -1,22 +1,141 @@
-import { GraphhopperGeorouter } from '../../adapters/secondaries/graphhopper-georouter';
+import { Test, TestingModule } from '@nestjs/testing';
+import { HttpService } from '@nestjs/axios';
+import { NamedRoute } from '../../domain/entities/named-route';
+import { GeorouterCreator } from '../../adapters/secondaries/georouter-creator';
+import { IGeorouter } from '../../domain/interfaces/georouter.interface';
+import { of } from 'rxjs';
+import { AxiosError } from 'axios';
+
+const mockHttpService = {
+  get: jest
+    .fn()
+    .mockImplementationOnce(() => {
+      throw new AxiosError('Axios error !');
+    })
+    .mockImplementation(() => {
+      return of({
+        status: 200,
+        data: [new NamedRoute()],
+      });
+    }),
+};

 describe('Graphhopper Georouter', () => {
-  it('should be defined', () => {
-    const graphhopperGeorouter: GraphhopperGeorouter = new GraphhopperGeorouter(
+  let georouterCreator: GeorouterCreator;
+  let graphhopperGeorouter: IGeorouter;
+
+  beforeAll(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      imports: [],
+      providers: [
+        GeorouterCreator,
+        {
+          provide: HttpService,
+          useValue: mockHttpService,
+        },
+      ],
+    }).compile();
+
+    georouterCreator = module.get<GeorouterCreator>(GeorouterCreator);
+    graphhopperGeorouter = georouterCreator.create(
+      'graphhopper',
      'http://localhost',
    );
+  });
+
+  it('should be defined', () => {
    expect(graphhopperGeorouter).toBeDefined();
  });
-  it('should throw an exception when calling route', () => {
-    const graphhopperGeorouter: GraphhopperGeorouter = new GraphhopperGeorouter(
-      'http://localhost',
-    );
-    expect(() =>
-      graphhopperGeorouter.route([], {
-        withDistance: false,
-        withPoints: false,
-        withTime: false,
-      }),
-    ).toThrow();
+
+  describe('route function', () => {
+    it('should fail on axios error', async () => {
+      await expect(
+        graphhopperGeorouter.route(
+          [
+            {
+              key: 'route1',
+              points: [
+                {
+                  lat: 49.440041,
+                  lon: 1.093912,
+                },
+                {
+                  lat: 50.630992,
+                  lon: 3.045432,
+                },
+              ],
+            },
+          ],
+          {
+            withDistance: false,
+            withPoints: false,
+            withTime: false,
+          },
+        ),
+      ).rejects.toBeInstanceOf(Error);
+    });
+    it('should create one route with all settings to false', async () => {
+      const routes = await graphhopperGeorouter.route(
+        [
+          {
+            key: 'route1',
+            points: [
+              {
+                lat: 49.440041,
+                lon: 1.093912,
+              },
+              {
+                lat: 50.630992,
+                lon: 3.045432,
+              },
+            ],
+          },
+        ],
+        {
+          withDistance: false,
+          withPoints: false,
+          withTime: false,
+        },
+      );
+      expect(routes).toHaveLength(1);
+    });
+    it('should create 2 routes with distance, points and time', async () => {
+      const routes = await graphhopperGeorouter.route(
+        [
+          {
+            key: 'route1',
+            points: [
+              {
+                lat: 49.440041,
+                lon: 1.093912,
+              },
+              {
+                lat: 50.630992,
+                lon: 3.045432,
+              },
+            ],
+          },
+          {
+            key: 'route2',
+            points: [
+              {
+                lat: 49.440041,
+                lon: 1.093912,
+              },
+              {
+                lat: 50.630992,
+                lon: 3.045432,
+              },
+            ],
+          },
+        ],
+        {
+          withDistance: true,
+          withPoints: true,
+          withTime: true,
+        },
+      );
+      expect(routes).toHaveLength(2);
+    });
  });
 });