Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DO NOT MERGE] Merge PR #8735

Open
wants to merge 37 commits into
base: feat/pipelines
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
acbf24f
add pipeline to Listen RPC
wu-hui Oct 15, 2024
93fdb23
Prepre for serializaion/deserialization
wu-hui Oct 17, 2024
8ac835e
fix rebase error
wu-hui Oct 17, 2024
422723a
quick hack to integrate with watch.
wu-hui Oct 21, 2024
ee64690
add api/pipelinesource and setup basic listen test
wu-hui Oct 31, 2024
1738c15
watch integration works.
wu-hui Nov 6, 2024
946a266
add pipeline canonify and eq
wu-hui Oct 22, 2024
6fc2050
runPipeline initial
wu-hui Oct 25, 2024
377e82f
initial offline function evaluation
wu-hui Oct 29, 2024
2b244a2
add basic tests for pipeline eval
wu-hui Oct 29, 2024
23b5135
runPipeline initial
wu-hui Oct 25, 2024
82b8303
Setting up QueryOrPipeline to replace Query
wu-hui Oct 30, 2024
7a3e789
type1 compiles
wu-hui Nov 12, 2024
4d7d917
introduce new variant for query_engine.test.ts
wu-hui Nov 13, 2024
c274177
Fix core/expression rebase error
wu-hui Nov 13, 2024
6e4a7e3
Add basic tests
wu-hui Nov 14, 2024
e6f860e
remove api/pipeline and use the lite one
wu-hui Nov 14, 2024
667c398
query_engine.test.ts pass with pipelines
wu-hui Nov 19, 2024
48a6324
local store tests PASS
wu-hui Nov 25, 2024
6ab2ba5
memory spec tests pass sans limitToLast
wu-hui Nov 28, 2024
58124c4
most spec tests PASS!
wu-hui Dec 2, 2024
0f63a54
limit to last, cursors and multitab for documents and database stages
wu-hui Dec 4, 2024
5945776
Merge remote-tracking branch 'origin/feat/pipelines' into wuandy/offp…
wu-hui Dec 6, 2024
5ad944e
fix merge errors
wu-hui Dec 10, 2024
da4dee3
Add expressions tests
wu-hui Dec 16, 2024
656e848
Ported all tests, plus some bug fixes
wu-hui Dec 19, 2024
90ce598
Merge remote-tracking branch 'origin/feat/pipelines' into wuandy/offp…
wu-hui Dec 19, 2024
34b3e71
fix merge errors
wu-hui Dec 19, 2024
c5678fe
fixed expressions tests
wu-hui Dec 20, 2024
f2a0585
Fixed all incompatibilities
wu-hui Dec 23, 2024
bc9ac99
Add gitignore entries
wu-hui Dec 28, 2024
0212394
Merge with feat/pipelines
wu-hui Dec 28, 2024
5e6cd52
Proto update sync and add query spec tests back
wu-hui Dec 28, 2024
2b4a5ad
merge with feat/pipeline
wu-hui Jan 28, 2025
ffc17e4
merge with feat/pipeline
wu-hui Jan 28, 2025
d2b18f9
Fix circular dependency caused by import from api_pipelines. Also sim…
MarkDuckworth Jan 29, 2025
d49ff96
Merge branch 'push-vwqlstsxuovu' of github.com:firebase/firebase-js-s…
MarkDuckworth Jan 29, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add basic tests for pipeline eval
  • Loading branch information
wu-hui committed Nov 6, 2024
commit 2b244a221b2806dbc49cf380bcb5bccd636eec86
2 changes: 1 addition & 1 deletion packages/firestore/src/core/expressions.ts
Original file line number Diff line number Diff line change
@@ -281,7 +281,7 @@ function asBigInt(protoNumber: { integerValue: number | string }): bigint {
}

const LongMaxValue = BigInt('0x7fffffffffffffff');
const LongMinValue = BigInt('-0x8000000000000000');
const LongMinValue = -BigInt('0x8000000000000000');

abstract class BigIntOrDoubleArithmetics<
T extends Add | Subtract | Multiply | Divide | Mod
339 changes: 333 additions & 6 deletions packages/firestore/test/unit/core/pipeline.test.ts
Original file line number Diff line number Diff line change
@@ -21,11 +21,23 @@ import { Firestore } from '../../../src/api/database';
import { CredentialsProvider } from '../../../src/api/credentials';
import { User } from '../../../src/auth/user';
import { DatabaseId } from '../../../src/core/database_info';
import { Field, eq, Constant, doc as docRef } from '../../../src';
import {
Field,
eq,
Constant,
doc as docRef,
lt,
lte,
add,
multiply,
gt,
gte
} from '../../../src';
import { canonifyPipeline, pipelineEq } from '../../../src/core/pipeline-util';
import { runPipeline } from '../../../src/core/pipeline_run';

import { doc } from '../../util/helpers';
import { and, or } from '../../../src/lite-api/expressions';

const fakeAuthProvider: CredentialsProvider<User> =
{} as unknown as CredentialsProvider<User>;
@@ -210,16 +222,331 @@ describe.only('runPipeline()', () => {
]);
});

it('works with simple where', () => {
const p = db.pipeline().collection('test').where(eq(`foo`, 42));
it('works with collection groups', () => {
const p = db.pipeline().collectionGroup('test');

expect(
runPipeline(p, [
doc('test/doc1', 1000, { foo: 'bar' }),
doc('testNot/doc2', 1000, { foo: 'baz' }),
doc('test/doc2', 1000, { foo: 42 }),
doc('test/doc3', 1000, { foo: '42' })
doc('testNot/doc2/test/doc2', 1000, { foo: 'baz' }),
doc('test1/doc2', 1000, { foo: 'bazzzz' })
])
).to.deep.equal([
doc('test/doc1', 1000, { foo: 'bar' }),
doc('testNot/doc2/test/doc2', 1000, { foo: 'baz' })
]);
});

it('works with database', () => {
const p = db.pipeline().database();

expect(
runPipeline(p, [
doc('test/doc1', 1000, { foo: 'bar' }),
doc('testNot/doc2/test/doc2', 1000, { foo: 'baz' }),
doc('test1/doc2', 1000, { foo: 'bazzzz' })
])
).to.deep.equal([
doc('test/doc1', 1000, { foo: 'bar' }),
doc('testNot/doc2/test/doc2', 1000, { foo: 'baz' }),
doc('test1/doc2', 1000, { foo: 'bazzzz' })
]);
});

it('works with simple wheres', () => {
const dataset = [
doc('test/doc1', 1000, { foo: 'bar' }),
doc('testNot/doc2', 1000, { foo: 'baz' }),
doc('test/doc2', 1000, { foo: 42 }),
doc('test/doc3', 1000, { foo: '42' })
];

expect(
runPipeline(
db.pipeline().collection('test').where(eq(`foo`, 42)),
dataset
)
).to.deep.equal([doc('test/doc2', 1000, { foo: 42 })]);

expect(
runPipeline(
db
.pipeline()
.collection('test')
.where(or(eq(`foo`, 42), eq('foo', 'bar'))),
dataset
)
).to.deep.equal([
doc('test/doc1', 1000, { foo: 'bar' }),
doc('test/doc2', 1000, { foo: 42 })
]);

expect(
runPipeline(
db.pipeline().collection('test').where(lte(`foo`, '42')),
dataset
)
).to.deep.equal([
doc('test/doc2', 1000, { foo: 42 }),
doc('test/doc3', 1000, { foo: '42' })
]);
});

// a representative dataset
const bookDataset = [
doc('test/book0', 1000, {
title: "The Hitchhiker's Guide to the Galaxy",
author: 'Douglas Adams',
genre: 'Science Fiction',
published: 1979,
rating: 4.2,
tags: ['comedy', 'space', 'adventure'],
awards: {
hugo: true,
nebula: false,
others: { unknown: { year: 1980 } }
},
nestedField: { 'level.1': { 'level.2': true } }
}),
doc('test/book1', 1000, {
title: 'Pride and Prejudice',
author: 'Jane Austen',
genre: 'Romance',
published: 1813,
rating: 4.5,
tags: ['classic', 'social commentary', 'love'],
awards: { none: true }
}),
doc('test/book2', 1000, {
title: 'One Hundred Years of Solitude',
author: 'Gabriel García Márquez',
genre: 'Magical Realism',
published: 1967,
rating: 4.3,
tags: ['family', 'history', 'fantasy'],
awards: { nobel: true, nebula: false }
}),
doc('test/book3', 1000, {
title: 'The Lord of the Rings',
author: 'J.R.R. Tolkien',
genre: 'Fantasy',
published: 1954,
rating: 4.7,
tags: ['adventure', 'magic', 'epic'],
awards: { hugo: false, nebula: false }
}),
doc('test/book4', 1000, {
title: "The Handmaid's Tale",
author: 'Margaret Atwood',
genre: 'Dystopian',
published: 1985,
rating: 4.1,
tags: ['feminism', 'totalitarianism', 'resistance'],
awards: { 'arthur c. clarke': true, 'booker prize': false }
}),
doc('test/book5', 1000, {
title: 'Crime and Punishment',
author: 'Fyodor Dostoevsky',
genre: 'Psychological Thriller',
published: 1866,
rating: 4.3,
tags: ['philosophy', 'crime', 'redemption'],
awards: { none: true }
}),
doc('test/book6', 1000, {
title: 'To Kill a Mockingbird',
author: 'Harper Lee',
genre: 'Southern Gothic',
published: 1960,
rating: 4.2,
tags: ['racism', 'injustice', 'coming-of-age'],
awards: { pulitzer: true }
}),
doc('test/book7', 1000, {
title: '1984',
author: 'George Orwell',
genre: 'Dystopian',
published: 1949,
rating: 4.2,
tags: ['surveillance', 'totalitarianism', 'propaganda'],
awards: { prometheus: true }
}),
doc('test/book8', 1000, {
title: 'The Great Gatsby',
author: 'F. Scott Fitzgerald',
genre: 'Modernist',
published: 1925,
rating: 4.0,
tags: ['wealth', 'american dream', 'love'],
awards: { none: true }
}),
doc('test/book9', 1000, {
title: 'Dune',
author: 'Frank Herbert',
genre: 'Science Fiction',
published: 1965,
rating: 4.6,
tags: ['politics', 'desert', 'ecology'],
awards: { hugo: true, nebula: true }
})
];

it('works with array contains', () => {
const p = db
.pipeline()
.collection('test')
.where(Field.of('tags').arrayContains('adventure'));

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[0],
bookDataset[3]
]);
});

it('works with array contains all', () => {
const p = db
.pipeline()
.collection('test')
.where(Field.of('tags').arrayContainsAll('adventure', 'magic'));

expect(runPipeline(p, bookDataset)).to.deep.equal([bookDataset[3]]);
});

it('works with array contains any', () => {
const p = db
.pipeline()
.collection('test')
.where(Field.of('tags').arrayContainsAny('adventure', 'classic'));

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[0],
bookDataset[1],
bookDataset[3]
]);
});

it('works with string queries', () => {
const p = db
.pipeline()
.collection('test')
.where(Field.of('title').startsWith('The'));

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[0],
bookDataset[3],
bookDataset[4],
bookDataset[8]
]);

const p2 = db
.pipeline()
.collection('test')
.where(Field.of('title').endsWith('Tale'));

expect(runPipeline(p2, bookDataset)).to.deep.equal([bookDataset[4]]);

const p3 = db
.pipeline()
.collection('test')
.where(Field.of('title').strContains('Guide'));

expect(runPipeline(p3, bookDataset)).to.deep.equal([bookDataset[0]]);
});

it('works with like queries', () => {
const p = db
.pipeline()
.collection('test')
.where(Field.of('title').like('%the%'));

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[0],
bookDataset[3]
]);
});

it('works with limit', () => {
const p = db.pipeline().collection('test').limit(3);

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[0],
bookDataset[1],
bookDataset[2]
]);
});

it('works with offset', () => {
const p = db.pipeline().collection('test').offset(3).limit(3);

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[3],
bookDataset[4],
bookDataset[5]
]);
});

it('works with regex operations', () => {
const p = db
.pipeline()
.collection('test')
.where(Field.of('title').regexMatch('^The.*ings'));

expect(runPipeline(p, bookDataset)).to.deep.equal([bookDataset[3]]);

const p2 = db
.pipeline()
.collection('test')
.where(Field.of('title').regexContains('Guide'));

expect(runPipeline(p2, bookDataset)).to.deep.equal([bookDataset[0]]);
});

it('works with arithmetics', () => {
const p = db
.pipeline()
.collection('test')
.where(multiply(Field.of('published'), Field.of('rating')).gte(9000));

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[3],
bookDataset[9]
]);
});

it('works with logical operators', () => {
const p = db
.pipeline()
.collection('test')
.where(
and(lt(Field.of('published'), 1900), gte(Field.of('rating'), 4.5))
);

expect(runPipeline(p, bookDataset)).to.deep.equal([bookDataset[1]]);
});

it('works with sort', () => {
const p = db
.pipeline()
.collection('test')
.sort(Field.of('published').ascending())
.limit(3);

expect(runPipeline(p, bookDataset)).to.deep.equal([
bookDataset[1],
bookDataset[5],
bookDataset[8]
]);

const p2 = db
.pipeline()
.collection('test')
.sort(Field.of('published').descending())
.limit(3);

expect(runPipeline(p2, bookDataset)).to.deep.equal([
bookDataset[4],
bookDataset[0],
bookDataset[2]
]);
});
});