Skip to content

Commit

Permalink
Merge pull request #369 from FgForrest/363-reference-attributes-are-r…
Browse files Browse the repository at this point in the history
…eturned-when-referencecontentall-is-used

#363 reference attributes are returned when referencecontentall is used
  • Loading branch information
novoj authored Dec 8, 2023
2 parents b75e83a + 4796159 commit c344ed0
Show file tree
Hide file tree
Showing 13 changed files with 292 additions and 46 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static java.util.Optional.ofNullable;

/**
* This predicate allows limiting number of references visible to the client based on query constraints.
*
Expand All @@ -64,6 +66,10 @@ public class ReferenceContractSerializablePredicate implements SerializablePredi
* Contains information about all reference names that has been fetched / requested for the entity.
*/
@Nonnull @Getter private final Map<String, AttributeRequest> referenceSet;
/**
* Contains information about default attribute request for references that has no explicit attribute request.
*/
@Nullable @Getter private final AttributeRequest defaultAttributeRequest;
/**
* Contains true if any of the references of the entity has been fetched / requested.
*/
Expand All @@ -88,6 +94,7 @@ public class ReferenceContractSerializablePredicate implements SerializablePredi
public ReferenceContractSerializablePredicate() {
this.requiresEntityReferences = true;
this.referenceSet = Collections.emptyMap();
this.defaultAttributeRequest = null;
this.implicitLocale = null;
this.locales = Collections.emptySet();
this.underlyingPredicate = null;
Expand All @@ -104,6 +111,9 @@ public ReferenceContractSerializablePredicate(@Nonnull EvitaRequest evitaRequest
entry -> entry.getValue().attributeRequest()
)
);
this.defaultAttributeRequest = ofNullable(evitaRequest.getDefaultReferenceRequirement())
.map(RequirementContext::attributeRequest)
.orElse(null);
this.implicitLocale = evitaRequest.getImplicitLocale();
this.locales = evitaRequest.getRequiredLocales();
this.underlyingPredicate = null;
Expand All @@ -112,6 +122,7 @@ public ReferenceContractSerializablePredicate(@Nonnull EvitaRequest evitaRequest
public ReferenceContractSerializablePredicate(boolean requiresEntityReferences) {
this.requiresEntityReferences = requiresEntityReferences;
this.referenceSet = Collections.emptyMap();
this.defaultAttributeRequest = null;
this.implicitLocale = null;
this.locales = Collections.emptySet();
this.underlyingPredicate = null;
Expand All @@ -137,18 +148,23 @@ public ReferenceContractSerializablePredicate(
entry -> entry.getValue().attributeRequest()
)
);
this.defaultAttributeRequest = ofNullable(evitaRequest.getDefaultReferenceRequirement())
.map(RequirementContext::attributeRequest)
.orElse(null);
this.implicitLocale = evitaRequest.getImplicitLocale();
this.locales = evitaRequest.getRequiredLocales();
this.underlyingPredicate = underlyingPredicate;
}

ReferenceContractSerializablePredicate(
@Nonnull Map<String, AttributeRequest> referenceSet,
@Nullable AttributeRequest defaultAttributeRequest,
boolean requiresEntityReferences,
@Nullable Locale implicitLocale,
@Nonnull Set<Locale> locales
) {
this.referenceSet = referenceSet;
this.defaultAttributeRequest = defaultAttributeRequest;
this.requiresEntityReferences = requiresEntityReferences;
this.implicitLocale = implicitLocale;
this.locales = locales;
Expand Down Expand Up @@ -209,16 +225,21 @@ public ReferenceContractSerializablePredicate createRicherCopyWith(@Nonnull Evit

final Map<String, AttributeRequest> requiredReferencedEntities = combineReferencedEntities(evitaRequest);
final boolean doesRequireEntityReferences = evitaRequest.isRequiresEntityReferences();
final AttributeRequest defaultAttributeRequest = ofNullable(evitaRequest.getDefaultReferenceRequirement())
.map(RequirementContext::attributeRequest)
.orElse(null);

if ((this.requiresEntityReferences || !doesRequireEntityReferences) &&
Objects.equals(this.referenceSet, requiredReferencedEntities) &&
Objects.equals(this.defaultAttributeRequest, defaultAttributeRequest) &&
Objects.equals(this.implicitLocale, evitaRequest.getImplicitLocale()) &&
Objects.equals(this.locales, requiredLocales)
) {
return this;
} else {
return new ReferenceContractSerializablePredicate(
requiredReferencedEntities,
mergeAttributeRequests(this.defaultAttributeRequest, defaultAttributeRequest),
this.requiresEntityReferences || doesRequireEntityReferences,
implicitLocale,
requiredLocales
Expand All @@ -232,11 +253,25 @@ public ReferenceAttributeValueSerializablePredicate getAttributePredicate(@Nonnu
this.implicitLocale,
this.locales,
this.referenceSet.isEmpty() ?
AttributeRequest.FULL :
(this.defaultAttributeRequest == null ? AttributeRequest.EMPTY : this.defaultAttributeRequest) :
this.referenceSet.getOrDefault(referenceName, AttributeRequest.EMPTY)
);
}

@Nullable
public Set<Locale> getAllLocales() {
if (this.implicitLocale != null && this.locales == null) {
return Set.of(this.implicitLocale);
} else if (this.implicitLocale != null) {
return Stream.concat(
Stream.of(implicitLocale),
locales.stream()
).collect(Collectors.toSet());
} else {
return this.locales;
}
}

@Nonnull
private Map<String, AttributeRequest> combineReferencedEntities(@Nonnull EvitaRequest evitaRequest) {
final Map<String, AttributeRequest> requiredReferences;
Expand All @@ -257,45 +292,38 @@ private Map<String, AttributeRequest> combineReferencedEntities(@Nonnull EvitaRe
for (Entry<String, RequirementContext> newEntry : referenceEntityFetch.entrySet()) {
final AttributeRequest existingAttributeRequest = requiredReferences.get(newEntry.getKey());
final AttributeRequest newAttributeRequest = newEntry.getValue().attributeRequest();
if (existingAttributeRequest == null) {
requiredReferences.put(newEntry.getKey(), newAttributeRequest);
} else {
final AttributeRequest attributeRequest;
if (existingAttributeRequest.isRequiresEntityAttributes() && existingAttributeRequest.attributeSet().isEmpty()) {
attributeRequest = existingAttributeRequest;
} else if (newAttributeRequest.isRequiresEntityAttributes() && newAttributeRequest.attributeSet().isEmpty()) {
attributeRequest = newAttributeRequest;
} else {
attributeRequest = new AttributeRequest(
CollectionUtils.combine(existingAttributeRequest.attributeSet(), newAttributeRequest.attributeSet()),
existingAttributeRequest.isRequiresEntityAttributes() ||
newAttributeRequest.isRequiresEntityAttributes()
);
}
requiredReferences.put(
newEntry.getKey(),
attributeRequest
);
}
final AttributeRequest mergedAttributeRequest = mergeAttributeRequests(existingAttributeRequest, newAttributeRequest);
requiredReferences.put(newEntry.getKey(), mergedAttributeRequest);
}
} else {
requiredReferences = this.referenceSet;
}
return requiredReferences;
}

@Nullable
public Set<Locale> getAllLocales() {
if (this.implicitLocale != null && this.locales == null) {
return Set.of(this.implicitLocale);
} else if (this.implicitLocale != null) {
return Stream.concat(
Stream.of(implicitLocale),
locales.stream()
).collect(Collectors.toSet());
private static AttributeRequest mergeAttributeRequests(
@Nullable AttributeRequest existingAttributeRequest,
@Nullable AttributeRequest newAttributeRequest
) {
final AttributeRequest mergedAttributeRequest;
if (existingAttributeRequest == null) {
mergedAttributeRequest = newAttributeRequest;
} else {
return this.locales;
final AttributeRequest attributeRequest;
if (existingAttributeRequest.isRequiresEntityAttributes() && existingAttributeRequest.attributeSet().isEmpty()) {
attributeRequest = existingAttributeRequest;
} else if (newAttributeRequest.isRequiresEntityAttributes() && newAttributeRequest.attributeSet().isEmpty()) {
attributeRequest = newAttributeRequest;
} else {
attributeRequest = new AttributeRequest(
CollectionUtils.combine(existingAttributeRequest.attributeSet(), newAttributeRequest.attributeSet()),
existingAttributeRequest.isRequiresEntityAttributes() ||
newAttributeRequest.isRequiresEntityAttributes()
);
}
mergedAttributeRequest = attributeRequest;
}
return mergedAttributeRequest;
}

@Nullable
Expand Down
12 changes: 12 additions & 0 deletions evita_engine/src/main/java/io/evitadb/core/query/QueryContext.java
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,18 @@ public int translateEntity(@Nonnull EntityContract entity) {
}
}

/**
* Method returns requested entity primary key by specifying its primary key (either virtual or real).
*/
public int translateToEntityPrimaryKey(int primaryKey) {
if (this.entityReferencePkSequence > 0) {
final EntityReferenceContract<EntityReference> referencedEntity = this.entityReferencePkIndex.get(primaryKey);
return referencedEntity == null ? primaryKey : referencedEntity.getPrimaryKey();
} else {
return primaryKey;
}
}

/**
* Method returns requested {@link EntityReference} by specifying its primary key (either virtual or real).
*/
Expand Down
26 changes: 24 additions & 2 deletions evita_engine/src/main/java/io/evitadb/core/query/QueryPlanner.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,10 @@
import io.evitadb.core.query.indexSelection.TargetIndexes;
import io.evitadb.core.query.sort.CacheableSorter;
import io.evitadb.core.query.sort.ConditionalSorter;
import io.evitadb.core.query.sort.NoSorter;
import io.evitadb.core.query.sort.OrderByVisitor;
import io.evitadb.core.query.sort.Sorter;
import io.evitadb.core.query.sort.primaryKey.TranslatedPrimaryKeySorter;
import io.evitadb.index.CatalogIndex;
import io.evitadb.index.EntityIndex;
import io.evitadb.index.EntityIndexType;
Expand Down Expand Up @@ -433,7 +435,7 @@ private static List<QueryPlanBuilder> createSorter(
// in case of debug cached variant tree or the entity is not known, we cannot use cache here
// and we need to retain original non-cached sorter
final Sorter sorter = orderByVisitor.getSorter();
builder.appendSorter(sorter);
builder.appendSorter(replaceNoSorterIfNecessary(queryContext, sorter));
} finally {
if (multipleAlternatives) {
queryContext.popStep();
Expand All @@ -456,7 +458,7 @@ private static List<QueryPlanBuilder> createSorter(
queryContext, builder.getFilterFormula(),
builder.getTargetIndexes(),
builder.getPrefetchFormulaVisitor(),
replacedSorter
replaceNoSorterIfNecessary(queryContext, replacedSorter)
)
);
} else {
Expand All @@ -472,6 +474,26 @@ private static List<QueryPlanBuilder> createSorter(
}
}

/**
* This method replaces no sorter - which should always represent primary keys in ascending order - with the special
* implementation in case the entity is not known in the query. In such case the primary keys are translated
* different ids and those ids are translated back at the end of the query. Unfortunately the order of the translated
* keys might be different than the original order of the primary keys, so we need to sort them here according to
* their original primary keys order in ascending fashion.
*
* @param queryContext query context
* @param sorter identified sorter
* @return sorter in input or new implementation that ensures proper sorting by primary keys in ascending order
*/
@Nonnull
private static Sorter replaceNoSorterIfNecessary(@Nonnull QueryContext queryContext, @Nonnull Sorter sorter) {
if (sorter instanceof NoSorter && !queryContext.isEntityTypeKnown()) {
return TranslatedPrimaryKeySorter.INSTANCE;
} else {
return sorter;
}
}

/**
* Method creates list of {@link ExtraResultProducer} implementations that fabricate requested extra data structures
* that are somehow connected with the processed query taking existing formula and their memoized results into
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,9 @@ public int sortAndSlice(@Nonnull QueryContext queryContext, @Nonnull Formula inp

// copy the sorted data to result
final int length = endIndex - startIndex;
System.arraycopy(filteredRecordIds, startIndex, result, peak, Math.min(result.length, length));
return length;
final int newPeak = Math.min(filteredRecordIds.length, length);
System.arraycopy(filteredRecordIds, startIndex, result, peak, newPeak);
return newPeak;
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
*
* _ _ ____ ____
* _____ _(_) |_ __ _| _ \| __ )
* / _ \ \ / / | __/ _` | | | | _ \
* | __/\ V /| | || (_| | |_| | |_) |
* \___| \_/ |_|\__\__,_|____/|____/
*
* Copyright (c) 2023
*
* Licensed under the Business Source License, Version 1.1 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://github.com/FgForrest/evitaDB/blob/main/LICENSE
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.evitadb.core.query.sort.primaryKey;

import io.evitadb.core.query.QueryContext;
import io.evitadb.core.query.algebra.Formula;
import io.evitadb.core.query.sort.Sorter;
import io.evitadb.index.bitmap.Bitmap;
import io.evitadb.utils.ArrayUtils;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

/**
* This sorter sorts translated primary keys according to original primary keys in ascending order. It is used with
* the special implementation in case the entity is not known in the query. In such case the primary keys are translated
* different ids and those ids are translated back at the end of the query. Unfortunately the order of the translated
* keys might be different than the original order of the primary keys, so we need to sort them here according to
* their original primary keys order in ascending fashion.
*
* @author Jan Novotný (novotny@fg.cz), FG Forrest a.s. (c) 2023
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class TranslatedPrimaryKeySorter implements Sorter {
public static final Sorter INSTANCE = new TranslatedPrimaryKeySorter();

@Nonnull
@Override
public Sorter cloneInstance() {
return INSTANCE;
}

@Nonnull
@Override
public Sorter andThen(Sorter sorterForUnknownRecords) {
return INSTANCE;
}

@Nullable
@Override
public Sorter getNextSorter() {
return null;
}

@Override
public int sortAndSlice(@Nonnull QueryContext queryContext, @Nonnull Formula input, int startIndex, int endIndex, @Nonnull int[] result, int peak) {
final Bitmap translatedPrimaryKeysBitmap = input.compute();
final int[] translatedPrimaryKeys = translatedPrimaryKeysBitmap.getArray();
final int[] originalPrimaryKeys = translatedPrimaryKeysBitmap.stream().map(queryContext::translateToEntityPrimaryKey).toArray();
// initialize order array
final int[] order = new int[originalPrimaryKeys.length];
for (int i = 0; i < order.length; i++) {
order[i] = i;
}

ArrayUtils.sortSecondAlongFirstArray(originalPrimaryKeys, order);
final int length = endIndex - startIndex;
for (int i = peak; i < translatedPrimaryKeys.length && i < length; i++) {
result[i] = translatedPrimaryKeys[order[i]];
}
return peak + Math.min(translatedPrimaryKeys.length, length);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
* Handles requests for multiple unknown entities identified by their URLs or codes.
Expand Down Expand Up @@ -72,8 +71,6 @@ protected EndpointResponse<List<EntityClassifier>> doHandleRequest(@Nonnull Rest
log.debug("Generated evitaDB query for unknown entity list fetch is `{}`.", query);

final List<EntityClassifier> entities = exchange.session().queryList(query, EntityClassifier.class);
log.info("Fetched list of unknown entities: {}", entities.stream().map(it -> it.getPrimaryKey().toString()).collect(Collectors.joining(", ")));

return new SuccessEndpointResponse<>(entities);
}

Expand Down
Loading

0 comments on commit c344ed0

Please sign in to comment.