diff --git a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml new file mode 100644 index 000000000000..69ad470fd4ee --- /dev/null +++ b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml @@ -0,0 +1,9 @@ +title: The package management API now has OpenAPI and SolrJ support. As part of this migration, the (experimental v2) API path moved from `/api/cluster/package` to `/api/cluster/packages`, and requests for a non-existent package or package version now return HTTP 404 instead of 400. +type: changed +authors: + - name: Eric Pugh +links: +- name: PR#4178 + url: https://github.com/apache/solr/pull/4178 +- name: SOLR-18212 + url: https://issues.apache.org/jira/browse/SOLR-18212 diff --git a/dev-docs/apis-v2-conventions.adoc b/dev-docs/apis-v2-conventions.adoc index 945c94afe725..848f460cedcf 100644 --- a/dev-docs/apis-v2-conventions.adoc +++ b/dev-docs/apis-v2-conventions.adoc @@ -156,6 +156,16 @@ Beyond that base, pick the narrowest existing response type that fits before wri If an API returns specific, known additional data beyond these bases (e.g. timing information, computed ranges, a resource's status fields), extend the appropriate base with typed `@JsonProperty` fields rather than reaching for `FlexibleSolrJerseyResponse` - see `SplitShardResponse.timing`, `SplitCoreResponse.ranges`, or `CollectionStatusResponse` for examples. Don't settle for a bare `SolrJerseyResponse` if the underlying operation actually produces more than a bare success/error - a v2 JSON caller has no other way to get that data, since (unlike v1) nothing else in the response pipeline will surface it (see the async caveat below for why this matters in practice). +=== Listing vs. Single-Resource Responses + +When an API exposes both a "list all" endpoint and a "get one by name" endpoint for the same kind of resource, give each its own response type rather than reusing the list's shape for a single item: + +* `ListResponse` - the collection endpoint's response. Its data is a map or list keyed/indexed across every matching resource (e.g. `ListAliasesResponse.aliases: Map`). +* `GetResponse` (or `GetByNameResponse`) - the single-resource endpoint's response. Its data is a flat object describing just that one resource, with no collection wrapper (e.g. `GetAliasByNameResponse.collections`, `GetAliasByNameResponse.properties`). + +Don't have the single-resource endpoint return the list type filtered down to one entry (e.g. a `Map` with one key) - that forces callers to re-extract by name something they already specified in the request, and conveys no information once only one key is possible. +See `ListAliasesResponse`/`GetAliasByNameResponse`, `ListClusterPropertiesResponse`/`GetClusterPropertyResponse`, and `ListPackagesResponse`/`GetPackageResponse` for examples of the pattern. + == Async Task Handling Solr has two distinct, non-interchangeable mechanisms for handling the `async` request parameter, depending on which base class the API extends: diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java new file mode 100644 index 000000000000..e126eb299d71 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.GetPackageResponse; +import org.apache.solr.client.api.model.ListPackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +/** V2 API definitions for managing Solr packages. */ +@Path("/cluster/packages") +public interface PackageApis { + + @GET + @Operation( + summary = "List all packages registered in this Solr cluster.", + tags = {"package"}) + ListPackagesResponse listPackages( + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version.") + @QueryParam("expectedVersion") + Integer expectedVersion); + + @GET + @Path("/{packageName}") + @Operation( + summary = "Get information about a specific package in this Solr cluster.", + tags = {"package"}) + GetPackageResponse getPackage( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version before responding.") + @QueryParam("expectedVersion") + Integer expectedVersion); + + @POST + @Path("/{packageName}/versions") + @Operation( + summary = "Add a version of a package to this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse addPackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @RequestBody(description = "Details of the package version to add.", required = true) + AddPackageVersionRequestBody requestBody); + + @DELETE + @Path("/{packageName}/versions/{version}") + @Operation( + summary = "Delete a specific version of a package from this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse deletePackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @Parameter(description = "The version of the package to delete.", required = true) + @PathParam("version") + String version); + + @POST + @Path("/{packageName}/refresh") + @Operation( + summary = "Refresh a package on all nodes in this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse refreshPackage( + @Parameter(description = "The name of the package to refresh.", required = true) + @PathParam("packageName") + String packageName, + @Parameter( + description = + "Indicates whether the refresh should only be done on the receiving node. For internal use only.") + @QueryParam("localOnly") + Boolean localOnly); + + @POST + @Path("/sync") + @Operation( + summary = + "Block until the receiving node's package data matches the given ZooKeeper version. " + + "Used internally for inter-node fan-out after a package change; not normally invoked directly.", + tags = {"package"}) + SolrJerseyResponse syncPackages( + @Parameter( + description = "The ZooKeeper version that the receiving node should catch up to.", + required = true) + @QueryParam("expectedVersion") + Integer expectedVersion); +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java similarity index 50% rename from solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java rename to solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java index 831d87d809f7..3b077c110d3b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java @@ -14,34 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.solr.client.api.model; -package org.apache.solr.client.solrj.request.beans; - +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; import java.util.List; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.util.ReflectMapWriter; - -/** Just a container class for POJOs used in Package APIs */ -public class PackagePayload { - public static class AddVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; - @JsonProperty(required = true) - public String version; +/** Request body for adding a version of a package. */ +public class AddPackageVersionRequestBody { - @JsonProperty(required = true) - public List files; + @JsonProperty("version") + @Schema(description = "The version string for this package version.", required = true) + public String version; - @JsonProperty public String manifest; - @JsonProperty public String manifestSHA512; - } + @JsonProperty("files") + @Schema( + description = "File paths from the file store to include in this version.", + required = true) + public List files; - public static class DelVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; + @JsonProperty("manifest") + @Schema(description = "Optional path to a manifest file in the file store.") + public String manifest; - @JsonProperty(required = true) - public String version; - } + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest file.") + public String manifestSHA512; } diff --git a/solr/api/src/java/org/apache/solr/client/api/model/GetPackageResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/GetPackageResponse.java new file mode 100644 index 000000000000..7e3d590012ec --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/GetPackageResponse.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; + +/** Response for the get-a-specific-package API. */ +public class GetPackageResponse extends SolrJerseyResponse { + + @JsonProperty("result") + @Schema(description = "The requested package's data.") + public PackageInfo result; + + /** Data for a single package, returned by the get-package API. */ + public static class PackageInfo { + @JsonProperty("znodeVersion") + @Schema(description = "The ZooKeeper version of the packages.json node.") + public int znodeVersion; + + @JsonProperty("name") + @Schema(description = "The package name.") + public String name; + + @JsonProperty("versions") + @Schema(description = "The versions registered for this package.") + public List versions; + } +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ListPackagesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ListPackagesResponse.java new file mode 100644 index 000000000000..5788b57967fb --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ListPackagesResponse.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; +import java.util.Map; + +/** Response for the package listing API. */ +public class ListPackagesResponse extends SolrJerseyResponse { + + @JsonProperty("result") + @Schema(description = "The package data including znode version and package definitions.") + public PackageData result; + + /** Package data returned by the package API. */ + public static class PackageData { + @JsonProperty("znodeVersion") + @Schema(description = "The ZooKeeper version of the packages.json node.") + public int znodeVersion; + + @JsonProperty("packages") + @Schema(description = "Map from package name to list of package versions.") + public Map> packages; + } + + /** Describes a single version of a package. */ + public static class PackageVersion { + @JsonProperty("package") + @Schema(description = "The package name.") + public String pkg; + + @JsonProperty("version") + @Schema(description = "The version string.") + public String version; + + @JsonProperty("files") + @Schema(description = "List of file paths from the file store included in this version.") + public List files; + + @JsonProperty("manifest") + @Schema(description = "Optional manifest reference.") + public String manifest; + + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest.") + public String manifestSHA512; + } +} diff --git a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java index 5f6a2e5a0888..e6ec84021081 100644 --- a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java +++ b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java @@ -334,7 +334,7 @@ public ApiInfo(PluginMetaHolder infoHolder, List errs) { coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); if (ver.isEmpty()) { // may be we are a bit early. Do a refresh and try again - coreContainer.getPackageLoader().getPackageAPI().refreshPackages(null); + coreContainer.getPackageLoader().getPackageStore().refreshPackages(null); ver = coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); } if (ver.isEmpty()) { diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java index 7db659bbca18..121789ebae36 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java @@ -50,12 +50,10 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; -import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.beans.PluginMeta; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; @@ -150,20 +148,9 @@ public void uninstall(String packageName, String version) // Delete the package by calling the Package API and remove the Jar runtime.printSuccess("Executing Package API to remove this package..."); - PackagePayload.DelVersion del = new PackagePayload.DelVersion(); - del.version = version; - del.pkg = packageName; - - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("delete", del)) - .build(); - try { - V2Response resp = req.process(solrClient); - runtime.printSuccess("Response: " + resp.jsonStr()); + new PackageApi.DeletePackageVersion(packageName, version).process(solrClient); + runtime.printSuccess("Package version deleted from Package API."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } @@ -176,9 +163,8 @@ public void uninstall(String packageName, String version) String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json")); for (String filePath : filesToDelete) { DistribFileStore.deleteZKFileEntry(zkClient, filePath); - String path = "/api/cluster/filestore/files" + filePath; - runtime.printSuccess("Deleting " + path); - solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.DELETE, path)); + runtime.printSuccess("Deleting " + filePath); + new FileStoreApi.DeleteFile(filePath).process(solrClient); } runtime.printSuccess("Package uninstalled: " + packageName + ":" + version + ":-)"); @@ -461,11 +447,8 @@ private Pair, List> deployCollectionPackage( // If updating, refresh the package version for this to take effect if (isUpdate || pegToLatest) { try { - SolrCLI.postJsonToSolr( - solrClient, - PackageUtils.PACKAGE_PATH, - "{\"refresh\": \"" + packageInstance.name + "\"}"); - } catch (Exception ex) { + new PackageApi.RefreshPackage(packageInstance.name).process(solrClient); + } catch (SolrServerException | IOException ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } } @@ -1079,9 +1062,8 @@ public void undeploy( solrClient, PackageUtils.getCollectionParamsPath(collection), "{set: {PKG_VERSIONS: {" + packageName + ": null}}}"); - SolrCLI.postJsonToSolr( - solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}"); - } catch (Exception ex) { + new PackageApi.RefreshPackage(packageName).process(solrClient); + } catch (Exception ex) { // postJsonToSolr above declares a plain throws Exception throw new SolrException(ErrorCode.SERVER_ERROR, ex); } diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java index 72d76ddf8443..a66a3a28bc34 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java @@ -60,7 +60,6 @@ public class PackageUtils { /** Represents a version which denotes the latest version available at the moment. */ public static String LATEST = "latest"; - public static String PACKAGE_PATH = "/api/cluster/package"; public static String CLUSTER_PLUGINS_PATH = "/api/cluster/plugin"; public static String REPOSITORIES_ZK_PATH = "/repositories.json"; public static String CLUSTERPROPS_PATH = "/api/cluster/zk/data/clusterprops.json"; diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java index 64af782131f4..8ff3fd5b15ec 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java @@ -41,21 +41,15 @@ import org.apache.solr.cli.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.client.api.util.SolrVersion; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.FileStoreApi; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.SystemApi; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -160,7 +154,7 @@ private String getRepositoriesJson(SolrZkClient zkClient) /** * Install a given package and version from the available repositories to Solr. The various steps * for doing so are, briefly, a) find upload a manifest to package store, b) download the - * artifacts and upload to package store, c) call {@link PackageAPI} to register the package. + * artifacts and upload to package store, c) call the Package API to register the package. */ private boolean installPackage(String packageName, String version) throws SolrException { SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); @@ -212,10 +206,9 @@ private boolean installPackage(String packageName, String version) throws SolrEx // Call Package API to add this version of the package runtime.printSuccess("Executing Package API to register this package..."); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = version; - add.pkg = packageName; - add.files = + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion(packageName); + addRequest.setVersion(version); + addRequest.setFiles( downloaded.stream() .map( file -> @@ -225,21 +218,12 @@ private boolean installPackage(String packageName, String version) throws SolrEx packageName, version, file.getFileName().toString())) - .collect(Collectors.toList()); - add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; - add.manifestSHA512 = manifestSHA512; - - GenericSolrRequest request = - new GenericV2SolrRequest(SolrRequest.METHOD.POST, PackageUtils.PACKAGE_PATH) { - @Override - public RequestWriter.ContentWriter getContentWriter(String expectedType) { - return new RequestWriter.StringPayloadContentWriter( - "{add:" + add.jsonStr() + "}", "application/json"); - } - }; + .collect(Collectors.toList())); + addRequest.setManifest("/package/" + packageName + "/" + version + "/manifest.json"); + addRequest.setManifestSHA512(manifestSHA512); try { - NamedList resp = solrClient.request(request); - runtime.printSuccess("Response: " + resp.jsonStr()); + addRequest.process(solrClient); + runtime.printSuccess("Package version registered successfully."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 5b73171eacc4..1290ea8d4f25 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -132,6 +132,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; +import org.apache.solr.pkg.ClusterPackage; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -841,8 +842,7 @@ private void loadInternal() { registerV2Api(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2Api(packageLoader.getPackageAPI().editAPI); - registerV2Api(packageLoader.getPackageAPI().readAPI); + registerV2Api(ClusterPackage.class); registerV2Api(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java index 88c0108822c5..5f6b4bac9a89 100644 --- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java @@ -50,7 +50,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -89,8 +89,8 @@ public ClusterFileStore( public UploadToFileStoreResponse uploadFile( String filePath, List sig, InputStream requestBody) { final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } try { coreContainer @@ -317,12 +317,12 @@ private void doDelete(String filePath, Boolean localDelete) { @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) public SolrJerseyResponse deleteFile(String filePath, Boolean localDelete) { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } validateName(filePath, true); - if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) { + if (coreContainer.getPackageLoader().getPackageStore().isJarInuse(filePath)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); } doDelete(filePath, localDelete); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java index cd89b1d3e9ed..b5915d77fd2d 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java @@ -47,8 +47,8 @@ import org.apache.solr.handler.SolrConfigHandler; import org.apache.solr.jersey.NullKeyTolerantMap; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.util.SolrPluginUtils; @@ -301,7 +301,7 @@ public static void filterByComponentName( List listeners = req.getCore().getPackageListeners().getListeners(); for (PackageListeners.Listener listener : listeners) { - Map infos = listener.packageDetails(); + Map infos = listener.packageDetails(); if (infos == null || infos.isEmpty()) continue; infos.forEach( (s, mapWriter) -> { diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index 76f62bd7ca9a..24813898f3b9 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -61,8 +61,8 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -201,7 +201,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(); } diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java new file mode 100644 index 000000000000..8a7dad3bf9a7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; + +import jakarta.inject.Inject; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.PackageApis; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.GetPackageResponse; +import org.apache.solr.client.api.model.ListPackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.jersey.PermissionName; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JAX-RS implementation of the package management API ({@code /api/cluster/packages}). + * + * @see PackageApis + */ +public class ClusterPackage extends JerseyResource implements PackageApis { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final int SYNC_MAX_RETRIES = 10; + private static final long SYNC_SLEEP_MS = 10L; + + private final CoreContainer coreContainer; + + @Inject + public ClusterPackage(CoreContainer coreContainer) { + this.coreContainer = coreContainer; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public ListPackagesResponse listPackages(Integer expectedVersion) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); + } + + final var response = new ListPackagesResponse(); + response.result = toPackageData(packageStore.pkgs); + return response; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public GetPackageResponse getPackage(String packageName, Integer expectedVersion) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); + } + + final List pkgVersions = packageStore.pkgs.packages.get(packageName); + if (pkgVersions == null || pkgVersions.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); + } + + final var response = new GetPackageResponse(); + response.result = new GetPackageResponse.PackageInfo(); + response.result.znodeVersion = packageStore.pkgs.znodeVersion; + response.result.name = packageName; + response.result.versions = + pkgVersions.stream().map(ClusterPackage::toPkgVersionResponse).collect(Collectors.toList()); + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse addPackageVersion( + String packageName, AddPackageVersionRequestBody requestBody) { + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + packageStore.requireEnabled(); + if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); + } + + final List errors = new ArrayList<>(); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), requestBody.files, true, errors::add); + if (!errors.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + } + + final PackageStore.Packages[] finalState = new PackageStore.Packages[1]; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages = packageStore.parseForUpdate(bytes); + List list = + packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); + for (PackageStore.PkgVersion pkgVersion : list) { + if (Objects.equals(pkgVersion.version, requestBody.version)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Version '" + requestBody.version + "' exists already"); + } + } + list.add(new PackageStore.PkgVersion(packageName, requestBody)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + packageStore.handleZkErr(e); + } + + if (finalState[0] != null) { + packageStore.pkgs = finalState[0]; + notifyAllNodesToSync(packageStore.pkgs.znodeVersion); + coreContainer.getPackageLoader().refreshPackageConf(); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse deletePackageVersion(String packageName, String version) { + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + packageStore.requireEnabled(); + + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages = packageStore.parseForUpdate(bytes); + List versions = packages.packages.get(packageName); + if (versions == null || versions.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); + } + int idxToRemove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, version)) { + idxToRemove = i; + break; + } + } + if (idxToRemove == -1) { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, "No such version: " + version); + } + versions.remove(idxToRemove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse refreshPackage(String packageName, Boolean localOnly) { + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (Boolean.TRUE.equals(localOnly)) { + packageStore.packageLoader.notifyListeners(packageName); + return response; + } + + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); + if (pkg == null) { + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); + } + // first refresh on the current node + packageStore.packageLoader.notifyListeners(packageName); + + final var request = new PackageApi.RefreshPackage(packageName); + request.setLocalOnly(true); + broadcastToLiveNodes(request, liveNode -> "Failed to refresh package on node: " + liveNode); + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse syncPackages(Integer expectedVersion) { + if (expectedVersion == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "expectedVersion is required"); + } + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + syncToVersion(packageStore, expectedVersion); + return response; + } + + private void syncToVersion(PackageStore packageStore, int expectedVersion) { + int origVersion = packageStore.pkgs.znodeVersion; + for (int i = 0; i < SYNC_MAX_RETRIES; i++) { + if (log.isDebugEnabled()) { + log.debug( + "my version is {} , and expected version {}", + packageStore.pkgs.znodeVersion, + expectedVersion); + } + if (packageStore.pkgs.znodeVersion >= expectedVersion) { + if (origVersion < packageStore.pkgs.znodeVersion) { + coreContainer.getPackageLoader().refreshPackageConf(); + } + return; + } + try { + Thread.sleep(SYNC_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + packageStore.pkgs = packageStore.readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } + } + } + + private void notifyAllNodesToSync(int expectedVersion) { + final var request = new PackageApi.SyncPackages(expectedVersion); + broadcastToLiveNodes( + request, + liveNode -> + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion); + } + + private void broadcastToLiveNodes( + SolrRequest request, Function errorMessage) { + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + request.processWithBaseUrl(solrClient, baseUrl, null); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, errorMessage.apply(liveNode), e); + } + } + } + + private static ListPackagesResponse.PackageData toPackageData(PackageStore.Packages packages) { + if (packages == null) { + return null; + } + final var data = new ListPackagesResponse.PackageData(); + data.znodeVersion = packages.znodeVersion; + data.packages = + packages.packages.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue().stream() + .map(ClusterPackage::toPkgVersionResponse) + .collect(Collectors.toList()))); + return data; + } + + private static ListPackagesResponse.PackageVersion toPkgVersionResponse( + PackageStore.PkgVersion pkgVersion) { + final var v = new ListPackagesResponse.PackageVersion(); + v.pkg = pkgVersion.pkg; + v.version = pkgVersion.version; + v.files = pkgVersion.files; + v.manifest = pkgVersion.manifest; + v.manifestSHA512 = pkgVersion.manifestSHA512; + return v; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java deleted file mode 100644 index 4574f7b3318f..000000000000 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ /dev/null @@ -1,476 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.pkg; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import org.apache.solr.api.Command; -import org.apache.solr.api.EndPoint; -import org.apache.solr.api.PayloadObj; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.ReflectMapWriter; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.SolrJacksonAnnotationInspector; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** This implements the public end points (/api/cluster/package) of package API. */ -public class PackageAPI { - public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; - - final CoreContainer coreContainer; - private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - private final SolrPackageLoader packageLoader; - Packages pkgs; - - public final Edit editAPI = new Edit(); - public final Read readAPI = new Read(); - - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { - this.coreContainer = coreContainer; - this.packageLoader = loader; - pkgs = new Packages(); - SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - pkgs = new Packages(); - // ignore - } - try { - registerListener(zkClient); - } catch (KeeperException | InterruptedException e) { - SolrZkClient.checkInterrupted(e); - } - } - - private void registerListener(SolrZkClient zkClient) - throws KeeperException, InterruptedException { - zkClient.exists( - SOLR_PKGS_PATH, - new Watcher() { - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher - if (Event.EventType.None.equals(event.getType())) { - return; - } - synchronized (this) { - log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); - // remake watch - final Watcher thisWatch = this; - refreshPackages(thisWatch); - } - } - }); - } - - public void refreshPackages(Watcher watcher) { - final Stat stat = new Stat(); - try { - final byte[] data = - coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); - pkgs = readPkgsFromZk(data, stat); - packageLoader.refreshPackageConf(); - } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); - } catch (KeeperException e) { - log.error("A ZK error has occurred", e); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); - } - } - - private Packages readPkgsFromZk(byte[] data, Stat stat) - throws KeeperException, InterruptedException { - - if (data == null || stat == null) { - stat = new Stat(); - data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); - } - Packages packages = null; - if (data == null || data.length == 0) { - packages = new Packages(); - } else { - try { - packages = mapper.readValue(data, Packages.class); - packages.znodeVersion = stat.getVersion(); - } catch (IOException e) { - // invalid data in packages - // TODO handle properly; - return new Packages(); - } - } - return packages; - } - - public static class Packages implements ReflectMapWriter { - @JsonProperty public int znodeVersion = -1; - - @JsonProperty public Map> packages = new LinkedHashMap<>(); - - public Packages copy() { - Packages p = new Packages(); - p.znodeVersion = this.znodeVersion; - p.packages = new LinkedHashMap<>(); - packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); - return p; - } - } - - public static class PkgVersion implements ReflectMapWriter { - - @JsonProperty("package") - public String pkg; - - @JsonProperty public String version; - - @JsonProperty public List files; - - @JsonProperty public String manifest; - - @JsonProperty public String manifestSHA512; - - public PkgVersion() {} - - public PkgVersion(PackagePayload.AddVersion addVersion) { - this.pkg = addVersion.pkg; - this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); - this.manifest = addVersion.manifest; - this.manifestSHA512 = addVersion.manifestSHA512; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof PkgVersion that) { - return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(version); - } - - @Override - public String toString() { - try { - return Utils.writeJson(this, new StringWriter(), false).toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public PkgVersion copy() { - PkgVersion result = new PkgVersion(); - result.pkg = this.pkg; - result.version = this.version; - result.files = this.files; - result.manifest = this.manifest; - result.manifestSHA512 = this.manifestSHA512; - return result; - } - } - - @EndPoint( - method = SolrRequest.METHOD.POST, - path = "/cluster/package", - permission = PACKAGE_EDIT_PERM) - public class Edit { - - @Command(name = "refresh") - public void refresh(PayloadObj payload) { - String p = payload.get(); - if (p == null) { - payload.addError("Package null"); - return; - } - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(p); - if (pkg == null) { - payload.addError("No such package: " + p); - return; - } - // first refresh my own - packageLoader.notifyListeners(p); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", p); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - request.processWithBaseUrl(solrClient, baseUrl, null); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } - } - - @Command(name = "add") - public void add(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.AddVersion add = payload.get(); - if (add.files.isEmpty()) { - payload.addError("No files specified"); - return; - } - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); - if (payload.hasError()) return; - Packages[] finalState = new Packages[1]; - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = - bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new Packages(); - } - List list = - packages.packages.computeIfAbsent(add.pkg, o -> new ArrayList<>()); - for (PkgVersion version : list) { - if (Objects.equals(version.version, add.version)) { - payload.addError("Version '" + add.version + "' exists already"); - return null; - } - } - list.add(new PkgVersion(add)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - handleZkErr(e); - } - if (finalState[0] != null) { - // succeeded in updating - pkgs = finalState[0]; - notifyAllNodesToSync(pkgs.znodeVersion); - packageLoader.refreshPackageConf(); - } - } - - @Command(name = "delete") - public void del(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.DelVersion delVersion = payload.get(); - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new Packages(); - } - - List versions = packages.packages.get(delVersion.pkg); - if (versions == null || versions.isEmpty()) { - payload.addError("No such package: " + delVersion.pkg); - return null; // no change - } - int idxToremove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, delVersion.version)) { - idxToremove = i; - break; - } - } - if (idxToremove == -1) { - payload.addError("No such version: " + delVersion.version); - return null; - } - versions.remove(idxToremove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - - public boolean isEnabled() { - return enablePackages; - } - - private boolean checkEnabled(CommandOperation payload) { - if (!enablePackages) { - payload.addError(ERR_MSG); - return false; - } - return true; - } - - public class Read { - @EndPoint( - method = SolrRequest.METHOD.GET, - path = {"/cluster/package/", "/cluster/package/{name}"}, - permission = PACKAGE_READ_PERM) - public void get(SolrQueryRequest req, SolrQueryResponse rsp) { - String refresh = req.getParams().get("refreshPackage"); - if (refresh != null) { - packageLoader.notifyListeners(refresh); - return; - } - - int expectedVersion = req.getParams().getInt("expectedVersion", -1); - if (expectedVersion != -1) { - syncToVersion(expectedVersion); - } - String name = req.getPathTemplateValues().get("name"); - if (name == null) { - rsp.add("result", pkgs); - } else { - rsp.add("result", Map.of(name, pkgs.packages.get(name))); - } - } - - private void syncToVersion(int expectedVersion) { - for (int i = 0; i < 10; i++) { - log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion); - if (pkgs.znodeVersion >= expectedVersion) { - // Blocks until any in-flight refresh started by the ZK watcher has been applied - packageLoader.refreshPackageConf(); - return; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - } - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - } - - void notifyAllNodesToSync(int expected) { - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expected)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - request.processWithBaseUrl(solrClient, baseUrl, null); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " + liveNode + " to sync expected package version: " + expected, - e); - } - } - } - - public void handleZkErr(Exception e) { - log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); - } - - public boolean isJarInuse(String path) { - Packages pkg = null; - try { - pkg = readPkgsFromZk(null, null); - } catch (KeeperException.NoNodeException nne) { - return false; - } catch (InterruptedException | KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - for (List vers : pkg.packages.values()) { - for (PkgVersion ver : vers) { - if (ver.files.contains(path)) { - return true; - } - } - } - return false; - } -} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index ff57f4775faa..36ffe53303f6 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -112,7 +112,7 @@ public interface Listener { String packageName(); /** fetch the package versions of class names */ - Map packageDetails(); + Map packageDetails(); /** A callback when the package is updated */ void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java index 272de544b6e7..8c9fa0a97889 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -42,7 +42,7 @@ public class PackageListeningClassLoader implements SolrClassLoader, PackageList private final Function pkgVersionSupplier; /** package name and the versions that we are tracking */ - private Map packageVersions = new ConcurrentHashMap<>(1); + private Map packageVersions = new ConcurrentHashMap<>(1); private Map classNameVsPackageName = new ConcurrentHashMap<>(); private final Runnable reloadAction; @@ -99,7 +99,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( p.getLatest(pkgVersionSupplier.apply(cName.pkg)); if (registerListener) { classNameVsPackageName.put(cName.original, cName.pkg); - PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion(); + PackageStore.PkgVersion pkgVersion = theVersion.getPkgVersion(); if (pkgVersion != null) packageVersions.put(cName.pkg, pkgVersion); } return theVersion; @@ -111,7 +111,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( @Override public MapWriter getPackageVersion(PluginInfo.ClassName cName) { if (cName.pkg == null) return null; - PackageAPI.PkgVersion p = packageVersions.get(cName.pkg); + PackageStore.PkgVersion p = packageVersions.get(cName.pkg); return p == null ? null : p::writeMap; } @@ -162,15 +162,15 @@ public String packageName() { } @Override - public Map packageDetails() { - Map result = new LinkedHashMap<>(); + public Map packageDetails() { + Map result = new LinkedHashMap<>(); classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v))); return result; } @Override public void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx) { - PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name); + PackageStore.PkgVersion currVer = packageVersions.get(pkg.name); if (currVer == null) { // not watching this return; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 887b7b84dc12..d67fd9cb146b 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -58,7 +58,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(info.cName.original, pkgVersion.getPkgVersion()); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java new file mode 100644 index 000000000000..99c055c0ea1f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.util.SolrJacksonAnnotationInspector; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds package data loaded from ZooKeeper ({@code /solr/packages.json}) and manages ZK watchers. + */ +public class PackageStore { + public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String ERR_MSG = + "Package loading is not enabled, start your nodes with -Dsolr.packages.enabled=true"; + + final CoreContainer coreContainer; + final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + final SolrPackageLoader packageLoader; + Packages pkgs; + + public PackageStore(CoreContainer coreContainer, SolrPackageLoader loader) { + this.coreContainer = coreContainer; + this.packageLoader = loader; + pkgs = new Packages(); + SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + try { + pkgs = readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + pkgs = new Packages(); + // ignore + } + try { + registerListener(zkClient); + } catch (KeeperException | InterruptedException e) { + SolrZkClient.checkInterrupted(e); + } + } + + private void registerListener(SolrZkClient zkClient) + throws KeeperException, InterruptedException { + zkClient.exists( + SOLR_PKGS_PATH, + new Watcher() { + + @Override + public void process(WatchedEvent event) { + // session events are not change events, and do not remove the watcher + if (Event.EventType.None.equals(event.getType())) { + return; + } + synchronized (this) { + log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); + // remake watch + final Watcher thisWatch = this; + refreshPackages(thisWatch); + } + } + }); + } + + public void refreshPackages(Watcher watcher) { + final Stat stat = new Stat(); + try { + final byte[] data = + coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); + pkgs = readPkgsFromZk(data, stat); + packageLoader.refreshPackageConf(); + } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { + log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); + } catch (KeeperException e) { + log.error("A ZK error has occurred", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } catch (InterruptedException e) { + // Restore the interrupted status + Thread.currentThread().interrupt(); + log.warn("Interrupted", e); + } + } + + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + + if (data == null || stat == null) { + stat = new Stat(); + data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + } + Packages packages; + if (data == null || data.length == 0) { + packages = new Packages(); + } else { + try { + packages = mapper.readValue(data, Packages.class); + packages.znodeVersion = stat.getVersion(); + } catch (IOException e) { + // invalid data in packages + // TODO handle properly; + return new Packages(); + } + } + return packages; + } + + public static class Packages implements ReflectMapWriter { + @JsonProperty public int znodeVersion = -1; + + @JsonProperty public Map> packages = new LinkedHashMap<>(); + + public Packages copy() { + Packages p = new Packages(); + p.znodeVersion = this.znodeVersion; + p.packages = new LinkedHashMap<>(); + packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); + return p; + } + } + + public static class PkgVersion implements ReflectMapWriter { + + @JsonProperty("package") + public String pkg; + + @JsonProperty public String version; + + @JsonProperty public List files; + + @JsonProperty public String manifest; + + @JsonProperty public String manifestSHA512; + + public PkgVersion() {} + + public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { + this.pkg = packageName; + this.version = addVersion.version; + this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.manifest = addVersion.manifest; + this.manifestSHA512 = addVersion.manifestSHA512; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof PkgVersion that) { + return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(version); + } + + @Override + public String toString() { + try { + return Utils.writeJson(this, new StringWriter(), false).toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public PkgVersion copy() { + PkgVersion result = new PkgVersion(); + result.pkg = this.pkg; + result.version = this.version; + result.files = this.files; + result.manifest = this.manifest; + result.manifestSHA512 = this.manifestSHA512; + return result; + } + } + + public boolean isEnabled() { + return enablePackages; + } + + public void requireEnabled() { + if (!isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ERR_MSG); + } + } + + public void handleZkErr(Exception e) { + log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); + } + + /** + * Deserializes {@code packages.json} bytes for a copy-on-write update, defaulting to a fresh + * instance on missing or invalid data. + */ + Packages parseForUpdate(byte[] bytes) { + try { + Packages packages = bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); + return packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + return new Packages(); + } + } + + public boolean isJarInuse(String path) { + Packages pkg; + try { + pkg = readPkgsFromZk(null, null); + } catch (KeeperException.NoNodeException nne) { + return false; + } catch (InterruptedException | KeeperException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + for (List vers : pkg.packages.values()) { + for (PkgVersion ver : vers) { + if (ver.files.contains(path)) { + return true; + } + } + } + return false; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index a512fe77914e..adbab3bbee0b 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -52,9 +52,9 @@ public class SolrPackageLoader implements Closeable { private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); - private PackageAPI.Packages myCopy = new PackageAPI.Packages(); + private PackageStore.Packages myCopy = new PackageStore.Packages(); - private PackageAPI packageAPI; + private PackageStore packageStore; public Optional getPackageVersion(String pkg, String version) { SolrPackage p = packageClassLoaders.get(pkg); @@ -64,12 +64,12 @@ public Optional getPackageVersion(String pkg, String versio public SolrPackageLoader(CoreContainer coreContainer) { this.coreContainer = coreContainer; - packageAPI = new PackageAPI(coreContainer, this); + packageStore = new PackageStore(coreContainer, this); refreshPackageConf(); } - public PackageAPI getPackageAPI() { - return packageAPI; + public PackageStore getPackageStore() { + return packageStore; } public SolrPackage getPackage(String key) { @@ -82,12 +82,12 @@ public Map getPackages() { public synchronized void refreshPackageConf() { log.debug( - "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion); + "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageStore.pkgs.znodeVersion); List updated = new ArrayList<>(); - Map> modified = getModified(myCopy, packageAPI.pkgs); + Map> modified = getModified(myCopy, packageStore.pkgs); - for (Map.Entry> e : modified.entrySet()) { + for (Map.Entry> e : modified.entrySet()) { if (e.getValue() != null) { SolrPackage p = packageClassLoaders.get(e.getKey()); if (e.getValue() != null && p == null) { @@ -106,14 +106,14 @@ public synchronized void refreshPackageConf() { } } coreContainer.forEachLoadedCore(core -> core.getPackageListeners().packagesUpdated(updated)); - myCopy = packageAPI.pkgs; + myCopy = packageStore.pkgs; } - public Map> getModified( - PackageAPI.Packages old, PackageAPI.Packages newPkgs) { - Map> changed = new HashMap<>(); - for (Map.Entry> e : newPkgs.packages.entrySet()) { - List versions = old.packages.get(e.getKey()); + public Map> getModified( + PackageStore.Packages old, PackageStore.Packages newPkgs) { + Map> changed = new HashMap<>(); + for (Map.Entry> e : newPkgs.packages.entrySet()) { + List versions = old.packages.get(e.getKey()); if (versions != null) { if (!Objects.equals(e.getValue(), versions)) { if (log.isInfoEnabled()) { @@ -131,7 +131,7 @@ public Map> getModified( // some packages are deleted altogether for (String s : old.packages.keySet()) { if (!newPkgs.packages.keySet().contains(s)) { - log.info("Package: {} is removed althogether", s); + log.info("Package: {} is removed altogether", s); changed.put(s, null); } } @@ -167,8 +167,8 @@ public Set allVersions() { return myVersions.keySet(); } - private synchronized void updateVersions(List modified) { - for (PackageAPI.PkgVersion v : modified) { + private synchronized void updateVersions(List modified) { + for (PackageStore.PkgVersion v : modified) { Version version = myVersions.get(v.version); if (version == null) { log.info( @@ -189,7 +189,7 @@ private synchronized void updateVersions(List modified) { } Set newVersions = new HashSet<>(); - for (PackageAPI.PkgVersion v : modified) { + for (PackageStore.PkgVersion v : modified) { newVersions.add(v.version); } for (String s : new HashSet<>(myVersions.keySet())) { @@ -253,7 +253,7 @@ public class Version implements MapWriter, Closeable { private final SolrPackage parent; private SolrResourceLoader loader; - private final PackageAPI.PkgVersion version; + private final PackageStore.PkgVersion version; @Override public void writeMap(EntryWriter ew) throws IOException { @@ -261,7 +261,7 @@ public void writeMap(EntryWriter ew) throws IOException { version.writeMap(ew); } - Version(SolrPackage parent, PackageAPI.PkgVersion v) { + Version(SolrPackage parent, PackageStore.PkgVersion v) { this.parent = parent; this.version = v; List paths = new ArrayList<>(); @@ -288,7 +288,7 @@ public String getVersion() { return version.version; } - public PackageAPI.PkgVersion getPkgVersion() { + public PackageStore.PkgVersion getPkgVersion() { return version.copy(); } @@ -325,7 +325,7 @@ static class PackageResourceLoader extends SolrResourceLoader { @Override public boolean addToCoreAware(T obj) { // do not do anything - // this class is not aware of a SolrCore and it is totally not tied to + // this class is not aware of a SolrCore, and it is totally not tied to // the lifecycle of SolrCore. So, this returns 'false' & it should be // taken care of by the caller return false; diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 7a6affe1b9e4..466e99844496 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -32,6 +32,7 @@ import java.util.concurrent.Callable; import java.util.function.Predicate; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.solr.client.api.model.UploadToFileStoreResponse; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -39,12 +40,9 @@ import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SimpleSolrResponse; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.SuppressForbidden; @@ -96,7 +94,7 @@ public void testFileStoreManagement() throws Exception { "/package/mypkg/v1.0/runtimelibs.jar", "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - NavigableObject rsp = + UploadToFileStoreResponse rsp = postFile( cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"), @@ -104,8 +102,7 @@ public void testFileStoreManagement() throws Exception { "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); assertTrue( - Objects.requireNonNullElse(rsp._getStr("message"), "") - .contains("File with same metadata exists ")); + Objects.requireNonNullElse(rsp.message, "").contains("File with same metadata exists ")); assertResponseValues( 10, @@ -348,25 +345,33 @@ public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster clu false); } - public static NavigableObject postFile( + public static UploadToFileStoreResponse postFile( SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { - String resource = "/cluster/filestore/files" + name; - ModifiableSolrParams params = new ModifiableSolrParams(); - params.add("sig", sig); - V2Response rsp = - new V2Request.Builder(resource) - .withMethod(SolrRequest.METHOD.PUT) - .withPayload(buffer) - .forceV2(true) - .withMimeType("application/octet-stream") - .withParams(params) - .build() - .process(client); - assertEquals(name, rsp.getResponse().get(CommonParams.FILE)); + final var uploadReq = new FileStoreApi.UploadFile(name, new ByteBufferInputStream(buffer)); + if (sig != null) { + uploadReq.setSig(List.of(sig)); + } + final UploadToFileStoreResponse rsp = uploadReq.process(client); + assertEquals(name, rsp.file); return rsp; } + /** + * Upload a file from the test classpath to the distributed file store under the given path, then + * wait until every node in the cluster reports the expected SHA-512 for that path. + */ + public static void postFileAndWait( + MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { + ByteBuffer fileContent = getFileContent(fname); + @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() + String sha512 = DigestUtils.sha512Hex(fileContent.array()); + + postFile(cluster.getSolrClient(), fileContent, path, sig); + + checkAllNodesForFile(cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); + } + /** * Read and return the contents of the file-like resource * diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index deed5251cf13..1aee5a42e8d1 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -41,8 +41,8 @@ import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.request.beans.PluginMeta; import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.ClusterSingleton; @@ -57,10 +57,9 @@ import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.filestore.TestDistribFileStore.Fetcher; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; -import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -92,7 +91,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return null; // only used to print meta information } @@ -290,12 +289,12 @@ public void testApiFromPackage() throws Exception { byte[] derFile = readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.1.jar.bin", FILE1, "pmrmWCDafdNpYle2rueAGnU2J6NYlcAey9mkZYbqh+5RdYo2Ln+llLF9voyRj+DDivK9GV1XdtKvD9rgCxlD7Q=="); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.2.jar.bin", FILE2, @@ -304,23 +303,16 @@ public void testApiFromPackage() throws Exception { // We have two versions of the plugin in 2 different jar files. they are already uploaded to // the package store listener.reset(); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = List.of(FILE1); - V2Request addPkgVersionReq = - new V2Request.Builder("/cluster/package") - .forceV2(forceV2) - .POST() - .withPayload(Map.of("add", add)) - .build(); + PackageApi.AddPackageVersion addPkgVersionReq = new PackageApi.AddPackageVersion("mypkg"); + addPkgVersionReq.setVersion("1.0"); + addPkgVersionReq.setFiles(List.of(FILE1)); addPkgVersionReq.process(cluster.getSolrClient()); assertTrue( "core package listeners did not notify", listener.waitFor(cluster.getJettySolrRunners().size())); waitForAllNodesToSync( - "/cluster/package", + "/cluster/packages", Map.of( ":result:packages:mypkg[0]:version", "1.0", @@ -331,7 +323,7 @@ public void testApiFromPackage() throws Exception { PluginMeta plugin = new PluginMeta(); plugin.name = "myplugin"; plugin.klass = "mypkg:org.apache.solr.handler.MyPlugin"; - plugin.version = add.version; + plugin.version = "1.0"; final V2Request addPluginReq = postPlugin(Map.of("add", plugin)); addPluginReq.process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); @@ -347,12 +339,12 @@ public void testApiFromPackage() throws Exception { TestDistribFileStore.assertResponseValues(invokePlugin, Map.of("/myplugin.version", "1.0")); // now let's upload the jar file for version 2.0 of the plugin - add.version = "2.0"; - add.files = List.of(FILE2); + addPkgVersionReq.setVersion("2.0"); + addPkgVersionReq.setFiles(List.of(FILE2)); addPkgVersionReq.process(cluster.getSolrClient()); // here the plugin version is updated - plugin.version = add.version; + plugin.version = "2.0"; postPlugin(Map.of("update", plugin)).process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java index e581c574016c..47fd64270c47 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java @@ -119,7 +119,7 @@ public void testFramework() { } public void testPayload() { - String json = "{package:pkg1, version: '0.1', files :[a.jar, b.jar]}"; + String json = "{name:item1, version: '0.1', files :[a.jar, b.jar]}"; Utils.fromJSONString(json); ApiBag apiBag = new ApiBag(false); @@ -129,25 +129,25 @@ public void testPayload() { assertEquals("POST", spec._getStr("/methods[0]")); assertEquals("POST", spec._getStr("/methods[0]")); - assertEquals("/cluster/package", spec._getStr("/url/paths[0]")); - assertEquals("string", spec._getStr("/commands/add/properties/package/type")); + assertEquals("/foos", spec._getStr("/url/paths[0]")); + assertEquals("string", spec._getStr("/commands/add/properties/name/type")); assertEquals("array", spec._getStr("/commands/add/properties/files/type")); assertEquals("string", spec._getStr("/commands/add/properties/files/items/type")); assertEquals("string", spec._getStr("/commands/delete/items/type")); SolrQueryResponse rsp = v2ApiInvoke( apiBag, - "/cluster/package", + "/foos", "POST", new ModifiableSolrParams(), new ByteArrayInputStream( - "{add:{package:mypkg, version: '1.0', files : [a.jar, b.jar]}}".getBytes(UTF_8))); + "{add:{name:myfoo, version: '1.0', files : [a.jar, b.jar]}}".getBytes(UTF_8))); - AddVersion addversion = (AddVersion) rsp.getValues().get("add"); - assertEquals("mypkg", addversion.pkg); - assertEquals("1.0", addversion.version); - assertEquals("a.jar", addversion.files.get(0)); - assertEquals("b.jar", addversion.files.get(1)); + FooVersion fooVersion = (FooVersion) rsp.getValues().get("add"); + assertEquals("myfoo", fooVersion.name); + assertEquals("1.0", fooVersion.version); + assertEquals("a.jar", fooVersion.files.get(0)); + assertEquals("b.jar", fooVersion.files.get(1)); apiBag.registerObject(new C()); rsp = @@ -157,24 +157,24 @@ public void testPayload() { "POST", new ModifiableSolrParams(), new ByteArrayInputStream( - "{\"package\":\"mypkg\", \"version\": \"1.0\", \"files\" : [\"a.jar\", \"b.jar\"]}" + "{\"name\":\"myfoo\", \"version\": \"1.0\", \"files\" : [\"a.jar\", \"b.jar\"]}" .getBytes(UTF_8))); - assertEquals("mypkg", rsp.getValues()._getStr("payload/package")); + assertEquals("myfoo", rsp.getValues()._getStr("payload/name")); assertEquals("1.0", rsp.getValues()._getStr("payload/version")); } public static class C { @EndPoint(path = "/path1", method = POST, permission = PermissionNameProvider.Name.ALL) - public void m1(PayloadObj add) { + public void m1(PayloadObj add) { add.getResponse().add("payload", add.get()); } } - @EndPoint(method = POST, path = "/cluster/package", permission = PermissionNameProvider.Name.ALL) + @EndPoint(method = POST, path = "/foos", permission = PermissionNameProvider.Name.ALL) public static class ApiTest { @Command(name = "add") - public void add(SolrQueryRequest req, SolrQueryResponse rsp, AddVersion addVersion) { - rsp.add("add", addVersion); + public void add(SolrQueryRequest req, SolrQueryResponse rsp, FooVersion fooVersion) { + rsp.add("add", fooVersion); } @Command(name = "delete") @@ -183,9 +183,9 @@ public void del(SolrQueryRequest req, SolrQueryResponse rsp, List names) } } - public static class AddVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; + public static class FooVersion implements ReflectMapWriter { + @JsonProperty(value = "name", required = true) + public String name; @JsonProperty(value = "version", required = true) public String version; diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java new file mode 100644 index 000000000000..e8785c7aae9a --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.util.List; +import org.apache.solr.client.api.model.GetPackageResponse; +import org.apache.solr.client.api.model.ListPackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.RemoteSolrException; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests for the JAX-RS-based {@link ClusterPackage}. + * + *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its + * cluster-level operations. A one-node SolrCloud cluster is used instead. + */ +public class ClusterPackageTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("solr.packages.enabled", "true"); + configureCluster(1) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + + @Test + public void testListPackagesReturnsResult() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + ListPackagesResponse response = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null response from GET /cluster/packages", response); + assertNotNull("Expected 'result' field in GET /cluster/packages response", response.result); + } + } + + @Test + public void testAddAndDeletePackageVersion() throws Exception { + String FILE1 = "/pkgapitestpkg/runtimelibs.jar"; + + // Upload a key and a signed jar file to the filestore + byte[] derFile = TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + TestDistribFileStore.postFileAndWait( + cluster, + "runtimecode/runtimelibs.jar.bin", + FILE1, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // Add a package version via POST /cluster/packages/{name}/versions + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of(FILE1)); + addRequest.process(client); + + // Verify the package was added via GET /cluster/packages + ListPackagesResponse listResponse = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response", listResponse); + assertNotNull("Expected non-null result", listResponse.result); + assertNotNull( + "Expected pkgapitestpkg in packages", listResponse.result.packages.get("pkgapitestpkg")); + assertFalse( + "Expected at least one version", + listResponse.result.packages.get("pkgapitestpkg").isEmpty()); + + // Verify GET /cluster/packages/{name} returns only this package + GetPackageResponse getByNameResponse = + new PackageApi.GetPackage("pkgapitestpkg").process(client); + assertNotNull("Expected non-null get-by-name response", getByNameResponse); + assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); + assertEquals("pkgapitestpkg", getByNameResponse.result.name); + assertFalse( + "Expected at least one version in get-by-name response", + getByNameResponse.result.versions.isEmpty()); + + // Delete the package version via DELETE /cluster/packages/{name}/versions/{version} + new PackageApi.DeletePackageVersion("pkgapitestpkg", "1.0").process(client); + + // Verify it's deleted + ListPackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response after delete", listAfterDelete); + assertNotNull("Expected non-null result after delete", listAfterDelete.result); + // After deleting the only version, the package entry should be empty or absent + List versionsAfterDelete = listAfterDelete.result.packages.get("pkgapitestpkg"); + assertTrue( + "Expected no versions after delete", + versionsAfterDelete == null || versionsAfterDelete.isEmpty()); + } + } + + @Test + public void testAddPackageVersionValidatesFiles() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("testpkg_invalid"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of("/nonexistent/file.jar")); + + RemoteSolrException ex = + expectThrows(RemoteSolrException.class, () -> addRequest.process(client)); + assertEquals("Expected 400 for non-existent file", 400, ex.code()); + assertTrue( + "Expected error message to mention the file: " + ex.getMessage(), + ex.getMessage().contains("No such file")); + } + } + + @Test + public void testRefreshNonExistentPackage() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + RemoteSolrException ex = + expectThrows( + RemoteSolrException.class, + () -> new PackageApi.RefreshPackage("nonexistentpkg_test").process(client)); + assertEquals("Expected 404 for non-existent package", 404, ex.code()); + assertTrue( + "Expected error message to mention the package: " + ex.getMessage(), + ex.getMessage().contains("No such package")); + } + } + + @Test + public void testGetNonExistentPackage() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + RemoteSolrException ex = + expectThrows( + RemoteSolrException.class, + () -> new PackageApi.GetPackage("nonexistentpkg_get").process(client)); + assertEquals("Expected 404 for non-existent package", 404, ex.code()); + assertTrue( + "Expected error message to mention the package: " + ex.getMessage(), + ex.getMessage().contains("No such package")); + } + } + + @Test + public void testRefreshPackageLocalOnlyAcceptsUnknownPackage() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // localOnly=true is the internal inter-node refresh signal: it calls notifyListeners on this + // node only and returns OK even when the package doesn't exist. + PackageApi.RefreshPackage req = new PackageApi.RefreshPackage("nonexistentpkg_refresh"); + req.setLocalOnly(true); + SolrJerseyResponse response = req.process(client); + assertNotNull(response); + } + } + + @Test + public void testSyncPackagesCommandRequiresExpectedVersion() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // expectedVersion at-or-below current is a no-op in syncToVersion and must return promptly. + PackageApi.SyncPackages req = new PackageApi.SyncPackages(0); + SolrJerseyResponse response = req.process(client); + assertNotNull(response); + } + } + + @Test + public void testListAndGetPackageAcceptExpectedVersionQueryParam() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // expectedVersion at-or-below current is a no-op in syncToVersion and must return promptly. + PackageApi.ListPackages listReq = new PackageApi.ListPackages(); + listReq.setExpectedVersion(0); + ListPackagesResponse listResponse = listReq.process(client); + assertNotNull(listResponse); + assertNotNull(listResponse.result); + + // expectedVersion is honored even on the path that ends in a 404 for an unknown package. + PackageApi.GetPackage getReq = new PackageApi.GetPackage("nonexistent_expected_version_pkg"); + getReq.setExpectedVersion(0); + RemoteSolrException ex = + expectThrows(RemoteSolrException.class, () -> getReq.process(client)); + assertEquals("Expected 404 for non-existent package", 404, ex.code()); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java similarity index 82% rename from solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java rename to solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java index 8faac9b68f9e..f5854b6e296f 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java @@ -27,11 +27,11 @@ import java.security.Signature; import java.util.Base64; import java.util.List; -import java.util.Map; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.request.FileStoreApi; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.filestore.FileStoreAPI; @@ -39,7 +39,12 @@ import org.junit.Before; import org.junit.Test; -public class PackageStoreSchemaPluginsTest extends SolrCloudTestCase { +/** + * Verifies that a configset which pins a specific package version (via {@code params.json}) + * resolves classes from that exact version's jar — even when other versions of the same package are + * also registered. + */ +public class ConfigsetPinnedPackageVersionTest extends SolrCloudTestCase { private static final KeyPair KEY_PAIR; @@ -123,32 +128,17 @@ private static String signature(byte[] content) throws Exception { private void uploadPluginJar(String version, Path jarPath) throws Exception { var pluginRequest = - new V2Request.Builder("/cluster/filestore/files/my-plugin/plugin-" + version + ".jar") - .PUT() - .withParams(params("sig", signature(Files.readAllBytes(jarPath)))) - .withPayload(Files.newInputStream(jarPath)) - .forceV2(true) - .build(); - processRequest(client, pluginRequest); + new FileStoreApi.UploadFile( + "/my-plugin/plugin-" + version + ".jar", Files.newInputStream(jarPath)); + pluginRequest.setSig(List.of(signature(Files.readAllBytes(jarPath)))); + pluginRequest.process(client); } private void registerPackage(String version) throws Exception { - var packageRequest = - new V2Request.Builder("/cluster/package") - .POST() - .forceV2(true) - .withPayload( - Map.of( - "add", - Map.of( - "package", - "mypkg", - "version", - version, - "files", - List.of("/my-plugin/plugin-" + version + ".jar")))) - .build(); - processRequest(client, packageRequest); + var addRequest = new PackageApi.AddPackageVersion("mypkg"); + addRequest.setVersion(version); + addRequest.setFiles(List.of("/my-plugin/plugin-" + version + ".jar")); + addRequest.process(client); } private void createCollection() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java new file mode 100644 index 000000000000..7e94025ffa7e --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; +import static org.apache.solr.filestore.TestDistribFileStore.readFile; +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.util.Arrays; +import java.util.Map; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.core.SolrCore; +import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Verifies that bumping a package version that backs a schema plugin causes the affected core's + * schema to reload and pick up classes from the new package's classloader. + */ +public class PackageSchemaReloadTest extends SolrCloudTestCase { + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + System.setProperty("solr.packages.enabled", "true"); + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf1", configset("schema-package")) + .configure(); + } + + @After + @Override + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + super.tearDown(); + } + + @Test + @SuppressWarnings("ReferenceEquality") // Identity, not equality, of the schema instance matters + public void testSchemaReloadOnPackageVersionBump() throws Exception { + String COLLECTION_NAME = "testSchemaLoadingColl"; + System.setProperty("managed.schema.mutable", "true"); + + IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core + + String FILE1 = "/schemapkg/schema-plugins.jar"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + postFileAndWait( + cluster, + "runtimecode/schema-plugins.jar.bin", + FILE1, + "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); + + String FILE2 = "/schemapkg/payload-component.jar"; + postFileAndWait( + cluster, + "runtimecode/payload-component.jar.bin", + FILE2, + "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); + + // upload package v1.0 + PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); + addReq.setVersion("1.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/packages") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[0]:version", + "1.0", + ":result:packages:schemapkg[0]:files[0]", + FILE1)); + + CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + // make note of the schema instance for one of the cores + SolrCore.Provider coreProvider = + cluster.getJettySolrRunners().stream() + .flatMap( + jetty -> + jetty.getCoreContainer().getAllCoreNames().stream() + .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) + .findFirst() + .orElseThrow(); + + coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); + + // upload package v2.0 + addReq.setVersion("2.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/packages") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[1]:version", + "2.0", + ":result:packages:schemapkg[1]:files[0]", + FILE1)); + + // even though package version 2.0 uses exactly the same files + // as version 1.0, the core schema should still reload, and + // the core should be associated with a different schema instance + TestDistribFileStore.assertResponseValues( + 10, + () -> { + coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); + return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); + }, + Map.of("schemaReloaded", "yes")); + + // after the reload, the custom field type class now comes from package v2.0 + String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; + + FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); + + FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); + + assertNotEquals( + "my.pkg.MyTextField classes should be from different classloaders", + fieldTypeV1.getClass(), + fieldTypeV2.getClass()); + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index b519e2c3d3a1..85210bd32dc6 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -17,11 +17,10 @@ package org.apache.solr.pkg; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; -import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; import static org.apache.solr.filestore.TestDistribFileStore.checkAllNodesForFile; +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; import static org.apache.solr.filestore.TestDistribFileStore.readFile; import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; @@ -30,15 +29,12 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.Callable; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; @@ -49,14 +45,13 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; import org.apache.solr.common.SolrInputDocument; @@ -64,7 +59,6 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrCore; @@ -74,21 +68,32 @@ import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.QParser; import org.apache.solr.search.QParserPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.util.LogLevel; import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.zookeeper.data.Stat; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.HttpClient; import org.junit.After; import org.junit.Before; import org.junit.Test; -@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") +/** + * Multi-node tests covering how Solr consumes packages at runtime: plugin loading via + * {@link SolrPackageLoader} / {@link PackagePluginHolder}, core reload on package version change, + * cross-node refresh fan-out, and package-file download to newly-joined nodes. + * + *

API-contract tests for the {@link ClusterPackage} JAX-RS endpoints live in {@link + * ClusterPackageTest}. Schema-reload-on-version-bump lives in {@link PackageSchemaReloadTest}. + * Configset-pinned-version behavior lives in {@link ConfigsetPinnedPackageVersionTest}. + * + *

Note on the class name: {@code BasePatternReplaceCharFilterFactory} and {@code + * BaseWhitespaceTokenizerFactory} (defined below) are referenced by fully-qualified name from + * pre-compiled test jars in {@code src/test-files/runtimecode/}. Renaming this class would break + * those binary fixtures. + */ +@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.ClusterPackage=DEBUG") public class TestPackages extends SolrCloudTestCase { @Before @@ -132,22 +137,15 @@ public void testCoreReloadingPlugin() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = Arrays.asList(new String[] {FILE1}); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1)); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), @@ -162,14 +160,14 @@ public void testCoreReloadingPlugin() throws Exception { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); verifyComponent( - cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version); + cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", "1.0"); - add.version = "2.0"; + req.setVersion("2.0"); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), @@ -213,16 +211,9 @@ public void testPluginLoading() throws Exception { EXPR1, "ZOT11arAiPmPZYOHzqodiNnxO9pRyRozWZEBX8XGjU1/HJptFnZK+DI7eXnUtbNaMcbXE2Ze8hh4M/eGyhY8BQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = Arrays.asList(new String[] {FILE1, URP1, EXPR1}); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1, URP1, EXPR1)); req.process(cluster.getSolrClient()); @@ -233,7 +224,7 @@ public void testPluginLoading() throws Exception { TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), @@ -344,8 +335,8 @@ public void testPluginLoading() throws Exception { URP2, "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw=="); // add the version using package API - add.version = "1.1"; - add.files = Arrays.asList(new String[] {FILE2, URP2, EXPR1}); + req.setVersion("1.1"); + req.setFiles(List.of(FILE2, URP2, EXPR1)); req.process(cluster.getSolrClient()); verifyComponent( @@ -373,8 +364,8 @@ public void testPluginLoading() throws Exception { FILE3, "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); - add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // now let's verify that the classes are updated @@ -404,16 +395,7 @@ public void testPluginLoading() throws Exception { assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.pkg = "mypkg"; - delVersion.version = "1.0"; - V2Request delete = - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.POST) - .forceV2(true) - .withPayload(Map.of("delete", delVersion)) - .build(); - delete.process(cluster.getSolrClient()); + new PackageApi.DeletePackageVersion("mypkg", "1.0").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -424,9 +406,8 @@ public void testPluginLoading() throws Exception { verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1"); - // now remove the hughest version. So, it will roll back to the next highest one - delVersion.version = "2.1"; - delete.process(cluster.getSolrClient()); + // now remove the highest version. So, it will roll back to the next highest one + new PackageApi.DeletePackageVersion("mypkg", "2.1").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "1.1"); @@ -448,8 +429,8 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { } }.setRequiresCollection(true).process(cluster.getSolrClient()); - add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // the collections mypkg is set to use version 1.1 @@ -475,12 +456,7 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { // now, let's force every collection using 'mypkg' to refresh // so that it uses version 2.1 - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{refresh : mypkg}") - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.RefreshPackage("mypkg").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -578,130 +554,6 @@ private void verifyComponent( ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version)); } - @Test - @SuppressWarnings("unchecked") - public void testAPI() throws Exception { - String errPath = "/details[0]/errorMessages[0]"; - String FILE1 = "/mypkg/v.0.12/jar_a.jar"; - String FILE2 = "/mypkg/v.0.12/jar_b.jar"; - String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "0.12"; - add.pkg = "test_pkg"; - add.files = List.of(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - - // the files are not yet there. The command should fail with error saying "No such file" - expectError(req, cluster.getSolrClient(), errPath, "No such file:"); - - // post the jar file. No signature is sent - postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null); - - add.files = List.of(FILE1); - expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); - // now we upload the keys - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - // and upload the same file with a different name, but it has proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs.jar.bin", - FILE2, - "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - // with correct signature - // after uploading the file, let's delete the keys to see if we get proper error message - add.files = List.of(FILE2); - /*expectError(req, cluster.getSolrClient(), errPath, - "ZooKeeper does not have any public keys");*/ - - // Now lets' put the keys back - - // this time we have a file with proper signature, public keys are in ZK - // so the add {} command should succeed - req.process(cluster.getSolrClient()); - - // Now verify the data in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.12", ":packages:test_pkg[0]:files[0]", FILE2)); - - // post a new jar with a proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs_v2.jar.bin", - FILE3, - "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); - - // this time we are adding the second version of the package (0.13) - add.version = "0.13"; - add.pkg = "test_pkg"; - add.files = List.of(FILE3); - - // this request should succeed - req.process(cluster.getSolrClient()); - // no verify the data (/packages.json) in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); - - // Now we will just delete one version - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.version = "0.1"; // this version does not exist - delVersion.pkg = "test_pkg"; - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("delete", delVersion)) - .build(); - - // we are expecting an error - expectError(req, cluster.getSolrClient(), errPath, "No such version:"); - - delVersion.version = "0.12"; // correct version. Should succeed - req.process(cluster.getSolrClient()); - // Verify with ZK that the data is correct - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.13", ":packages:test_pkg[0]:files[0]", FILE3)); - - // So far we have been verifying the details with ZK directly - // use the package read API to verify with each node that it has the correct data - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - String path = jetty.getBaseURLV2().toString() + "/cluster/package?wt=javabin"; - TestDistribFileStore.assertResponseValues( - 10, - new Callable() { - @Override - public NavigableObject call() throws Exception { - HttpClient solrClient = jetty.getSolrClient().getHttpClient(); - byte[] bytes = solrClient.GET(path).getContent(); - return (NavigableObject) new JavaBinCodec().unmarshal(bytes); - } - }, - Map.of( - ":result:packages:test_pkg[0]:version", - "0.13", - ":result:packages:test_pkg[0]:files[0]", - FILE3)); - } - } - public static class C extends RequestHandlerBase implements SolrCoreAware { static boolean informCalled = false; @@ -739,138 +591,8 @@ public QParser createParser( } } - @Test - @SuppressWarnings("ReferenceEquality") - public void testSchemaPlugins() throws Exception { - String COLLECTION_NAME = "testSchemaLoadingColl"; - System.setProperty("managed.schema.mutable", "true"); - - IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core - - String FILE1 = "/schemapkg/schema-plugins.jar"; - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - postFileAndWait( - cluster, - "runtimecode/schema-plugins.jar.bin", - FILE1, - "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); - - String FILE2 = "/schemapkg/payload-component.jar"; - postFileAndWait( - cluster, - "runtimecode/payload-component.jar.bin", - FILE2, - "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); - - // upload package v1.0 - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "schemapkg"; - add.files = Arrays.asList(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - req.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[0]:version", - "1.0", - ":result:packages:schemapkg[0]:files[0]", - FILE1)); - - CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); - - // make note of the schema instance for one of the cores - SolrCore.Provider coreProvider = - cluster.getJettySolrRunners().stream() - .flatMap( - jetty -> - jetty.getCoreContainer().getAllCoreNames().stream() - .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) - .findFirst() - .orElseThrow(); - - coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); - - // upload package v2.0 - add = new PackagePayload.AddVersion(); - add.version = "2.0"; - add.pkg = "schemapkg"; - add.files = Arrays.asList(FILE1, FILE2); - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - req.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[1]:version", - "2.0", - ":result:packages:schemapkg[1]:files[0]", - FILE1)); - - // even though package version 2.0 uses exactly the same files - // as version 1.0, the core schema should still reload, and - // the core should be associated with a different schema instance - TestDistribFileStore.assertResponseValues( - 10, - () -> { - coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); - return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); - }, - Map.of("schemaReloaded", "yes")); - - // after the reload, the custom field type class now comes from package v2.0 - String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; - - FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); - - FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); - - assertNotEquals( - "my.pkg.MyTextField classes should be from different classloaders", - fieldTypeV1.getClass(), - fieldTypeV2.getClass()); - } - - public static void postFileAndWait( - MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { - ByteBuffer fileContent = getFileContent(fname); - @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() - String sha512 = DigestUtils.sha512Hex(fileContent.array()); - - TestDistribFileStore.postFile( - cluster.getSolrClient(), fileContent, path, sig); // has file, but no signature - - TestDistribFileStore.checkAllNodesForFile( - cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); - } - - private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) + private void expectError( + SolrRequest req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException { try { req.process(client); @@ -900,7 +622,7 @@ public BaseWhitespaceTokenizerFactory(Map args) { } /* - //copy the jav files to a package and then run the main method + //copy the java files to a package and then run the main method public static void main(String[] args) throws Exception { persistZip("/tmp/x.jar", MyPatternReplaceCharFilterFactory.class, MyTextField.class, MyWhitespaceTokenizerFactory.class); }*/ diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc index b11c8a45ad3e..19db9a2b35ce 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc @@ -178,18 +178,19 @@ For example: == API Endpoints -* `GET /api/cluster/package` Get the list of packages -* `POST /api/cluster/package` edit packages -** `add` command: add a version of a package -** `delete` command: delete a version of a package +* `GET /api/cluster/packages` Get the list of all packages +* `GET /api/cluster/packages/{name}` Get the details of a specific package +* `POST /api/cluster/packages/{name}/versions` Add a version of a package +* `DELETE /api/cluster/packages/{name}/versions/{version}` Delete a specific version of a package +* `POST /api/cluster/packages/{name}/refresh` Refresh a package on all nodes in the cluster === How to Upgrade? -Use the `add` command to add a version that is higher than the current version. +Use the add version endpoint to add a version that is higher than the current version. === How to Downgrade? -Use the `delete` command to delete the highest version and choose the next highest version. +Use the delete version endpoint to delete the highest version and choose the next highest version. === Using Multiple Versions in Parallel @@ -244,18 +245,16 @@ The plugins loaded from packages cannot depend on core level classes. + [source,bash] ---- -curl 'http://localhost:8983/api/cluster/package' -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"1.0", - "files" :["/mypkg/1.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/packages/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"1.0","files":["/mypkg/1.0/myplugins.jar"]}' ---- . Verify the created package: + [source,bash] ---- -curl 'http://localhost:8983/api/cluster/package?omitHeader=true' +curl http://localhost:8983/api/cluster/packages?omitHeader=true ---- + [source,json] @@ -350,11 +349,9 @@ curl 'http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader= + [source,bash] ---- -curl 'http://localhost:8983/api/cluster/package' -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"2.0", - "files" :["/mypkg/2.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/packages/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"2.0","files":["/mypkg/2.0/myplugins.jar"]}' ---- . Verify the plugin to see if the correct version of the package is being used: