From a86cf713ad07048a8eb92548e946486901c01829 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 13:42:15 +0000 Subject: [PATCH 01/31] Initial plan From 9422c8c7dd9b9d443cef08fe3c7a49a3624eea39 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:08:56 +0000 Subject: [PATCH 02/31] Update TestPackages.java to use new JAX-RS PackageAPI endpoints - Replace PackagePayload.AddVersion with AddPackageVersionRequestBody - Move package name from body to URL: POST /cluster/package/{name}/versions - Replace delete command pattern with DELETE /cluster/package/{name}/versions/{version} - Replace refresh command with POST /cluster/package/{name}/refresh - Update errPath in testAPI from /details[0]/errorMessages[0] to /msg - Remove all add.pkg assignments (pkg now lives in URL path) - Replace add.pkg references in verifyComponent() with literal package name strings - Update import: remove PackagePayload, add AddPackageVersionRequestBody (sorted) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../solr/client/api/endpoint/PackageApis.java | 94 +++++ .../model/AddPackageVersionRequestBody.java | 41 ++ .../client/api/model/PackagesResponse.java | 64 ++++ .../org/apache/solr/core/CoreContainer.java | 4 +- .../java/org/apache/solr/pkg/PackageAPI.java | 253 +------------ .../org/apache/solr/pkg/PackageAPIJaxRs.java | 351 ++++++++++++++++++ .../org/apache/solr/pkg/TestPackages.java | 106 +++--- 7 files changed, 607 insertions(+), 306 deletions(-) create mode 100644 solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java new file mode 100644 index 000000000000..9aeb40119d76 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +/** V2 API definitions for managing Solr packages. */ +@Path("/cluster/package") +public interface PackageApis { + + @GET + @Operation( + summary = "List all packages registered in this Solr cluster.", + tags = {"package"}) + PackagesResponse listPackages( + @Parameter(description = "If provided, the named package is refreshed on this node.") + @QueryParam("refreshPackage") + String refreshPackage, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version.") + @QueryParam("expectedVersion") + Integer expectedVersion); + + @GET + @Path("/{packageName}") + @Operation( + summary = "Get information about a specific package in this Solr cluster.", + tags = {"package"}) + PackagesResponse getPackage( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName); + + @POST + @Path("/{packageName}/versions") + @Operation( + summary = "Add a version of a package to this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse addPackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @RequestBody(description = "Details of the package version to add.", required = true) + AddPackageVersionRequestBody requestBody); + + @DELETE + @Path("/{packageName}/versions/{version}") + @Operation( + summary = "Delete a specific version of a package from this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse deletePackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @Parameter(description = "The version of the package to delete.", required = true) + @PathParam("version") + String version); + + @POST + @Path("/{packageName}/refresh") + @Operation( + summary = "Refresh a package on all nodes in this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse refreshPackage( + @Parameter(description = "The name of the package to refresh.", required = true) + @PathParam("packageName") + String packageName); +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java new file mode 100644 index 000000000000..1ed868128627 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; + +/** Request body for adding a version of a package. */ +public class AddPackageVersionRequestBody { + + @JsonProperty("version") + @Schema(description = "The version string for this package version.", required = true) + public String version; + + @JsonProperty("files") + @Schema(description = "File paths from the file store to include in this version.", required = true) + public List files; + + @JsonProperty("manifest") + @Schema(description = "Optional path to a manifest file in the file store.") + public String manifest; + + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest file.") + public String manifestSHA512; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java new file mode 100644 index 000000000000..0dce425c807b --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; +import java.util.Map; + +/** Response for the package listing API. */ +public class PackagesResponse extends SolrJerseyResponse { + + @JsonProperty("result") + @Schema(description = "The package data including znode version and package definitions.") + public PackageData result; + + /** Package data returned by the package API. */ + public static class PackageData { + @JsonProperty("znodeVersion") + @Schema(description = "The ZooKeeper version of the packages.json node.") + public int znodeVersion; + + @JsonProperty("packages") + @Schema(description = "Map from package name to list of package versions.") + public Map> packages; + } + + /** Describes a single version of a package. */ + public static class PackageVersion { + @JsonProperty("package") + @Schema(description = "The package name.") + public String pkg; + + @JsonProperty("version") + @Schema(description = "The version string.") + public String version; + + @JsonProperty("files") + @Schema(description = "List of file paths from the file store included in this version.") + public List files; + + @JsonProperty("manifest") + @Schema(description = "Optional manifest reference.") + public String manifest; + + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest.") + public String manifestSHA512; + } +} diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index f6bf1dfb36ab..04d7450aa645 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -136,6 +136,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; +import org.apache.solr.pkg.PackageAPIJaxRs; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -841,8 +842,7 @@ private void loadInternal() { registerV2ApiIfEnabled(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI); - registerV2ApiIfEnabled(packageLoader.getPackageAPI().readAPI); + registerV2ApiIfEnabled(PackageAPIJaxRs.class); registerV2ApiIfEnabled(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 24ea22cd3874..f92414b5808e 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -18,8 +18,6 @@ package org.apache.solr.pkg; import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -31,27 +29,15 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import org.apache.solr.api.Command; -import org.apache.solr.api.EndPoint; -import org.apache.solr.api.PayloadObj; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.common.SolrException; import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.util.SolrJacksonAnnotationInspector; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; @@ -69,13 +55,10 @@ public class PackageAPI { "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; final CoreContainer coreContainer; - private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - private final SolrPackageLoader packageLoader; + final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + final SolrPackageLoader packageLoader; Packages pkgs; - public final Edit editAPI = new Edit(); - public final Read readAPI = new Read(); - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { this.coreContainer = coreContainer; this.packageLoader = loader; @@ -135,7 +118,7 @@ public void refreshPackages(Watcher watcher) { } } - private Packages readPkgsFromZk(byte[] data, Stat stat) + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { if (data == null || stat == null) { @@ -187,10 +170,11 @@ public static class PkgVersion implements ReflectMapWriter { public PkgVersion() {} - public PkgVersion(PackagePayload.AddVersion addVersion) { - this.pkg = addVersion.pkg; + public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { + this.pkg = packageName; this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.files = + addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); this.manifest = addVersion.manifest; this.manifestSHA512 = addVersion.manifestSHA512; } @@ -228,231 +212,10 @@ public PkgVersion copy() { } } - @EndPoint( - method = SolrRequest.METHOD.POST, - path = "/cluster/package", - permission = PACKAGE_EDIT_PERM) - public class Edit { - - @Command(name = "refresh") - public void refresh(PayloadObj payload) { - String p = payload.get(); - if (p == null) { - payload.addError("Package null"); - return; - } - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(p); - if (pkg == null) { - payload.addError("No such package: " + p); - return; - } - // first refresh my own - packageLoader.notifyListeners(p); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", p); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } - } - - @Command(name = "add") - public void add(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.AddVersion add = payload.get(); - if (add.files.isEmpty()) { - payload.addError("No files specified"); - return; - } - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); - if (payload.hasError()) return; - Packages[] finalState = new Packages[1]; - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = - bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new Packages(); - } - List list = - packages.packages.computeIfAbsent(add.pkg, o -> new ArrayList<>()); - for (PkgVersion version : list) { - if (Objects.equals(version.version, add.version)) { - payload.addError("Version '" + add.version + "' exists already"); - return null; - } - } - list.add(new PkgVersion(add)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - handleZkErr(e); - } - if (finalState[0] != null) { - // succeeded in updating - pkgs = finalState[0]; - notifyAllNodesToSync(pkgs.znodeVersion); - packageLoader.refreshPackageConf(); - } - } - - @Command(name = "delete") - public void del(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.DelVersion delVersion = payload.get(); - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new Packages(); - } - - List versions = packages.packages.get(delVersion.pkg); - if (versions == null || versions.isEmpty()) { - payload.addError("No such package: " + delVersion.pkg); - return null; // no change - } - int idxToremove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, delVersion.version)) { - idxToremove = i; - break; - } - } - if (idxToremove == -1) { - payload.addError("No such version: " + delVersion.version); - return null; - } - versions.remove(idxToremove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - public boolean isEnabled() { return enablePackages; } - private boolean checkEnabled(CommandOperation payload) { - if (!enablePackages) { - payload.addError(ERR_MSG); - return false; - } - return true; - } - - public class Read { - @EndPoint( - method = SolrRequest.METHOD.GET, - path = {"/cluster/package/", "/cluster/package/{name}"}, - permission = PACKAGE_READ_PERM) - public void get(SolrQueryRequest req, SolrQueryResponse rsp) { - String refresh = req.getParams().get("refreshPackage"); - if (refresh != null) { - packageLoader.notifyListeners(refresh); - return; - } - - int expectedVersion = req.getParams().getInt("expectedVersion", -1); - if (expectedVersion != -1) { - syncToVersion(expectedVersion); - } - String name = req.getPathTemplateValues().get("name"); - if (name == null) { - rsp.add("result", pkgs); - } else { - rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name))); - } - } - - private void syncToVersion(int expectedVersion) { - int origVersion = pkgs.znodeVersion; - for (int i = 0; i < 10; i++) { - log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion); - if (pkgs.znodeVersion >= expectedVersion) { - if (origVersion < pkgs.znodeVersion) { - packageLoader.refreshPackageConf(); - } - return; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - } - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - } - - void notifyAllNodesToSync(int expected) { - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expected)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " + liveNode + " to sync expected package version: " + expected, - e); - } - } - } - public void handleZkErr(Exception e) { log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java new file mode 100644 index 000000000000..2e2dd0a629bc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; + +import jakarta.inject.Inject; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.PackageApis; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.response.JavaBinResponseParser; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JAX-RS implementation of the package management API ({@code /api/cluster/package}). + * + * @see PackageApis + */ +public class PackageAPIJaxRs extends JerseyResource implements PackageApis { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final CoreContainer coreContainer; + private final SolrQueryRequest solrQueryRequest; + private final SolrQueryResponse solrQueryResponse; + + @Inject + public PackageAPIJaxRs( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + this.coreContainer = coreContainer; + this.solrQueryRequest = solrQueryRequest; + this.solrQueryResponse = solrQueryResponse; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + if (refreshPackage != null) { + packageAPI.packageLoader.notifyListeners(refreshPackage); + return instantiateJerseyResponse(PackagesResponse.class); + } + + if (expectedVersion != null) { + syncToVersion(packageAPI, expectedVersion); + } + + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageAPI.pkgs); + return response; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse getPackage(String packageName) { + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageAPI.pkgs); + // Filter to only the requested package + if (response.result != null && response.result.packages != null) { + final var pkgVersions = response.result.packages.get(packageName); + response.result.packages = Collections.singletonMap(packageName, pkgVersions); + } + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse addPackageVersion( + String packageName, AddPackageVersionRequestBody requestBody) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + if (!packageAPI.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); + } + if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); + } + + final List errors = new ArrayList<>(); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), requestBody.files, true, errors::add); + if (!errors.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + } + + final PackageAPI.Packages[] finalState = new PackageAPI.Packages[1]; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageAPI.Packages packages; + try { + packages = + bytes == null + ? new PackageAPI.Packages() + : packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + packages = new PackageAPI.Packages(); + } + List list = + packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); + for (PackageAPI.PkgVersion pkgVersion : list) { + if (Objects.equals(pkgVersion.version, requestBody.version)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Version '" + requestBody.version + "' exists already"); + } + } + list.add(new PackageAPI.PkgVersion(packageName, requestBody)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + packageAPI.handleZkErr(e); + } + + if (finalState[0] != null) { + packageAPI.pkgs = finalState[0]; + notifyAllNodesToSync(packageAPI.pkgs.znodeVersion); + coreContainer.getPackageLoader().refreshPackageConf(); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse deletePackageVersion(String packageName, String version) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + if (!packageAPI.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); + } + + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageAPI.Packages packages; + try { + packages = packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + packages = new PackageAPI.Packages(); + } + + List versions = packages.packages.get(packageName); + if (versions == null || versions.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + int idxToRemove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, version)) { + idxToRemove = i; + break; + } + } + if (idxToRemove == -1) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); + } + versions.remove(idxToRemove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + packageAPI.handleZkErr(e); + } + + return response; + } + + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse refreshPackage(String packageName) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); + + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); + if (pkg == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + // first refresh on the current node + packageAPI.packageLoader.notifyListeners(packageName); + + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("refreshPackage", packageName); + + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to refresh package on node: " + liveNode, + e); + } + } + + return response; + } + + private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { + int origVersion = packageAPI.pkgs.znodeVersion; + for (int i = 0; i < 10; i++) { + if (log.isDebugEnabled()) { + log.debug( + "my version is {} , and expected version {}", + packageAPI.pkgs.znodeVersion, + expectedVersion); + } + if (packageAPI.pkgs.znodeVersion >= expectedVersion) { + if (origVersion < packageAPI.pkgs.znodeVersion) { + coreContainer.getPackageLoader().refreshPackageConf(); + } + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + packageAPI.pkgs = packageAPI.readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + packageAPI.handleZkErr(e); + } + } + } + + private void notifyAllNodesToSync(int expectedVersion) { + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("expectedVersion", String.valueOf(expectedVersion)); + + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion, + e); + } + } + } + + private static PackagesResponse.PackageData toPackageData(PackageAPI.Packages packages) { + if (packages == null) { + return null; + } + final var data = new PackagesResponse.PackageData(); + data.znodeVersion = packages.znodeVersion; + data.packages = + packages.packages.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue().stream() + .map(PackageAPIJaxRs::toPkgVersionResponse) + .collect(Collectors.toList()))); + return data; + } + + private static PackagesResponse.PackageVersion toPkgVersionResponse( + PackageAPI.PkgVersion pkgVersion) { + final var v = new PackagesResponse.PackageVersion(); + v.pkg = pkgVersion.pkg; + v.version = pkgVersion.version; + v.files = pkgVersion.files; + v.manifest = pkgVersion.manifest; + v.manifestSHA512 = pkgVersion.manifestSHA512; + return v; + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 96d5f86b28a9..0804bbee9a1d 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -42,6 +42,7 @@ import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; import org.apache.lucene.util.ResourceLoaderAware; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -54,7 +55,6 @@ import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.MiniSolrCloudCluster; @@ -130,15 +130,14 @@ public void testCoreReloadingPlugin() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "mypkg"; add.files = Arrays.asList(new String[] {FILE1}); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); req.process(cluster.getSolrClient()); @@ -160,7 +159,7 @@ public void testCoreReloadingPlugin() throws Exception { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); verifyComponent( - cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version); + cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", add.version); add.version = "2.0"; req.process(cluster.getSolrClient()); @@ -211,15 +210,14 @@ public void testPluginLoading() throws Exception { EXPR1, "ZOT11arAiPmPZYOHzqodiNnxO9pRyRozWZEBX8XGjU1/HJptFnZK+DI7eXnUtbNaMcbXE2Ze8hh4M/eGyhY8BQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "mypkg"; add.files = Arrays.asList(new String[] {FILE1, URP1, EXPR1}); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); req.process(cluster.getSolrClient()); @@ -402,16 +400,11 @@ public void testPluginLoading() throws Exception { assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.pkg = "mypkg"; - delVersion.version = "1.0"; - V2Request delete = - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.POST) - .forceV2(true) - .withPayload(Collections.singletonMap("delete", delVersion)) - .build(); - delete.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/mypkg/versions/1.0") + .withMethod(SolrRequest.METHOD.DELETE) + .forceV2(true) + .build() + .process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -422,9 +415,12 @@ public void testPluginLoading() throws Exception { verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1"); - // now remove the hughest version. So, it will roll back to the next highest one - delVersion.version = "2.1"; - delete.process(cluster.getSolrClient()); + // now remove the highest version. So, it will roll back to the next highest one + new V2Request.Builder("/cluster/package/mypkg/versions/2.1") + .withMethod(SolrRequest.METHOD.DELETE) + .forceV2(true) + .build() + .process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "1.1"); @@ -473,9 +469,8 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { // now, let's force every collection using 'mypkg' to refresh // so that it uses version 2.1 - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/refresh") .withMethod(SolrRequest.METHOD.POST) - .withPayload("{refresh : mypkg}") .forceV2(true) .build() .process(cluster.getSolrClient()); @@ -578,20 +573,19 @@ private void verifyComponent( @Test @SuppressWarnings("unchecked") public void testAPI() throws Exception { - String errPath = "/details[0]/errorMessages[0]"; + String errPath = "/msg"; String FILE1 = "/mypkg/v.0.12/jar_a.jar"; String FILE2 = "/mypkg/v.0.12/jar_b.jar"; String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "0.12"; - add.pkg = "test_pkg"; add.files = List.of(FILE1, FILE2); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/test_pkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); // the files are not yet there. The command should fail with error saying "No such file" @@ -640,7 +634,6 @@ public void testAPI() throws Exception { // this time we are adding the second version of the package (0.13) add.version = "0.13"; - add.pkg = "test_pkg"; add.files = Collections.singletonList(FILE3); // this request should succeed @@ -654,21 +647,20 @@ public void testAPI() throws Exception { Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); // Now we will just delete one version - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.version = "0.1"; // this version does not exist - delVersion.pkg = "test_pkg"; - req = - new V2Request.Builder("/cluster/package") + V2Request deleteReq = + new V2Request.Builder("/cluster/package/test_pkg/versions/0.1") .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("delete", delVersion)) + .withMethod(SolrRequest.METHOD.DELETE) .build(); // we are expecting an error - expectError(req, cluster.getSolrClient(), errPath, "No such version:"); + expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - delVersion.version = "0.12"; // correct version. Should succeed - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed + .forceV2(true) + .withMethod(SolrRequest.METHOD.DELETE) + .build() + .process(cluster.getSolrClient()); // Verify with ZK that the data is correct TestDistribFileStore.assertResponseValues( 1, @@ -762,17 +754,15 @@ public void testSchemaPlugins() throws Exception { "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); // upload package v1.0 - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "schemapkg"; add.files = Arrays.asList(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) - .build(); - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/schemapkg/versions") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(add) + .build() + .process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -804,17 +794,15 @@ public void testSchemaPlugins() throws Exception { coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); // upload package v2.0 - add = new PackagePayload.AddVersion(); + add = new AddPackageVersionRequestBody(); add.version = "2.0"; - add.pkg = "schemapkg"; add.files = Arrays.asList(FILE1, FILE2); - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) - .build(); - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/schemapkg/versions") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(add) + .build() + .process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, From 744e8ff8a5a23e66277f7b4f07c5439176f6d2db Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:14:51 +0000 Subject: [PATCH 03/31] Migrate PackageAPI from @EndPoint/@Command to JAX-RS annotations Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../model/AddPackageVersionRequestBody.java | 4 +- .../java/org/apache/solr/pkg/PackageAPI.java | 6 +- .../org/apache/solr/pkg/PackageAPIJaxRs.java | 6 +- .../apache/solr/pkg/PackageAPIJaxRsTest.java | 164 ++++++++++++++++++ .../org/apache/solr/pkg/TestPackages.java | 3 +- .../pages/package-manager-internals.adoc | 29 ++-- 6 files changed, 186 insertions(+), 26 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java diff --git a/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java index 1ed868128627..3b077c110d3b 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java @@ -28,7 +28,9 @@ public class AddPackageVersionRequestBody { public String version; @JsonProperty("files") - @Schema(description = "File paths from the file store to include in this version.", required = true) + @Schema( + description = "File paths from the file store to include in this version.", + required = true) public List files; @JsonProperty("manifest") diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index f92414b5808e..d2394aefe917 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -118,8 +118,7 @@ public void refreshPackages(Watcher watcher) { } } - Packages readPkgsFromZk(byte[] data, Stat stat) - throws KeeperException, InterruptedException { + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { if (data == null || stat == null) { stat = new Stat(); @@ -173,8 +172,7 @@ public PkgVersion() {} public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { this.pkg = packageName; this.version = addVersion.version; - this.files = - addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); this.manifest = addVersion.manifest; this.manifestSHA512 = addVersion.manifestSHA512; } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java index 2e2dd0a629bc..9d72b1cff543 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java @@ -123,8 +123,7 @@ public SolrJerseyResponse addPackageVersion( FileStoreUtils.validateFiles( coreContainer.getFileStore(), requestBody.files, true, errors::add); if (!errors.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); } final PackageAPI.Packages[] finalState = new PackageAPI.Packages[1]; @@ -303,8 +302,7 @@ private void notifyAllNodesToSync(int expectedVersion) { request.setResponseParser(new JavaBinResponseParser()); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { var solrClient = coreContainer.getDefaultHttpSolrClient(); solrClient.requestWithBaseUrl(baseUrl, request::process); diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java new file mode 100644 index 000000000000..ecfaf7b965b2 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.net.URL; +import java.util.List; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.solr.client.solrj.apache.HttpClientUtil; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.util.Utils; +import org.apache.solr.filestore.ClusterFileStore; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests for the JAX-RS-based {@link PackageAPIJaxRs}. + * + *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its + * cluster-level operations. A one-node SolrCloud cluster is used instead. + */ +public class PackageAPIJaxRsTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("solr.packages.enabled", "true"); + configureCluster(1) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + + @Test + public void testListPackagesReturnsEmptyResult() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String packageUrl = + cluster.getJettySolrRunner(0).getBaseURLV2().toString() + "/cluster/package"; + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + Object response = + HttpClientUtil.executeGET(client.getHttpClient(), packageUrl, Utils.JSONCONSUMER); + assertNotNull("Expected non-null response from GET /cluster/package", response); + // The response should have a 'result' field with 'packages' and 'znodeVersion' + Object result = Utils.getObjectByPath(response, true, "result"); + assertNotNull("Expected 'result' field in GET /cluster/package response", result); + } + } + + @Test + public void testAddDeletePackageVersion() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); + String FILE1 = "/testpkg/runtimelibs.jar"; + + // Upload a key and a signed jar file to the filestore + byte[] derFile = + org.apache.solr.filestore.TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + org.apache.solr.pkg.TestPackages.postFileAndWait( + cluster, + "runtimecode/runtimelibs.jar.bin", + FILE1, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Add a package version via POST /cluster/package/{name}/versions + String addUrl = baseUrlV2 + "/cluster/package/testpkg/versions"; + HttpPost httpPost = new HttpPost(addUrl); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity(new StringEntity("{\"version\":\"1.0\",\"files\":[\"" + FILE1 + "\"]}")); + Object addResponse = + HttpClientUtil.executeHttpMethod( + client.getHttpClient(), addUrl, Utils.JSONCONSUMER, httpPost); + assertNotNull( + "Expected non-null response from POST /cluster/package/testpkg/versions", addResponse); + + // Verify the package was added via GET /cluster/package + String listUrl = baseUrlV2 + "/cluster/package"; + Object listResponse = + HttpClientUtil.executeGET(client.getHttpClient(), listUrl, Utils.JSONCONSUMER); + assertNotNull(listResponse); + + // Verify the package appears in the list + @SuppressWarnings("unchecked") + List versions = + (List) Utils.getObjectByPath(listResponse, true, "result/packages/testpkg"); + assertNotNull("Expected testpkg to be present in packages", versions); + assertFalse("Expected at least one version", versions.isEmpty()); + + // Verify GET /cluster/package/{name} returns only this package + String getByNameUrl = baseUrlV2 + "/cluster/package/testpkg"; + Object getByNameResponse = + HttpClientUtil.executeGET(client.getHttpClient(), getByNameUrl, Utils.JSONCONSUMER); + assertNotNull(getByNameResponse); + @SuppressWarnings("unchecked") + List versionsFromGet = + (List) Utils.getObjectByPath(getByNameResponse, true, "result/packages/testpkg"); + assertNotNull("Expected testpkg in GET by name response", versionsFromGet); + + // Delete the package version via DELETE /cluster/package/{name}/versions/{version} + String deleteUrl = baseUrlV2 + "/cluster/package/testpkg/versions/1.0"; + HttpDelete httpDelete = new HttpDelete(deleteUrl); + Object deleteResponse = + HttpClientUtil.executeHttpMethod( + client.getHttpClient(), deleteUrl, Utils.JSONCONSUMER, httpDelete); + assertNotNull("Expected non-null response from DELETE", deleteResponse); + } + } + + @Test + public void testAddPackageVersionValidatesFiles() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Try to add a package version with a non-existent file + String addUrl = baseUrlV2 + "/cluster/package/testpkg2/versions"; + HttpPost httpPost = new HttpPost(addUrl); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setEntity( + new StringEntity("{\"version\":\"1.0\",\"files\":[\"/nonexistent/file.jar\"]}")); + + org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); + int statusCode = httpResponse.getStatusLine().getStatusCode(); + assertEquals("Expected 400 BAD_REQUEST when specifying non-existent file", 400, statusCode); + } + } + + @Test + public void testRefreshNonExistentPackage() throws Exception { + URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); + + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Try to refresh a non-existent package + String refreshUrl = baseUrlV2 + "/cluster/package/nonexistentpkg/refresh"; + HttpPost httpPost = new HttpPost(refreshUrl); + httpPost.setHeader("Content-Type", "application/json"); + + org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); + int statusCode = httpResponse.getStatusLine().getStatusCode(); + assertEquals( + "Expected 400 BAD_REQUEST when refreshing non-existent package", 400, statusCode); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 0804bbee9a1d..685c133c3e56 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -656,7 +656,8 @@ public void testAPI() throws Exception { // we are expecting an error expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - new V2Request.Builder("/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed + new V2Request.Builder( + "/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed .forceV2(true) .withMethod(SolrRequest.METHOD.DELETE) .build() diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc index 323aa9a3bfe3..4814ebb929a4 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc @@ -178,18 +178,19 @@ For example: == API Endpoints -* `GET /api/cluster/package` Get the list of packages -* `POST /api/cluster/package` edit packages -** `add` command: add a version of a package -** `delete` command: delete a version of a package +* `GET /api/cluster/package` Get the list of all packages +* `GET /api/cluster/package/{name}` Get the details of a specific package +* `POST /api/cluster/package/{name}/versions` Add a version of a package +* `DELETE /api/cluster/package/{name}/versions/{version}` Delete a specific version of a package +* `POST /api/cluster/package/{name}/refresh` Refresh a package on all nodes in the cluster === How to Upgrade? -Use the `add` command to add a version that is higher than the current version. +Use the add version endpoint to add a version that is higher than the current version. === How to Downgrade? -Use the `delete` command to delete the highest version and choose the next highest version. +Use the delete version endpoint to delete the highest version and choose the next highest version. === Using Multiple Versions in Parallel @@ -244,11 +245,9 @@ The plugins loaded from packages cannot depend on core level classes. + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"1.0", - "files" :["/mypkg/1.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"1.0","files":["/mypkg/1.0/myplugins.jar"]}' ---- . Verify the created package: @@ -350,11 +349,9 @@ curl http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=t + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"2.0", - "files" :["/mypkg/2.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"2.0","files":["/mypkg/2.0/myplugins.jar"]}' ---- . Verify the plugin to see if the correct version of the package is being used: From cc2641206f6795e9ffc33a4c152eb928609a5b8f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:32:38 +0000 Subject: [PATCH 04/31] Fix PackageAPIJaxRsTest to use generated request classes and check response.error field Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../apache/solr/pkg/PackageAPIJaxRsTest.java | 152 ++++++++---------- 1 file changed, 68 insertions(+), 84 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java index ecfaf7b965b2..8e3c1f4b72ca 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java @@ -18,15 +18,11 @@ import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; -import java.net.URL; import java.util.List; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.solr.client.solrj.apache.HttpClientUtil; +import org.apache.solr.client.api.model.PackagesResponse; import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.junit.BeforeClass; import org.junit.Test; @@ -49,116 +45,104 @@ public static void setupCluster() throws Exception { } @Test - public void testListPackagesReturnsEmptyResult() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String packageUrl = - cluster.getJettySolrRunner(0).getBaseURLV2().toString() + "/cluster/package"; - - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { - Object response = - HttpClientUtil.executeGET(client.getHttpClient(), packageUrl, Utils.JSONCONSUMER); + public void testListPackagesReturnsResult() throws Exception { + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + PackagesResponse response = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null response from GET /cluster/package", response); - // The response should have a 'result' field with 'packages' and 'znodeVersion' - Object result = Utils.getObjectByPath(response, true, "result"); - assertNotNull("Expected 'result' field in GET /cluster/package response", result); + assertNotNull("Expected 'result' field in GET /cluster/package response", response.result); } } @Test - public void testAddDeletePackageVersion() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); - String FILE1 = "/testpkg/runtimelibs.jar"; + public void testAddAndDeletePackageVersion() throws Exception { + String FILE1 = "/jaxrstestpkg/runtimelibs.jar"; // Upload a key and a signed jar file to the filestore byte[] derFile = org.apache.solr.filestore.TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - org.apache.solr.pkg.TestPackages.postFileAndWait( + TestPackages.postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { // Add a package version via POST /cluster/package/{name}/versions - String addUrl = baseUrlV2 + "/cluster/package/testpkg/versions"; - HttpPost httpPost = new HttpPost(addUrl); - httpPost.setHeader("Content-Type", "application/json"); - httpPost.setEntity(new StringEntity("{\"version\":\"1.0\",\"files\":[\"" + FILE1 + "\"]}")); - Object addResponse = - HttpClientUtil.executeHttpMethod( - client.getHttpClient(), addUrl, Utils.JSONCONSUMER, httpPost); - assertNotNull( - "Expected non-null response from POST /cluster/package/testpkg/versions", addResponse); + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("jaxrstestpkg"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of(FILE1)); + addRequest.process(client); // Verify the package was added via GET /cluster/package - String listUrl = baseUrlV2 + "/cluster/package"; - Object listResponse = - HttpClientUtil.executeGET(client.getHttpClient(), listUrl, Utils.JSONCONSUMER); - assertNotNull(listResponse); - - // Verify the package appears in the list - @SuppressWarnings("unchecked") - List versions = - (List) Utils.getObjectByPath(listResponse, true, "result/packages/testpkg"); - assertNotNull("Expected testpkg to be present in packages", versions); - assertFalse("Expected at least one version", versions.isEmpty()); + PackagesResponse listResponse = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response", listResponse); + assertNotNull("Expected non-null result", listResponse.result); + assertNotNull( + "Expected jaxrstestpkg in packages", listResponse.result.packages.get("jaxrstestpkg")); + assertFalse( + "Expected at least one version", + listResponse.result.packages.get("jaxrstestpkg").isEmpty()); // Verify GET /cluster/package/{name} returns only this package - String getByNameUrl = baseUrlV2 + "/cluster/package/testpkg"; - Object getByNameResponse = - HttpClientUtil.executeGET(client.getHttpClient(), getByNameUrl, Utils.JSONCONSUMER); - assertNotNull(getByNameResponse); - @SuppressWarnings("unchecked") - List versionsFromGet = - (List) Utils.getObjectByPath(getByNameResponse, true, "result/packages/testpkg"); - assertNotNull("Expected testpkg in GET by name response", versionsFromGet); + PackagesResponse getByNameResponse = + new PackageApi.GetPackage("jaxrstestpkg").process(client); + assertNotNull("Expected non-null get-by-name response", getByNameResponse); + assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); + assertNotNull( + "Expected jaxrstestpkg in get-by-name response", + getByNameResponse.result.packages.get("jaxrstestpkg")); // Delete the package version via DELETE /cluster/package/{name}/versions/{version} - String deleteUrl = baseUrlV2 + "/cluster/package/testpkg/versions/1.0"; - HttpDelete httpDelete = new HttpDelete(deleteUrl); - Object deleteResponse = - HttpClientUtil.executeHttpMethod( - client.getHttpClient(), deleteUrl, Utils.JSONCONSUMER, httpDelete); - assertNotNull("Expected non-null response from DELETE", deleteResponse); + new PackageApi.DeletePackageVersion("jaxrstestpkg", "1.0").process(client); + + // Verify it's deleted + PackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response after delete", listAfterDelete); + assertNotNull("Expected non-null result after delete", listAfterDelete.result); + // After deleting the only version, the package entry should be empty or absent + List versionsAfterDelete = listAfterDelete.result.packages.get("jaxrstestpkg"); + assertTrue( + "Expected no versions after delete", + versionsAfterDelete == null || versionsAfterDelete.isEmpty()); } } @Test public void testAddPackageVersionValidatesFiles() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); - - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { - // Try to add a package version with a non-existent file - String addUrl = baseUrlV2 + "/cluster/package/testpkg2/versions"; - HttpPost httpPost = new HttpPost(addUrl); - httpPost.setHeader("Content-Type", "application/json"); - httpPost.setEntity( - new StringEntity("{\"version\":\"1.0\",\"files\":[\"/nonexistent/file.jar\"]}")); - - org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); - int statusCode = httpResponse.getStatusLine().getStatusCode(); - assertEquals("Expected 400 BAD_REQUEST when specifying non-existent file", 400, statusCode); + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + // Try to add a package version with a non-existent file. + // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; + // instead, the error is in the response body's 'error' field. + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("testpkg_invalid"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of("/nonexistent/file.jar")); + + var response = addRequest.process(client); + assertNotNull("Expected error in response for non-existent file", response.error); + assertEquals("Expected 400 for non-existent file", 400, (int) response.error.code); + assertTrue( + "Expected error message to mention the file", + response.error.msg.contains("No such file")); } } @Test public void testRefreshNonExistentPackage() throws Exception { - URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); - String baseUrlV2 = cluster.getJettySolrRunner(0).getBaseURLV2().toString(); - - try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { - // Try to refresh a non-existent package - String refreshUrl = baseUrlV2 + "/cluster/package/nonexistentpkg/refresh"; - HttpPost httpPost = new HttpPost(refreshUrl); - httpPost.setHeader("Content-Type", "application/json"); - - org.apache.http.HttpResponse httpResponse = client.getHttpClient().execute(httpPost); - int statusCode = httpResponse.getStatusLine().getStatusCode(); - assertEquals( - "Expected 400 BAD_REQUEST when refreshing non-existent package", 400, statusCode); + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + // Try to refresh a non-existent package. + // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; + // instead, the error is in the response body's 'error' field. + var response = new PackageApi.RefreshPackage("nonexistentpkg_jaxrs").process(client); + assertNotNull("Expected error in response for non-existent package", response.error); + assertEquals("Expected 400 for non-existent package", 400, (int) response.error.code); + assertTrue( + "Expected error message to mention the package", + response.error.msg.contains("No such package")); } } } From 5040f5b47b0b1c20e27ad422d56178d8b704ec31 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 14:33:45 +0000 Subject: [PATCH 05/31] Extract magic numbers as named constants in PackageAPIJaxRs Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java index 9d72b1cff543..92147673c5e6 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java @@ -58,6 +58,9 @@ public class PackageAPIJaxRs extends JerseyResource implements PackageApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final int SYNC_MAX_RETRIES = 10; + private static final long SYNC_SLEEP_MS = 10L; + private final CoreContainer coreContainer; private final SolrQueryRequest solrQueryRequest; private final SolrQueryResponse solrQueryResponse; @@ -266,7 +269,7 @@ public SolrJerseyResponse refreshPackage(String packageName) { private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { int origVersion = packageAPI.pkgs.znodeVersion; - for (int i = 0; i < 10; i++) { + for (int i = 0; i < SYNC_MAX_RETRIES; i++) { if (log.isDebugEnabled()) { log.debug( "my version is {} , and expected version {}", @@ -280,7 +283,7 @@ private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { return; } try { - Thread.sleep(10); + Thread.sleep(SYNC_SLEEP_MS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } From 83c2f5c36c7ff96db4bbebd7386af5ecd2b9b5c9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 16:13:55 +0000 Subject: [PATCH 06/31] Rename PackageAPIJaxRs to PackageAPI and old PackageAPI to PackageStore Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../solr/api/ContainerPluginsRegistry.java | 2 +- .../org/apache/solr/core/CoreContainer.java | 4 +- .../solr/filestore/ClusterFileStore.java | 12 +- .../solr/handler/SolrConfigHandler.java | 4 +- .../solr/handler/component/SearchHandler.java | 4 +- .../packagemanager/RepositoryManager.java | 1 - .../java/org/apache/solr/pkg/PackageAPI.java | 431 +++++++++++------- .../org/apache/solr/pkg/PackageAPIJaxRs.java | 352 -------------- .../org/apache/solr/pkg/PackageListeners.java | 2 +- .../solr/pkg/PackageListeningClassLoader.java | 12 +- .../apache/solr/pkg/PackagePluginHolder.java | 2 +- .../org/apache/solr/pkg/PackageStore.java | 241 ++++++++++ .../apache/solr/pkg/SolrPackageLoader.java | 40 +- .../solr/handler/TestContainerPlugin.java | 4 +- ...eAPIJaxRsTest.java => PackageAPITest.java} | 24 +- 15 files changed, 568 insertions(+), 567 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageStore.java rename solr/core/src/test/org/apache/solr/pkg/{PackageAPIJaxRsTest.java => PackageAPITest.java} (89%) diff --git a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java index 20cd2440bbe3..f58cb95fbcb5 100644 --- a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java +++ b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java @@ -335,7 +335,7 @@ public ApiInfo(PluginMetaHolder infoHolder, List errs) { coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); if (ver.isEmpty()) { // may be we are a bit early. Do a refresh and try again - coreContainer.getPackageLoader().getPackageAPI().refreshPackages(null); + coreContainer.getPackageLoader().getPackageStore().refreshPackages(null); ver = coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); } if (ver.isEmpty()) { diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 04d7450aa645..66daf22de020 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -136,7 +136,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; -import org.apache.solr.pkg.PackageAPIJaxRs; +import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -842,7 +842,7 @@ private void loadInternal() { registerV2ApiIfEnabled(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2ApiIfEnabled(PackageAPIJaxRs.class); + registerV2ApiIfEnabled(PackageAPI.class); registerV2ApiIfEnabled(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java index c3b76dca4bba..bdda47d55f7d 100644 --- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java @@ -47,7 +47,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -86,8 +86,8 @@ public ClusterFileStore( public UploadToFileStoreResponse uploadFile( String filePath, List sig, InputStream requestBody) { final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } try { coreContainer @@ -302,12 +302,12 @@ private void doDelete(String filePath, Boolean localDelete) { @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) public SolrJerseyResponse deleteFile(String filePath, Boolean localDelete) { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } validateName(filePath, true); - if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) { + if (coreContainer.getPackageLoader().getPackageStore().isJarInuse(filePath)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); } doDelete(filePath, localDelete); diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index cba47bad98d2..e7a7f9f98f4d 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -88,8 +88,8 @@ import org.apache.solr.handler.admin.api.GetConfigAPI; import org.apache.solr.handler.admin.api.ModifyConfigComponentAPI; import org.apache.solr.handler.admin.api.ModifyParamSetAPI; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; @@ -303,7 +303,7 @@ private void handleGET() { List listeners = req.getCore().getPackageListeners().getListeners(); for (PackageListeners.Listener listener : listeners) { - Map infos = listener.packageDetails(); + Map infos = listener.packageDetails(); if (infos == null || infos.isEmpty()) continue; infos.forEach( (s, mapWriter) -> { diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index d3bb6216b8ae..54b434eca793 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -62,8 +62,8 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -202,7 +202,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Collections.emptyMap(); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 013d3639e55b..5ae49bd02a39 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -58,7 +58,6 @@ import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index d2394aefe917..0e0318bd0505 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -14,226 +14,339 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.solr.pkg; import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; -import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.inject.Inject; import java.io.IOException; -import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.PackageApis; import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.response.JavaBinResponseParser; import org.apache.solr.common.SolrException; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.util.SolrJacksonAnnotationInspector; +import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** This implements the public end points (/api/cluster/package) of package API. */ -public class PackageAPI { - public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); +/** + * JAX-RS implementation of the package management API ({@code /api/cluster/package}). + * + * @see PackageApis + */ +public class PackageAPI extends JerseyResource implements PackageApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + private static final int SYNC_MAX_RETRIES = 10; + private static final long SYNC_SLEEP_MS = 10L; - final CoreContainer coreContainer; - final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - final SolrPackageLoader packageLoader; - Packages pkgs; + private final CoreContainer coreContainer; + private final SolrQueryRequest solrQueryRequest; + private final SolrQueryResponse solrQueryResponse; - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { + @Inject + public PackageAPI( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { this.coreContainer = coreContainer; - this.packageLoader = loader; - pkgs = new Packages(); - SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - pkgs = new Packages(); - // ignore + this.solrQueryRequest = solrQueryRequest; + this.solrQueryResponse = solrQueryResponse; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (refreshPackage != null) { + packageStore.packageLoader.notifyListeners(refreshPackage); + return instantiateJerseyResponse(PackagesResponse.class); } - try { - registerListener(zkClient); - } catch (KeeperException | InterruptedException e) { - SolrZkClient.checkInterrupted(e); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); } - } - private void registerListener(SolrZkClient zkClient) - throws KeeperException, InterruptedException { - zkClient.exists( - SOLR_PKGS_PATH, - new Watcher() { - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher - if (Event.EventType.None.equals(event.getType())) { - return; - } - synchronized (this) { - log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); - // remake watch - final Watcher thisWatch = this; - refreshPackages(thisWatch); - } - } - }); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageStore.pkgs); + return response; } - public void refreshPackages(Watcher watcher) { - final Stat stat = new Stat(); - try { - final byte[] data = - coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); - pkgs = readPkgsFromZk(data, stat); - packageLoader.refreshPackageConf(); - } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); - } catch (KeeperException e) { - log.error("A ZK error has occurred", e); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse getPackage(String packageName) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageStore.pkgs); + // Filter to only the requested package + if (response.result != null && response.result.packages != null) { + final var pkgVersions = response.result.packages.get(packageName); + response.result.packages = Collections.singletonMap(packageName, pkgVersions); } + return response; } - Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse addPackageVersion( + String packageName, AddPackageVersionRequestBody requestBody) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - if (data == null || stat == null) { - stat = new Stat(); - data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); } - Packages packages = null; - if (data == null || data.length == 0) { - packages = new Packages(); - } else { - try { - packages = mapper.readValue(data, Packages.class); - packages.znodeVersion = stat.getVersion(); - } catch (IOException e) { - // invalid data in packages - // TODO handle properly; - return new Packages(); - } + if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); } - return packages; - } - public static class Packages implements ReflectMapWriter { - @JsonProperty public int znodeVersion = -1; + final List errors = new ArrayList<>(); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), requestBody.files, true, errors::add); + if (!errors.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + } - @JsonProperty public Map> packages = new LinkedHashMap<>(); + final PackageStore.Packages[] finalState = new PackageStore.Packages[1]; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = + bytes == null + ? new PackageStore.Packages() + : packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + packages = new PackageStore.Packages(); + } + List list = + packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); + for (PackageStore.PkgVersion pkgVersion : list) { + if (Objects.equals(pkgVersion.version, requestBody.version)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Version '" + requestBody.version + "' exists already"); + } + } + list.add(new PackageStore.PkgVersion(packageName, requestBody)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + packageStore.handleZkErr(e); + } - public Packages copy() { - Packages p = new Packages(); - p.znodeVersion = this.znodeVersion; - p.packages = new LinkedHashMap<>(); - packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); - return p; + if (finalState[0] != null) { + packageStore.pkgs = finalState[0]; + notifyAllNodesToSync(packageStore.pkgs.znodeVersion); + coreContainer.getPackageLoader().refreshPackageConf(); } - } - public static class PkgVersion implements ReflectMapWriter { + return response; + } - @JsonProperty("package") - public String pkg; + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse deletePackageVersion(String packageName, String version) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - @JsonProperty public String version; + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); + } - @JsonProperty public List files; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + packages = new PackageStore.Packages(); + } - @JsonProperty public String manifest; + List versions = packages.packages.get(packageName); + if (versions == null || versions.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + int idxToRemove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, version)) { + idxToRemove = i; + break; + } + } + if (idxToRemove == -1) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); + } + versions.remove(idxToRemove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } - @JsonProperty public String manifestSHA512; + return response; + } - public PkgVersion() {} + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse refreshPackage(String packageName) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { - this.pkg = packageName; - this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); - this.manifest = addVersion.manifest; - this.manifestSHA512 = addVersion.manifestSHA512; + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); + if (pkg == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); } + // first refresh on the current node + packageStore.packageLoader.notifyListeners(packageName); - @Override - public boolean equals(Object obj) { - if (obj instanceof PkgVersion that) { - return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); - } - return false; - } + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("refreshPackage", packageName); - @Override - public int hashCode() { - return Objects.hash(version); - } + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); - @Override - public String toString() { + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { - return Utils.writeJson(this, new StringWriter(), false).toString(); - } catch (IOException e) { - throw new RuntimeException(e); + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to refresh package on node: " + liveNode, + e); } } - public PkgVersion copy() { - PkgVersion result = new PkgVersion(); - result.pkg = this.pkg; - result.version = this.version; - result.files = this.files; - result.manifest = this.manifest; - result.manifestSHA512 = this.manifestSHA512; - return result; - } + return response; } - public boolean isEnabled() { - return enablePackages; + private void syncToVersion(PackageStore packageStore, int expectedVersion) { + int origVersion = packageStore.pkgs.znodeVersion; + for (int i = 0; i < SYNC_MAX_RETRIES; i++) { + if (log.isDebugEnabled()) { + log.debug( + "my version is {} , and expected version {}", + packageStore.pkgs.znodeVersion, + expectedVersion); + } + if (packageStore.pkgs.znodeVersion >= expectedVersion) { + if (origVersion < packageStore.pkgs.znodeVersion) { + coreContainer.getPackageLoader().refreshPackageConf(); + } + return; + } + try { + Thread.sleep(SYNC_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + try { + packageStore.pkgs = packageStore.readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } + } } - public void handleZkErr(Exception e) { - log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); - } + private void notifyAllNodesToSync(int expectedVersion) { + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("expectedVersion", String.valueOf(expectedVersion)); - public boolean isJarInuse(String path) { - Packages pkg = null; - try { - pkg = readPkgsFromZk(null, null); - } catch (KeeperException.NoNodeException nne) { - return false; - } catch (InterruptedException | KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - for (List vers : pkg.packages.values()) { - for (PkgVersion ver : vers) { - if (ver.files.contains(path)) { - return true; - } + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); + + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + try { + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion, + e); } } - return false; + } + + private static PackagesResponse.PackageData toPackageData(PackageStore.Packages packages) { + if (packages == null) { + return null; + } + final var data = new PackagesResponse.PackageData(); + data.znodeVersion = packages.znodeVersion; + data.packages = + packages.packages.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue().stream() + .map(PackageAPI::toPkgVersionResponse) + .collect(Collectors.toList()))); + return data; + } + + private static PackagesResponse.PackageVersion toPkgVersionResponse( + PackageStore.PkgVersion pkgVersion) { + final var v = new PackagesResponse.PackageVersion(); + v.pkg = pkgVersion.pkg; + v.version = pkgVersion.version; + v.files = pkgVersion.files; + v.manifest = pkgVersion.manifest; + v.manifestSHA512 = pkgVersion.manifestSHA512; + return v; } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java deleted file mode 100644 index 92147673c5e6..000000000000 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPIJaxRs.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.pkg; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; - -import jakarta.inject.Inject; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import org.apache.solr.api.JerseyResource; -import org.apache.solr.client.api.endpoint.PackageApis; -import org.apache.solr.client.api.model.AddPackageVersionRequestBody; -import org.apache.solr.client.api.model.PackagesResponse; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; -import org.apache.solr.jersey.PermissionName; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * JAX-RS implementation of the package management API ({@code /api/cluster/package}). - * - * @see PackageApis - */ -public class PackageAPIJaxRs extends JerseyResource implements PackageApis { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final int SYNC_MAX_RETRIES = 10; - private static final long SYNC_SLEEP_MS = 10L; - - private final CoreContainer coreContainer; - private final SolrQueryRequest solrQueryRequest; - private final SolrQueryResponse solrQueryResponse; - - @Inject - public PackageAPIJaxRs( - CoreContainer coreContainer, - SolrQueryRequest solrQueryRequest, - SolrQueryResponse solrQueryResponse) { - this.coreContainer = coreContainer; - this.solrQueryRequest = solrQueryRequest; - this.solrQueryResponse = solrQueryResponse; - } - - @Override - @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - if (refreshPackage != null) { - packageAPI.packageLoader.notifyListeners(refreshPackage); - return instantiateJerseyResponse(PackagesResponse.class); - } - - if (expectedVersion != null) { - syncToVersion(packageAPI, expectedVersion); - } - - final var response = instantiateJerseyResponse(PackagesResponse.class); - response.result = toPackageData(packageAPI.pkgs); - return response; - } - - @Override - @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse getPackage(String packageName) { - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - final var response = instantiateJerseyResponse(PackagesResponse.class); - response.result = toPackageData(packageAPI.pkgs); - // Filter to only the requested package - if (response.result != null && response.result.packages != null) { - final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = Collections.singletonMap(packageName, pkgVersions); - } - return response; - } - - @Override - @PermissionName(PACKAGE_EDIT_PERM) - public SolrJerseyResponse addPackageVersion( - String packageName, AddPackageVersionRequestBody requestBody) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - if (!packageAPI.isEnabled()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); - } - if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); - } - - final List errors = new ArrayList<>(); - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), requestBody.files, true, errors::add); - if (!errors.isEmpty()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); - } - - final PackageAPI.Packages[] finalState = new PackageAPI.Packages[1]; - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - PackageAPI.Packages packages; - try { - packages = - bytes == null - ? new PackageAPI.Packages() - : packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new PackageAPI.Packages(); - } - List list = - packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); - for (PackageAPI.PkgVersion pkgVersion : list) { - if (Objects.equals(pkgVersion.version, requestBody.version)) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Version '" + requestBody.version + "' exists already"); - } - } - list.add(new PackageAPI.PkgVersion(packageName, requestBody)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - packageAPI.handleZkErr(e); - } - - if (finalState[0] != null) { - packageAPI.pkgs = finalState[0]; - notifyAllNodesToSync(packageAPI.pkgs.znodeVersion); - coreContainer.getPackageLoader().refreshPackageConf(); - } - - return response; - } - - @Override - @PermissionName(PACKAGE_EDIT_PERM) - public SolrJerseyResponse deletePackageVersion(String packageName, String version) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - if (!packageAPI.isEnabled()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageAPI.ERR_MSG); - } - - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - PackageAPI.Packages packages; - try { - packages = packageAPI.mapper.readValue(bytes, PackageAPI.Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new PackageAPI.Packages(); - } - - List versions = packages.packages.get(packageName); - if (versions == null || versions.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); - } - int idxToRemove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, version)) { - idxToRemove = i; - break; - } - } - if (idxToRemove == -1) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); - } - versions.remove(idxToRemove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - packageAPI.handleZkErr(e); - } - - return response; - } - - @Override - @PermissionName(PACKAGE_EDIT_PERM) - public SolrJerseyResponse refreshPackage(String packageName) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - PackageAPI packageAPI = coreContainer.getPackageLoader().getPackageAPI(); - - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); - if (pkg == null) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); - } - // first refresh on the current node - packageAPI.packageLoader.notifyListeners(packageName); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", packageName); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } - - return response; - } - - private void syncToVersion(PackageAPI packageAPI, int expectedVersion) { - int origVersion = packageAPI.pkgs.znodeVersion; - for (int i = 0; i < SYNC_MAX_RETRIES; i++) { - if (log.isDebugEnabled()) { - log.debug( - "my version is {} , and expected version {}", - packageAPI.pkgs.znodeVersion, - expectedVersion); - } - if (packageAPI.pkgs.znodeVersion >= expectedVersion) { - if (origVersion < packageAPI.pkgs.znodeVersion) { - coreContainer.getPackageLoader().refreshPackageConf(); - } - return; - } - try { - Thread.sleep(SYNC_SLEEP_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - try { - packageAPI.pkgs = packageAPI.readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - packageAPI.handleZkErr(e); - } - } - } - - private void notifyAllNodesToSync(int expectedVersion) { - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expectedVersion)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " - + liveNode - + " to sync expected package version: " - + expectedVersion, - e); - } - } - } - - private static PackagesResponse.PackageData toPackageData(PackageAPI.Packages packages) { - if (packages == null) { - return null; - } - final var data = new PackagesResponse.PackageData(); - data.znodeVersion = packages.znodeVersion; - data.packages = - packages.packages.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - e -> - e.getValue().stream() - .map(PackageAPIJaxRs::toPkgVersionResponse) - .collect(Collectors.toList()))); - return data; - } - - private static PackagesResponse.PackageVersion toPkgVersionResponse( - PackageAPI.PkgVersion pkgVersion) { - final var v = new PackagesResponse.PackageVersion(); - v.pkg = pkgVersion.pkg; - v.version = pkgVersion.version; - v.files = pkgVersion.files; - v.manifest = pkgVersion.manifest; - v.manifestSHA512 = pkgVersion.manifestSHA512; - return v; - } -} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index 7535bb2c7fe9..b892ae4a9561 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -110,7 +110,7 @@ public interface Listener { String packageName(); /** fetch the package versions of class names */ - Map packageDetails(); + Map packageDetails(); /** A callback when the package is updated */ void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java index 272de544b6e7..8c9fa0a97889 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -42,7 +42,7 @@ public class PackageListeningClassLoader implements SolrClassLoader, PackageList private final Function pkgVersionSupplier; /** package name and the versions that we are tracking */ - private Map packageVersions = new ConcurrentHashMap<>(1); + private Map packageVersions = new ConcurrentHashMap<>(1); private Map classNameVsPackageName = new ConcurrentHashMap<>(); private final Runnable reloadAction; @@ -99,7 +99,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( p.getLatest(pkgVersionSupplier.apply(cName.pkg)); if (registerListener) { classNameVsPackageName.put(cName.original, cName.pkg); - PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion(); + PackageStore.PkgVersion pkgVersion = theVersion.getPkgVersion(); if (pkgVersion != null) packageVersions.put(cName.pkg, pkgVersion); } return theVersion; @@ -111,7 +111,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( @Override public MapWriter getPackageVersion(PluginInfo.ClassName cName) { if (cName.pkg == null) return null; - PackageAPI.PkgVersion p = packageVersions.get(cName.pkg); + PackageStore.PkgVersion p = packageVersions.get(cName.pkg); return p == null ? null : p::writeMap; } @@ -162,15 +162,15 @@ public String packageName() { } @Override - public Map packageDetails() { - Map result = new LinkedHashMap<>(); + public Map packageDetails() { + Map result = new LinkedHashMap<>(); classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v))); return result; } @Override public void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx) { - PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name); + PackageStore.PkgVersion currVer = packageVersions.get(pkg.name); if (currVer == null) { // not watching this return; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index b0731d05095f..2ac38f3b7ab3 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -59,7 +59,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Collections.singletonMap(info.cName.original, pkgVersion.getPkgVersion()); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java new file mode 100644 index 000000000000..77c32899c392 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.util.SolrJacksonAnnotationInspector; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds package data loaded from ZooKeeper ({@code /solr/packages.json}) and manages ZK watchers. + */ +public class PackageStore { + public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String ERR_MSG = + "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + + final CoreContainer coreContainer; + final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + final SolrPackageLoader packageLoader; + Packages pkgs; + + public PackageStore(CoreContainer coreContainer, SolrPackageLoader loader) { + this.coreContainer = coreContainer; + this.packageLoader = loader; + pkgs = new Packages(); + SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + try { + pkgs = readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + pkgs = new Packages(); + // ignore + } + try { + registerListener(zkClient); + } catch (KeeperException | InterruptedException e) { + SolrZkClient.checkInterrupted(e); + } + } + + private void registerListener(SolrZkClient zkClient) + throws KeeperException, InterruptedException { + zkClient.exists( + SOLR_PKGS_PATH, + new Watcher() { + + @Override + public void process(WatchedEvent event) { + // session events are not change events, and do not remove the watcher + if (Event.EventType.None.equals(event.getType())) { + return; + } + synchronized (this) { + log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); + // remake watch + final Watcher thisWatch = this; + refreshPackages(thisWatch); + } + } + }); + } + + public void refreshPackages(Watcher watcher) { + final Stat stat = new Stat(); + try { + final byte[] data = + coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); + pkgs = readPkgsFromZk(data, stat); + packageLoader.refreshPackageConf(); + } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { + log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); + } catch (KeeperException e) { + log.error("A ZK error has occurred", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } catch (InterruptedException e) { + // Restore the interrupted status + Thread.currentThread().interrupt(); + log.warn("Interrupted", e); + } + } + + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + + if (data == null || stat == null) { + stat = new Stat(); + data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + } + Packages packages = null; + if (data == null || data.length == 0) { + packages = new Packages(); + } else { + try { + packages = mapper.readValue(data, Packages.class); + packages.znodeVersion = stat.getVersion(); + } catch (IOException e) { + // invalid data in packages + // TODO handle properly; + return new Packages(); + } + } + return packages; + } + + public static class Packages implements ReflectMapWriter { + @JsonProperty public int znodeVersion = -1; + + @JsonProperty public Map> packages = new LinkedHashMap<>(); + + public Packages copy() { + Packages p = new Packages(); + p.znodeVersion = this.znodeVersion; + p.packages = new LinkedHashMap<>(); + packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); + return p; + } + } + + public static class PkgVersion implements ReflectMapWriter { + + @JsonProperty("package") + public String pkg; + + @JsonProperty public String version; + + @JsonProperty public List files; + + @JsonProperty public String manifest; + + @JsonProperty public String manifestSHA512; + + public PkgVersion() {} + + public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { + this.pkg = packageName; + this.version = addVersion.version; + this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.manifest = addVersion.manifest; + this.manifestSHA512 = addVersion.manifestSHA512; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof PkgVersion that) { + return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(version); + } + + @Override + public String toString() { + try { + return Utils.writeJson(this, new StringWriter(), false).toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public PkgVersion copy() { + PkgVersion result = new PkgVersion(); + result.pkg = this.pkg; + result.version = this.version; + result.files = this.files; + result.manifest = this.manifest; + result.manifestSHA512 = this.manifestSHA512; + return result; + } + } + + public boolean isEnabled() { + return enablePackages; + } + + public void handleZkErr(Exception e) { + log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); + } + + public boolean isJarInuse(String path) { + Packages pkg = null; + try { + pkg = readPkgsFromZk(null, null); + } catch (KeeperException.NoNodeException nne) { + return false; + } catch (InterruptedException | KeeperException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + for (List vers : pkg.packages.values()) { + for (PkgVersion ver : vers) { + if (ver.files.contains(path)) { + return true; + } + } + } + return false; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index f3a86dd55d7b..98593a355a25 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -53,9 +53,9 @@ public class SolrPackageLoader implements Closeable { private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); - private PackageAPI.Packages myCopy = new PackageAPI.Packages(); + private PackageStore.Packages myCopy = new PackageStore.Packages(); - private PackageAPI packageAPI; + private PackageStore packageStore; public Optional getPackageVersion(String pkg, String version) { SolrPackage p = packageClassLoaders.get(pkg); @@ -65,12 +65,12 @@ public Optional getPackageVersion(String pkg, String versio public SolrPackageLoader(CoreContainer coreContainer) { this.coreContainer = coreContainer; - packageAPI = new PackageAPI(coreContainer, this); + packageStore = new PackageStore(coreContainer, this); refreshPackageConf(); } - public PackageAPI getPackageAPI() { - return packageAPI; + public PackageStore getPackageStore() { + return packageStore; } public SolrPackage getPackage(String key) { @@ -83,12 +83,12 @@ public Map getPackages() { public void refreshPackageConf() { log.debug( - "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion); + "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageStore.pkgs.znodeVersion); List updated = new ArrayList<>(); - Map> modified = getModified(myCopy, packageAPI.pkgs); + Map> modified = getModified(myCopy, packageStore.pkgs); - for (Map.Entry> e : modified.entrySet()) { + for (Map.Entry> e : modified.entrySet()) { if (e.getValue() != null) { SolrPackage p = packageClassLoaders.get(e.getKey()); if (e.getValue() != null && p == null) { @@ -109,14 +109,14 @@ public void refreshPackageConf() { for (SolrCore core : coreContainer.getCores()) { core.getPackageListeners().packagesUpdated(updated); } - myCopy = packageAPI.pkgs; + myCopy = packageStore.pkgs; } - public Map> getModified( - PackageAPI.Packages old, PackageAPI.Packages newPkgs) { - Map> changed = new HashMap<>(); - for (Map.Entry> e : newPkgs.packages.entrySet()) { - List versions = old.packages.get(e.getKey()); + public Map> getModified( + PackageStore.Packages old, PackageStore.Packages newPkgs) { + Map> changed = new HashMap<>(); + for (Map.Entry> e : newPkgs.packages.entrySet()) { + List versions = old.packages.get(e.getKey()); if (versions != null) { if (!Objects.equals(e.getValue(), versions)) { if (log.isInfoEnabled()) { @@ -172,8 +172,8 @@ public Set allVersions() { return myVersions.keySet(); } - private synchronized void updateVersions(List modified) { - for (PackageAPI.PkgVersion v : modified) { + private synchronized void updateVersions(List modified) { + for (PackageStore.PkgVersion v : modified) { Version version = myVersions.get(v.version); if (version == null) { log.info( @@ -194,7 +194,7 @@ private synchronized void updateVersions(List modified) { } Set newVersions = new HashSet<>(); - for (PackageAPI.PkgVersion v : modified) { + for (PackageStore.PkgVersion v : modified) { newVersions.add(v.version); } for (String s : new HashSet<>(myVersions.keySet())) { @@ -258,7 +258,7 @@ public class Version implements MapWriter, Closeable { private final SolrPackage parent; private SolrResourceLoader loader; - private final PackageAPI.PkgVersion version; + private final PackageStore.PkgVersion version; @Override public void writeMap(EntryWriter ew) throws IOException { @@ -266,7 +266,7 @@ public void writeMap(EntryWriter ew) throws IOException { version.writeMap(ew); } - Version(SolrPackage parent, PackageAPI.PkgVersion v) { + Version(SolrPackage parent, PackageStore.PkgVersion v) { this.parent = parent; this.version = v; List paths = new ArrayList<>(); @@ -293,7 +293,7 @@ public String getVersion() { return version.version; } - public PackageAPI.PkgVersion getPkgVersion() { + public PackageStore.PkgVersion getPkgVersion() { return version.copy(); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 06593f4b3ab9..4d451c17eaf3 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -58,8 +58,8 @@ import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.filestore.TestDistribFileStore.Fetcher; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; @@ -93,7 +93,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return null; // only used to print meta information } diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java similarity index 89% rename from solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java rename to solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java index 8e3c1f4b72ca..b1a29c819464 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPIJaxRsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java @@ -28,12 +28,12 @@ import org.junit.Test; /** - * Integration tests for the JAX-RS-based {@link PackageAPIJaxRs}. + * Integration tests for the JAX-RS-based {@link PackageAPI}. * *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its * cluster-level operations. A one-node SolrCloud cluster is used instead. */ -public class PackageAPIJaxRsTest extends SolrCloudTestCase { +public class PackageAPITest extends SolrCloudTestCase { @BeforeClass public static void setupCluster() throws Exception { @@ -56,7 +56,7 @@ public void testListPackagesReturnsResult() throws Exception { @Test public void testAddAndDeletePackageVersion() throws Exception { - String FILE1 = "/jaxrstestpkg/runtimelibs.jar"; + String FILE1 = "/pkgapitestpkg/runtimelibs.jar"; // Upload a key and a signed jar file to the filestore byte[] derFile = @@ -71,7 +71,7 @@ public void testAddAndDeletePackageVersion() throws Exception { try (HttpSolrClient client = new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { // Add a package version via POST /cluster/package/{name}/versions - PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("jaxrstestpkg"); + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); addRequest.setVersion("1.0"); addRequest.setFiles(List.of(FILE1)); addRequest.process(client); @@ -81,29 +81,29 @@ public void testAddAndDeletePackageVersion() throws Exception { assertNotNull("Expected non-null list response", listResponse); assertNotNull("Expected non-null result", listResponse.result); assertNotNull( - "Expected jaxrstestpkg in packages", listResponse.result.packages.get("jaxrstestpkg")); + "Expected pkgapitestpkg in packages", listResponse.result.packages.get("pkgapitestpkg")); assertFalse( "Expected at least one version", - listResponse.result.packages.get("jaxrstestpkg").isEmpty()); + listResponse.result.packages.get("pkgapitestpkg").isEmpty()); // Verify GET /cluster/package/{name} returns only this package PackagesResponse getByNameResponse = - new PackageApi.GetPackage("jaxrstestpkg").process(client); + new PackageApi.GetPackage("pkgapitestpkg").process(client); assertNotNull("Expected non-null get-by-name response", getByNameResponse); assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); assertNotNull( - "Expected jaxrstestpkg in get-by-name response", - getByNameResponse.result.packages.get("jaxrstestpkg")); + "Expected pkgapitestpkg in get-by-name response", + getByNameResponse.result.packages.get("pkgapitestpkg")); // Delete the package version via DELETE /cluster/package/{name}/versions/{version} - new PackageApi.DeletePackageVersion("jaxrstestpkg", "1.0").process(client); + new PackageApi.DeletePackageVersion("pkgapitestpkg", "1.0").process(client); // Verify it's deleted PackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null list response after delete", listAfterDelete); assertNotNull("Expected non-null result after delete", listAfterDelete.result); // After deleting the only version, the package entry should be empty or absent - List versionsAfterDelete = listAfterDelete.result.packages.get("jaxrstestpkg"); + List versionsAfterDelete = listAfterDelete.result.packages.get("pkgapitestpkg"); assertTrue( "Expected no versions after delete", versionsAfterDelete == null || versionsAfterDelete.isEmpty()); @@ -137,7 +137,7 @@ public void testRefreshNonExistentPackage() throws Exception { // Try to refresh a non-existent package. // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; // instead, the error is in the response body's 'error' field. - var response = new PackageApi.RefreshPackage("nonexistentpkg_jaxrs").process(client); + var response = new PackageApi.RefreshPackage("nonexistentpkg_test").process(client); assertNotNull("Expected error in response for non-existent package", response.error); assertEquals("Expected 400 for non-existent package", 400, (int) response.error.code); assertTrue( From d71f19933785eea0cf67781b85f42416300e7cbb Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 1 Mar 2026 11:56:51 -0500 Subject: [PATCH 07/31] Additional migrations from the old "command" style to the new RESTful style --- .../solr/packagemanager/PackageManager.java | 27 ++--------- .../packagemanager/RepositoryManager.java | 33 ++++--------- .../solr/handler/TestContainerPlugin.java | 23 ++++----- .../pkg/PackageStoreSchemaPluginsTest.java | 22 ++------- .../solrj/request/beans/PackagePayload.java | 47 ------------------- 5 files changed, 27 insertions(+), 125 deletions(-) delete mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 943c9248a62b..4dc807b44e2e 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -51,10 +51,8 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.beans.PluginMeta; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; @@ -152,20 +150,9 @@ public void uninstall(String packageName, String version) // Delete the package by calling the Package API and remove the Jar printGreen("Executing Package API to remove this package..."); - PackagePayload.DelVersion del = new PackagePayload.DelVersion(); - del.version = version; - del.pkg = packageName; - - V2Request req = - new V2Request.Builder(PackageUtils.PACKAGE_PATH) - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("delete", del)) - .build(); - try { - V2Response resp = req.process(solrClient); - printGreen("Response: " + resp.jsonStr()); + new PackageApi.DeletePackageVersion(packageName, version).process(solrClient); + printGreen("Package version deleted from Package API."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } @@ -467,10 +454,7 @@ private Pair, List> deployCollectionPackage( // If updating, refresh the package version for this to take effect if (isUpdate || pegToLatest) { try { - SolrCLI.postJsonToSolr( - solrClient, - PackageUtils.PACKAGE_PATH, - "{\"refresh\": \"" + packageInstance.name + "\"}"); + new PackageApi.RefreshPackage(packageInstance.name).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -1082,8 +1066,7 @@ public void undeploy( solrClient, PackageUtils.getCollectionParamsPath(collection), "{set: {PKG_VERSIONS: {" + packageName + ": null}}}"); - SolrCLI.postJsonToSolr( - solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}"); + new PackageApi.RefreshPackage(packageName).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 5ae49bd02a39..3a99673b1531 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -40,20 +40,15 @@ import org.apache.solr.cli.SolrCLI; import org.apache.solr.client.api.util.SolrVersion; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.FileStoreApi; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.SystemInfoRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.SystemInfoResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; @@ -226,10 +221,9 @@ private boolean installPackage(String packageName, String version) throws SolrEx // Call Package API to add this version of the package printGreen("Executing Package API to register this package..."); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = version; - add.pkg = packageName; - add.files = + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion(packageName); + addRequest.setVersion(version); + addRequest.setFiles( downloaded.stream() .map( file -> @@ -239,21 +233,12 @@ private boolean installPackage(String packageName, String version) throws SolrEx packageName, version, file.getFileName().toString())) - .collect(Collectors.toList()); - add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; - add.manifestSHA512 = manifestSHA512; - - GenericSolrRequest request = - new GenericV2SolrRequest(SolrRequest.METHOD.POST, PackageUtils.PACKAGE_PATH) { - @Override - public RequestWriter.ContentWriter getContentWriter(String expectedType) { - return new RequestWriter.StringPayloadContentWriter( - "{add:" + add.jsonStr() + "}", "application/json"); - } - }; + .collect(Collectors.toList())); + addRequest.setManifest("/package/" + packageName + "/" + version + "/manifest.json"); + addRequest.setManifestSHA512(manifestSHA512); try { - NamedList resp = solrClient.request(request); - printGreen("Response: " + resp.jsonStr()); + addRequest.process(solrClient); + printGreen("Package version registered successfully."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 4d451c17eaf3..3b312785e175 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -42,8 +42,8 @@ import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.request.beans.PluginMeta; import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.ClusterSingleton; @@ -309,16 +309,9 @@ public void testApiFromPackage() throws Exception { // We have two versions of the plugin in 2 different jar files. they are already uploaded to // the package store listener.reset(); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = singletonList(FILE1); - V2Request addPkgVersionReq = - new V2Request.Builder("/cluster/package") - .forceV2(forceV2) - .POST() - .withPayload(singletonMap("add", add)) - .build(); + PackageApi.AddPackageVersion addPkgVersionReq = new PackageApi.AddPackageVersion("mypkg"); + addPkgVersionReq.setVersion("1.0"); + addPkgVersionReq.setFiles(singletonList(FILE1)); addPkgVersionReq.process(cluster.getSolrClient()); assertTrue( "core package listeners did not notify", @@ -336,7 +329,7 @@ public void testApiFromPackage() throws Exception { PluginMeta plugin = new PluginMeta(); plugin.name = "myplugin"; plugin.klass = "mypkg:org.apache.solr.handler.MyPlugin"; - plugin.version = add.version; + plugin.version = "1.0"; final V2Request addPluginReq = postPlugin(singletonMap("add", plugin)); addPluginReq.process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); @@ -352,12 +345,12 @@ public void testApiFromPackage() throws Exception { TestDistribFileStore.assertResponseValues(invokePlugin, Map.of("/myplugin.version", "1.0")); // now let's upload the jar file for version 2.0 of the plugin - add.version = "2.0"; - add.files = singletonList(FILE2); + addPkgVersionReq.setVersion("2.0"); + addPkgVersionReq.setFiles(singletonList(FILE2)); addPkgVersionReq.process(cluster.getSolrClient()); // here the plugin version is updated - plugin.version = add.version; + plugin.version = "2.0"; postPlugin(singletonMap("update", plugin)).process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 8faac9b68f9e..4091472b3331 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -27,10 +27,10 @@ import java.security.Signature; import java.util.Base64; import java.util.List; -import java.util.Map; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.cloud.SolrCloudTestCase; @@ -133,22 +133,10 @@ private void uploadPluginJar(String version, Path jarPath) throws Exception { } private void registerPackage(String version) throws Exception { - var packageRequest = - new V2Request.Builder("/cluster/package") - .POST() - .forceV2(true) - .withPayload( - Map.of( - "add", - Map.of( - "package", - "mypkg", - "version", - version, - "files", - List.of("/my-plugin/plugin-" + version + ".jar")))) - .build(); - processRequest(client, packageRequest); + var addRequest = new PackageApi.AddPackageVersion("mypkg"); + addRequest.setVersion(version); + addRequest.setFiles(List.of("/my-plugin/plugin-" + version + ".jar")); + processRequest(client, addRequest); } private void createCollection() throws Exception { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java deleted file mode 100644 index 831d87d809f7..000000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.request.beans; - -import java.util.List; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.util.ReflectMapWriter; - -/** Just a container class for POJOs used in Package APIs */ -public class PackagePayload { - public static class AddVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; - - @JsonProperty(required = true) - public String version; - - @JsonProperty(required = true) - public List files; - - @JsonProperty public String manifest; - @JsonProperty public String manifestSHA512; - } - - public static class DelVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; - - @JsonProperty(required = true) - public String version; - } -} From 8994e034b79368b8b9c12d4c4ed0953f85123210 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 1 Mar 2026 12:53:51 -0500 Subject: [PATCH 08/31] fix test --- .../test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 4091472b3331..25f0e481cd72 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -136,7 +136,7 @@ private void registerPackage(String version) throws Exception { var addRequest = new PackageApi.AddPackageVersion("mypkg"); addRequest.setVersion(version); addRequest.setFiles(List.of("/my-plugin/plugin-" + version + ".jar")); - processRequest(client, addRequest); + addRequest.process(client); } private void createCollection() throws Exception { From 199b0f312bdb46536e3bede12004576b6f8a2969 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 3 Mar 2026 08:16:45 -0500 Subject: [PATCH 09/31] doc change --- changelog/unreleased/migrate-packageapi-to-jax-rs.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/migrate-packageapi-to-jax-rs.yml diff --git a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml new file mode 100644 index 000000000000..62157390e348 --- /dev/null +++ b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml @@ -0,0 +1,7 @@ +title: Migrate PackageAPI to JAX-RS. PackageAPI now has OpenAPI and SolrJ support. +type: changed +authors: + - name: Eric Pugh +links: +- name: PR#4178 + url: https://github.com/apache/solr/pull/4178 From ae94dd366c051d8b26e6ca9c85e847120262e6d6 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 07:43:48 -0400 Subject: [PATCH 10/31] Updates from merging in latest code from main. --- .../solr/handler/component/SearchHandler.java | 2 +- .../packagemanager/RepositoryManager.java | 6 +- .../java/org/apache/solr/pkg/PackageAPI.java | 3 +- .../apache/solr/pkg/PackagePluginHolder.java | 2 +- .../solr/handler/TestContainerPlugin.java | 2 +- .../org/apache/solr/pkg/PackageAPITest.java | 59 ++++++++++--------- .../org/apache/solr/pkg/TestPackages.java | 14 ++--- 7 files changed, 43 insertions(+), 45 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index c9f05e0c2563..20711a94b9f8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -201,7 +201,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index c74a80181422..6867d17b3653 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -43,12 +43,10 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.PackageApi; -import org.apache.solr.client.solrj.request.SystemInfoRequest; -import org.apache.solr.client.solrj.response.SystemInfoResponse; +import org.apache.solr.client.solrj.request.SystemApi; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; @@ -162,7 +160,7 @@ private String getRepositoriesJson(SolrZkClient zkClient) /** * Install a given package and version from the available repositories to Solr. The various steps * for doing so are, briefly, a) find upload a manifest to package store, b) download the - * artifacts and upload to package store, c) call {@link PackageAPI} to register the package. + * artifacts and upload to package store, c) call {@link PackageManager} to register the package. */ private boolean installPackage(String packageName, String version) throws SolrException { SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 0e0318bd0505..239e6a546329 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -103,7 +102,7 @@ public PackagesResponse getPackage(String packageName) { // Filter to only the requested package if (response.result != null && response.result.packages != null) { final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = Collections.singletonMap(packageName, pkgVersions); + response.result.packages = Map.of(packageName, pkgVersions); } return response; } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 419cb033835f..b68f275232fa 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -58,7 +58,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Map.of(info.cName.original, pkgVersion.getPkgVersion()); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 82b5847ea164..950842555669 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -310,7 +310,7 @@ public void testApiFromPackage() throws Exception { listener.reset(); PackageApi.AddPackageVersion addPkgVersionReq = new PackageApi.AddPackageVersion("mypkg"); addPkgVersionReq.setVersion("1.0"); - addPkgVersionReq.setFiles(singletonList(FILE1)); + addPkgVersionReq.setFiles(List.of(FILE1)); addPkgVersionReq.process(cluster.getSolrClient()); assertTrue( "core package listeners did not notify", diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java index b1a29c819464..aaadebc00ee3 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java @@ -20,10 +20,12 @@ import java.util.List; import org.apache.solr.client.api.model.PackagesResponse; -import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.RemoteSolrException; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; import org.junit.BeforeClass; import org.junit.Test; @@ -46,8 +48,9 @@ public static void setupCluster() throws Exception { @Test public void testListPackagesReturnsResult() throws Exception { - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { PackagesResponse response = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null response from GET /cluster/package", response); assertNotNull("Expected 'result' field in GET /cluster/package response", response.result); @@ -59,8 +62,7 @@ public void testAddAndDeletePackageVersion() throws Exception { String FILE1 = "/pkgapitestpkg/runtimelibs.jar"; // Upload a key and a signed jar file to the filestore - byte[] derFile = - org.apache.solr.filestore.TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); + byte[] derFile = TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); TestPackages.postFileAndWait( cluster, @@ -68,8 +70,9 @@ public void testAddAndDeletePackageVersion() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { // Add a package version via POST /cluster/package/{name}/versions PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); addRequest.setVersion("1.0"); @@ -111,38 +114,36 @@ public void testAddAndDeletePackageVersion() throws Exception { } @Test - public void testAddPackageVersionValidatesFiles() throws Exception { - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { - // Try to add a package version with a non-existent file. - // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; - // instead, the error is in the response body's 'error' field. + public void testAddPackageVersionValidatesFiles() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("testpkg_invalid"); addRequest.setVersion("1.0"); addRequest.setFiles(List.of("/nonexistent/file.jar")); - var response = addRequest.process(client); - assertNotNull("Expected error in response for non-existent file", response.error); - assertEquals("Expected 400 for non-existent file", 400, (int) response.error.code); + RemoteSolrException ex = + expectThrows(RemoteSolrException.class, () -> addRequest.process(client)); + assertEquals("Expected 400 for non-existent file", 400, ex.code()); assertTrue( - "Expected error message to mention the file", - response.error.msg.contains("No such file")); + "Expected error message to mention the file: " + ex.getMessage(), + ex.getMessage().contains("No such file")); } } @Test - public void testRefreshNonExistentPackage() throws Exception { - try (HttpSolrClient client = - new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { - // Try to refresh a non-existent package. - // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; - // instead, the error is in the response body's 'error' field. - var response = new PackageApi.RefreshPackage("nonexistentpkg_test").process(client); - assertNotNull("Expected error in response for non-existent package", response.error); - assertEquals("Expected 400 for non-existent package", 400, (int) response.error.code); + public void testRefreshNonExistentPackage() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + RemoteSolrException ex = + expectThrows( + RemoteSolrException.class, + () -> new PackageApi.RefreshPackage("nonexistentpkg_test").process(client)); + assertEquals("Expected 400 for non-existent package", 400, ex.code()); assertTrue( - "Expected error message to mention the package", - response.error.msg.contains("No such package")); + "Expected error message to mention the package: " + ex.getMessage(), + ex.getMessage().contains("No such package")); } } } diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index a7d1894bb0c4..249fd43ef725 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -134,7 +134,7 @@ public void testCoreReloadingPlugin() throws Exception { AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.files = Arrays.asList(new String[] {FILE1}); + add.files = List.of(FILE1); V2Request req = new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) @@ -214,7 +214,7 @@ public void testPluginLoading() throws Exception { AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.files = Arrays.asList(new String[] {FILE1, URP1, EXPR1}); + add.files = List.of(FILE1, URP1, EXPR1); V2Request req = new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) @@ -343,7 +343,7 @@ public void testPluginLoading() throws Exception { "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw=="); // add the version using package API add.version = "1.1"; - add.files = Arrays.asList(new String[] {FILE2, URP2, EXPR1}); + add.files = List.of(FILE2, URP2, EXPR1); req.process(cluster.getSolrClient()); verifyComponent( @@ -372,7 +372,7 @@ public void testPluginLoading() throws Exception { "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + add.files = List.of(FILE3, URP2, EXPR1); req.process(cluster.getSolrClient()); // now let's verify that the classes are updated @@ -445,7 +445,7 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { }.setRequiresCollection(true).process(cluster.getSolrClient()); add.version = "2.1"; - add.files = Arrays.asList(new String[] {FILE3, URP2, EXPR1}); + add.files = List.of(FILE3, URP2, EXPR1); req.process(cluster.getSolrClient()); // the collections mypkg is set to use version 1.1 @@ -637,7 +637,7 @@ public void testAPI() throws Exception { // this time we are adding the second version of the package (0.13) add.version = "0.13"; - add.files = Collections.singletonList(FILE3); + add.files = List.of(FILE3); // this request should succeed req.process(cluster.getSolrClient()); @@ -888,7 +888,7 @@ public BaseWhitespaceTokenizerFactory(Map args) { } /* - //copy the jav files to a package and then run the main method + //copy the java files to a package and then run the main method public static void main(String[] args) throws Exception { persistZip("/tmp/x.jar", MyPatternReplaceCharFilterFactory.class, MyTextField.class, MyWhitespaceTokenizerFactory.class); }*/ From 62212c3930f4f583449db014f372c517a0b8c4f5 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 07:49:45 -0400 Subject: [PATCH 11/31] PackageAPI and PackageApis super confusing. Rename PackageAPI to ClusterPackage to follow naming pattern of other V2 apis. --- changelog/unreleased/migrate-packageapi-to-jax-rs.yml | 2 +- solr/core/src/java/org/apache/solr/core/CoreContainer.java | 4 ++-- .../solr/pkg/{PackageAPI.java => ClusterPackage.java} | 6 +++--- .../pkg/{PackageAPITest.java => ClusterPackageTest.java} | 4 ++-- solr/core/src/test/org/apache/solr/pkg/TestPackages.java | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) rename solr/core/src/java/org/apache/solr/pkg/{PackageAPI.java => ClusterPackage.java} (98%) rename solr/core/src/test/org/apache/solr/pkg/{PackageAPITest.java => ClusterPackageTest.java} (98%) diff --git a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml index 62157390e348..f2d28ad93aba 100644 --- a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml +++ b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml @@ -1,4 +1,4 @@ -title: Migrate PackageAPI to JAX-RS. PackageAPI now has OpenAPI and SolrJ support. +title: Migrate PackageAPI to JAX-RS (now ClusterPackage). The package management API now has OpenAPI and SolrJ support. type: changed authors: - name: Eric Pugh diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 0325b920415e..60aff6f7dac2 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -135,7 +135,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; -import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.ClusterPackage; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -848,7 +848,7 @@ private void loadInternal() { registerV2Api(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2Api(PackageAPI.class); + registerV2Api(ClusterPackage.class); registerV2Api(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java similarity index 98% rename from solr/core/src/java/org/apache/solr/pkg/PackageAPI.java rename to solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 239e6a546329..4a58cb7cb96d 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -54,7 +54,7 @@ * * @see PackageApis */ -public class PackageAPI extends JerseyResource implements PackageApis { +public class ClusterPackage extends JerseyResource implements PackageApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final int SYNC_MAX_RETRIES = 10; @@ -65,7 +65,7 @@ public class PackageAPI extends JerseyResource implements PackageApis { private final SolrQueryResponse solrQueryResponse; @Inject - public PackageAPI( + public ClusterPackage( CoreContainer coreContainer, SolrQueryRequest solrQueryRequest, SolrQueryResponse solrQueryResponse) { @@ -333,7 +333,7 @@ private static PackagesResponse.PackageData toPackageData(PackageStore.Packages Map.Entry::getKey, e -> e.getValue().stream() - .map(PackageAPI::toPkgVersionResponse) + .map(ClusterPackage::toPkgVersionResponse) .collect(Collectors.toList()))); return data; } diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java similarity index 98% rename from solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java rename to solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index aaadebc00ee3..dcbde47a7c59 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -30,12 +30,12 @@ import org.junit.Test; /** - * Integration tests for the JAX-RS-based {@link PackageAPI}. + * Integration tests for the JAX-RS-based {@link ClusterPackage}. * *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its * cluster-level operations. A one-node SolrCloud cluster is used instead. */ -public class PackageAPITest extends SolrCloudTestCase { +public class ClusterPackageTest extends SolrCloudTestCase { @BeforeClass public static void setupCluster() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 249fd43ef725..22850ecbd139 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -88,7 +88,7 @@ import org.junit.Before; import org.junit.Test; -@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") +@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.ClusterPackage=DEBUG") public class TestPackages extends SolrCloudTestCase { @Before From 30ab7ccf3a9ac7170bfed1cd7c3195e7132fec8d Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:30:38 -0400 Subject: [PATCH 12/31] Fix up expectedVersion property that was missing. Add test, revamp description. --- .../solr/client/api/endpoint/PackageApis.java | 11 ++++-- .../org/apache/solr/pkg/ClusterPackage.java | 12 +++++-- .../apache/solr/pkg/ClusterPackageTest.java | 34 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java index 9aeb40119d76..cf3a50f8caff 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -38,7 +38,9 @@ public interface PackageApis { summary = "List all packages registered in this Solr cluster.", tags = {"package"}) PackagesResponse listPackages( - @Parameter(description = "If provided, the named package is refreshed on this node.") + @Parameter( + description = + "Name of a package to refresh on the receiving node only (reloads its listeners from ZooKeeper). Used internally for inter-node fan-out by the refresh endpoint; not normally invoked directly.") @QueryParam("refreshPackage") String refreshPackage, @Parameter( @@ -55,7 +57,12 @@ PackagesResponse listPackages( PackagesResponse getPackage( @Parameter(description = "The name of the package.", required = true) @PathParam("packageName") - String packageName); + String packageName, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version before responding.") + @QueryParam("expectedVersion") + Integer expectedVersion); @POST @Path("/{packageName}/versions") diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 4a58cb7cb96d..a091555e664f 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -95,14 +95,20 @@ public PackagesResponse listPackages(String refreshPackage, Integer expectedVers @Override @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse getPackage(String packageName) { + public PackagesResponse getPackage(String packageName, Integer expectedVersion) { PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); + } + final var response = instantiateJerseyResponse(PackagesResponse.class); response.result = toPackageData(packageStore.pkgs); - // Filter to only the requested package + // Filter to only the requested package; if absent, return an empty packages map. if (response.result != null && response.result.packages != null) { final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = Map.of(packageName, pkgVersions); + response.result.packages = + pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); } return response; } diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index dcbde47a7c59..324520161a1b 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -146,4 +146,38 @@ public void testRefreshNonExistentPackage() { ex.getMessage().contains("No such package")); } } + + @Test + public void testListPackagesAcceptsRefreshPackageQueryParam() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + PackageApi.ListPackages req = new PackageApi.ListPackages(); + req.setRefreshPackage("nonexistentpkg_refresh"); + // The inter-node refresh signal calls notifyListeners on this node and returns OK even when + // the package doesn't exist; main's behavior is identical. + PackagesResponse response = req.process(client); + assertNotNull(response); + } + } + + @Test + public void testListAndGetPackageAcceptExpectedVersionQueryParam() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // expectedVersion at-or-below current is a no-op in syncToVersion and must return promptly. + PackageApi.ListPackages listReq = new PackageApi.ListPackages(); + listReq.setExpectedVersion(0); + PackagesResponse listResponse = listReq.process(client); + assertNotNull(listResponse); + assertNotNull(listResponse.result); + + PackageApi.GetPackage getReq = new PackageApi.GetPackage("anything"); + getReq.setExpectedVersion(0); + PackagesResponse getResponse = getReq.process(client); + assertNotNull(getResponse); + assertNotNull(getResponse.result); + } + } } From 99d90e759a295d47f9d0d5e6bc1b2e6a6fe29ca9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:31:08 -0400 Subject: [PATCH 13/31] Use our generated solrj code instead of hand crafting. --- .../solr/filestore/TestDistribFileStore.java | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 8e7b5ab80ed0..4585696d7805 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -32,6 +32,7 @@ import java.util.concurrent.Callable; import java.util.function.Predicate; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.solr.client.api.model.UploadToFileStoreResponse; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -39,12 +40,9 @@ import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SimpleSolrResponse; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.SuppressForbidden; @@ -97,7 +95,7 @@ public void testFileStoreManagement() throws Exception { "/package/mypkg/v1.0/runtimelibs.jar", "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - NavigableObject rsp = + UploadToFileStoreResponse rsp = postFile( cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"), @@ -105,8 +103,7 @@ public void testFileStoreManagement() throws Exception { "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); assertTrue( - Objects.requireNonNullElse(rsp._getStr("message"), "") - .contains("File with same metadata exists ")); + Objects.requireNonNullElse(rsp.message, "").contains("File with same metadata exists ")); assertResponseValues( 10, @@ -344,22 +341,15 @@ public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster clu false); } - public static NavigableObject postFile( + public static UploadToFileStoreResponse postFile( SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { - String resource = "/cluster/filestore/files" + name; - ModifiableSolrParams params = new ModifiableSolrParams(); - params.add("sig", sig); - V2Response rsp = - new V2Request.Builder(resource) - .withMethod(SolrRequest.METHOD.PUT) - .withPayload(buffer) - .forceV2(true) - .withMimeType("application/octet-stream") - .withParams(params) - .build() - .process(client); - assertEquals(name, rsp.getResponse().get(CommonParams.FILE)); + final var uploadReq = new FileStoreApi.UploadFile(name, new ByteBufferInputStream(buffer)); + if (sig != null) { + uploadReq.setSig(List.of(sig)); + } + final UploadToFileStoreResponse rsp = uploadReq.process(client); + assertEquals(name, rsp.file); return rsp; } From 796757471054952b6e6f8e0265dc2b606a3c941b Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:34:03 -0400 Subject: [PATCH 14/31] tidy --- solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index a091555e664f..3036985eecd0 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -107,8 +107,7 @@ public PackagesResponse getPackage(String packageName, Integer expectedVersion) // Filter to only the requested package; if absent, return an empty packages map. if (response.result != null && response.result.packages != null) { final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = - pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); + response.result.packages = pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); } return response; } From 749173088ff17f8171b449ad2bb251799d038ee4 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:40:11 -0400 Subject: [PATCH 15/31] Small cleanups. --- solr/core/src/java/org/apache/solr/pkg/PackageStore.java | 6 +++--- .../src/java/org/apache/solr/pkg/SolrPackageLoader.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java index 77c32899c392..d51c8f56a2a7 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -54,7 +54,7 @@ public class PackageStore { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + "Package loading is not enabled, start your nodes with -Dsolr.packages.enabled=true"; final CoreContainer coreContainer; final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); @@ -126,7 +126,7 @@ Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, Interrup stat = new Stat(); data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); } - Packages packages = null; + Packages packages; if (data == null || data.length == 0) { packages = new Packages(); } else { @@ -221,7 +221,7 @@ public void handleZkErr(Exception e) { } public boolean isJarInuse(String path) { - Packages pkg = null; + Packages pkg; try { pkg = readPkgsFromZk(null, null); } catch (KeeperException.NoNodeException nne) { diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index c0b0eb288556..84fbea486af4 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -134,7 +134,7 @@ public Map> getModified( // some packages are deleted altogether for (String s : old.packages.keySet()) { if (!newPkgs.packages.keySet().contains(s)) { - log.info("Package: {} is removed althogether", s); + log.info("Package: {} is removed altogether", s); changed.put(s, null); } } @@ -330,7 +330,7 @@ static class PackageResourceLoader extends SolrResourceLoader { @Override public boolean addToCoreAware(T obj) { // do not do anything - // this class is not aware of a SolrCore and it is totally not tied to + // this class is not aware of a SolrCore, and it is totally not tied to // the lifecycle of SolrCore. So, this returns 'false' & it should be // taken care of by the caller return false; From deb2142d0dc49574650d1dfbb4b7c7bdd52bbefd Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 12:44:05 -0400 Subject: [PATCH 16/31] Now have a JIRA issue --- .../unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml | 7 +++++++ changelog/unreleased/migrate-packageapi-to-jax-rs.yml | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml delete mode 100644 changelog/unreleased/migrate-packageapi-to-jax-rs.yml diff --git a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml new file mode 100644 index 000000000000..67ac97f93f9c --- /dev/null +++ b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml @@ -0,0 +1,7 @@ +title: The package management API now has OpenAPI and SolrJ support. +type: changed +authors: + - name: Eric Pugh +links: +- name: PR#4178 + url: https://github.com/apache/solr/pull/4178 diff --git a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml deleted file mode 100644 index f2d28ad93aba..000000000000 --- a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml +++ /dev/null @@ -1,7 +0,0 @@ -title: Migrate PackageAPI to JAX-RS (now ClusterPackage). The package management API now has OpenAPI and SolrJ support. -type: changed -authors: - - name: Eric Pugh -links: -- name: PR#4178 - url: https://github.com/apache/solr/pull/4178 From 2578d2670791fbe0c3b285f18907b2d01df14874 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:18:53 -0400 Subject: [PATCH 17/31] shockingly, we had these unused objects --- .../src/java/org/apache/solr/pkg/ClusterPackage.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 3036985eecd0..22c52d88bad7 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -43,8 +43,6 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.filestore.FileStoreUtils; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,17 +59,10 @@ public class ClusterPackage extends JerseyResource implements PackageApis { private static final long SYNC_SLEEP_MS = 10L; private final CoreContainer coreContainer; - private final SolrQueryRequest solrQueryRequest; - private final SolrQueryResponse solrQueryResponse; @Inject - public ClusterPackage( - CoreContainer coreContainer, - SolrQueryRequest solrQueryRequest, - SolrQueryResponse solrQueryResponse) { + public ClusterPackage(CoreContainer coreContainer) { this.coreContainer = coreContainer; - this.solrQueryRequest = solrQueryRequest; - this.solrQueryResponse = solrQueryResponse; } @Override From bf46dc7c7bba1031a748ff3fe5ed31d71acd8bea Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:23:19 -0400 Subject: [PATCH 18/31] Don't need fancy instantiate method! --- .../src/java/org/apache/solr/pkg/ClusterPackage.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 22c52d88bad7..222e0a42c224 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -72,14 +72,14 @@ public PackagesResponse listPackages(String refreshPackage, Integer expectedVers if (refreshPackage != null) { packageStore.packageLoader.notifyListeners(refreshPackage); - return instantiateJerseyResponse(PackagesResponse.class); + return new PackagesResponse(); } if (expectedVersion != null) { syncToVersion(packageStore, expectedVersion); } - final var response = instantiateJerseyResponse(PackagesResponse.class); + final var response = new PackagesResponse(); response.result = toPackageData(packageStore.pkgs); return response; } @@ -93,7 +93,7 @@ public PackagesResponse getPackage(String packageName, Integer expectedVersion) syncToVersion(packageStore, expectedVersion); } - final var response = instantiateJerseyResponse(PackagesResponse.class); + final var response = new PackagesResponse(); response.result = toPackageData(packageStore.pkgs); // Filter to only the requested package; if absent, return an empty packages map. if (response.result != null && response.result.packages != null) { @@ -107,7 +107,7 @@ public PackagesResponse getPackage(String packageName, Integer expectedVersion) @PermissionName(PACKAGE_EDIT_PERM) public SolrJerseyResponse addPackageVersion( String packageName, AddPackageVersionRequestBody requestBody) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); if (!packageStore.isEnabled()) { @@ -174,7 +174,7 @@ public SolrJerseyResponse addPackageVersion( @Override @PermissionName(PACKAGE_EDIT_PERM) public SolrJerseyResponse deletePackageVersion(String packageName, String version) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); if (!packageStore.isEnabled()) { @@ -226,7 +226,7 @@ public SolrJerseyResponse deletePackageVersion(String packageName, String versio @Override @PermissionName(PACKAGE_EDIT_PERM) public SolrJerseyResponse refreshPackage(String packageName) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); From a0bef9075d19671d40fd03747892382ace90a3c0 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:30:44 -0400 Subject: [PATCH 19/31] Use SolrJ methods where possible. --- .../solr/packagemanager/PackageManager.java | 7 +++--- .../org/apache/solr/pkg/ClusterPackage.java | 23 ++++--------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 3dd95eece80b..4c7868e5aa7f 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -48,7 +48,7 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; -import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.GenericV2SolrRequest; import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.beans.PluginMeta; @@ -163,9 +163,8 @@ public void uninstall(String packageName, String version) String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json")); for (String filePath : filesToDelete) { DistribFileStore.deleteZKFileEntry(zkClient, filePath); - String path = "/api/cluster/filestore/files" + filePath; - printGreen("Deleting " + path); - solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.DELETE, path)); + printGreen("Deleting " + filePath); + new FileStoreApi.DeleteFile(filePath).process(solrClient); } printGreen("Package uninstalled: " + packageName + ":" + version + ":-)"); diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 222e0a42c224..e7bdc0979374 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -33,12 +33,9 @@ import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.client.api.model.PackagesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.filestore.FileStoreUtils; @@ -237,13 +234,8 @@ public SolrJerseyResponse refreshPackage(String packageName) { // first refresh on the current node packageStore.packageLoader.notifyListeners(packageName); - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", packageName); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); + final var request = new PackageApi.ListPackages(); + request.setRefreshPackage(packageName); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { final var baseUrl = @@ -291,13 +283,8 @@ private void syncToVersion(PackageStore packageStore, int expectedVersion) { } private void notifyAllNodesToSync(int expectedVersion) { - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expectedVersion)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); + final var request = new PackageApi.ListPackages(); + request.setExpectedVersion(expectedVersion); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); From ba61c1ba723541ac0d40c517229334009423d106 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 14:42:37 -0400 Subject: [PATCH 20/31] More conversion to SolrJ methods --- .../pkg/PackageStoreSchemaPluginsTest.java | 13 +- .../org/apache/solr/pkg/TestPackages.java | 123 ++++++------------ 2 files changed, 43 insertions(+), 93 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 25f0e481cd72..290943655ca7 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -30,8 +30,8 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.FileStoreApi; import org.apache.solr.client.solrj.request.PackageApi; -import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.filestore.FileStoreAPI; @@ -123,13 +123,10 @@ private static String signature(byte[] content) throws Exception { private void uploadPluginJar(String version, Path jarPath) throws Exception { var pluginRequest = - new V2Request.Builder("/cluster/filestore/files/my-plugin/plugin-" + version + ".jar") - .PUT() - .withParams(params("sig", signature(Files.readAllBytes(jarPath)))) - .withPayload(Files.newInputStream(jarPath)) - .forceV2(true) - .build(); - processRequest(client, pluginRequest); + new FileStoreApi.UploadFile( + "/my-plugin/plugin-" + version + ".jar", Files.newInputStream(jarPath)); + pluginRequest.setSig(List.of(signature(Files.readAllBytes(jarPath)))); + pluginRequest.process(client); } private void registerPackage(String version) throws Exception { diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 22850ecbd139..05a8999a0973 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -43,13 +43,13 @@ import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; import org.apache.lucene.util.ResourceLoaderAware; -import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -132,15 +132,9 @@ public void testCoreReloadingPlugin() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "1.0"; - add.files = List.of(FILE1); - V2Request req = - new V2Request.Builder("/cluster/package/mypkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1)); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( @@ -161,9 +155,9 @@ public void testCoreReloadingPlugin() throws Exception { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); verifyComponent( - cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", add.version); + cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", "1.0"); - add.version = "2.0"; + req.setVersion("2.0"); req.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -212,15 +206,9 @@ public void testPluginLoading() throws Exception { EXPR1, "ZOT11arAiPmPZYOHzqodiNnxO9pRyRozWZEBX8XGjU1/HJptFnZK+DI7eXnUtbNaMcbXE2Ze8hh4M/eGyhY8BQ=="); - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "1.0"; - add.files = List.of(FILE1, URP1, EXPR1); - V2Request req = - new V2Request.Builder("/cluster/package/mypkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("mypkg"); + req.setVersion("1.0"); + req.setFiles(List.of(FILE1, URP1, EXPR1)); req.process(cluster.getSolrClient()); @@ -342,8 +330,8 @@ public void testPluginLoading() throws Exception { URP2, "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw=="); // add the version using package API - add.version = "1.1"; - add.files = List.of(FILE2, URP2, EXPR1); + req.setVersion("1.1"); + req.setFiles(List.of(FILE2, URP2, EXPR1)); req.process(cluster.getSolrClient()); verifyComponent( @@ -371,8 +359,8 @@ public void testPluginLoading() throws Exception { FILE3, "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); - add.version = "2.1"; - add.files = List.of(FILE3, URP2, EXPR1); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // now let's verify that the classes are updated @@ -402,11 +390,7 @@ public void testPluginLoading() throws Exception { assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); - new V2Request.Builder("/cluster/package/mypkg/versions/1.0") - .withMethod(SolrRequest.METHOD.DELETE) - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.DeletePackageVersion("mypkg", "1.0").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -418,11 +402,7 @@ public void testPluginLoading() throws Exception { cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1"); // now remove the highest version. So, it will roll back to the next highest one - new V2Request.Builder("/cluster/package/mypkg/versions/2.1") - .withMethod(SolrRequest.METHOD.DELETE) - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.DeletePackageVersion("mypkg", "2.1").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "1.1"); @@ -444,8 +424,8 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { } }.setRequiresCollection(true).process(cluster.getSolrClient()); - add.version = "2.1"; - add.files = List.of(FILE3, URP2, EXPR1); + req.setVersion("2.1"); + req.setFiles(List.of(FILE3, URP2, EXPR1)); req.process(cluster.getSolrClient()); // the collections mypkg is set to use version 1.1 @@ -471,11 +451,7 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { // now, let's force every collection using 'mypkg' to refresh // so that it uses version 2.1 - new V2Request.Builder("/cluster/package/mypkg/refresh") - .withMethod(SolrRequest.METHOD.POST) - .forceV2(true) - .build() - .process(cluster.getSolrClient()); + new PackageApi.RefreshPackage("mypkg").process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -581,15 +557,9 @@ public void testAPI() throws Exception { String FILE2 = "/mypkg/v.0.12/jar_b.jar"; String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "0.12"; - add.files = List.of(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package/test_pkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build(); + PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("test_pkg"); + req.setVersion("0.12"); + req.setFiles(List.of(FILE1, FILE2)); // the files are not yet there. The command should fail with error saying "No such file" expectError(req, cluster.getSolrClient(), errPath, "No such file:"); @@ -597,7 +567,7 @@ public void testAPI() throws Exception { // post the jar file. No signature is sent postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null); - add.files = List.of(FILE1); + req.setFiles(List.of(FILE1)); expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); // now we upload the keys byte[] derFile = readFile("cryptokeys/pub_key512.der"); @@ -610,7 +580,7 @@ public void testAPI() throws Exception { "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); // with correct signature // after uploading the file, let's delete the keys to see if we get proper error message - add.files = List.of(FILE2); + req.setFiles(List.of(FILE2)); /*expectError(req, cluster.getSolrClient(), errPath, "ZooKeeper does not have any public keys");*/ @@ -636,8 +606,8 @@ public void testAPI() throws Exception { "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); // this time we are adding the second version of the package (0.13) - add.version = "0.13"; - add.files = List.of(FILE3); + req.setVersion("0.13"); + req.setFiles(List.of(FILE3)); // this request should succeed req.process(cluster.getSolrClient()); @@ -650,21 +620,14 @@ public void testAPI() throws Exception { Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); // Now we will just delete one version - V2Request deleteReq = - new V2Request.Builder("/cluster/package/test_pkg/versions/0.1") - .forceV2(true) - .withMethod(SolrRequest.METHOD.DELETE) - .build(); + PackageApi.DeletePackageVersion deleteReq = + new PackageApi.DeletePackageVersion("test_pkg", "0.1"); // we are expecting an error expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - new V2Request.Builder( - "/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed - .forceV2(true) - .withMethod(SolrRequest.METHOD.DELETE) - .build() - .process(cluster.getSolrClient()); + // correct version. Should succeed + new PackageApi.DeletePackageVersion("test_pkg", "0.12").process(cluster.getSolrClient()); // Verify with ZK that the data is correct TestDistribFileStore.assertResponseValues( 1, @@ -756,15 +719,10 @@ public void testSchemaPlugins() throws Exception { "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); // upload package v1.0 - AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); - add.version = "1.0"; - add.files = Arrays.asList(FILE1, FILE2); - new V2Request.Builder("/cluster/package/schemapkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build() - .process(cluster.getSolrClient()); + PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); + addReq.setVersion("1.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -796,15 +754,9 @@ public void testSchemaPlugins() throws Exception { coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); // upload package v2.0 - add = new AddPackageVersionRequestBody(); - add.version = "2.0"; - add.files = Arrays.asList(FILE1, FILE2); - new V2Request.Builder("/cluster/package/schemapkg/versions") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(add) - .build() - .process(cluster.getSolrClient()); + addReq.setVersion("2.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -858,7 +810,8 @@ public static void postFileAndWait( cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); } - private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) + private void expectError( + SolrRequest req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException { try { req.process(client); From 670562d9c91bd402fe31a2815fd69fe8c43f0e5c Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 25 Apr 2026 15:02:23 -0400 Subject: [PATCH 21/31] Rework the tests. I am a bit on the fence about detailed docs. --- .../solr/filestore/TestDistribFileStore.java | 15 ++ .../solr/handler/TestContainerPlugin.java | 5 +- .../apache/solr/pkg/ClusterPackageTest.java | 2 +- ...=> ConfigsetPinnedPackageVersionTest.java} | 7 +- .../solr/pkg/PackageSchemaReloadTest.java | 166 ++++++++++++ .../org/apache/solr/pkg/TestPackages.java | 249 ++---------------- 6 files changed, 205 insertions(+), 239 deletions(-) rename solr/core/src/test/org/apache/solr/pkg/{PackageStoreSchemaPluginsTest.java => ConfigsetPinnedPackageVersionTest.java} (94%) create mode 100644 solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 4585696d7805..3d0f2fd74e0c 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -353,6 +353,21 @@ public static UploadToFileStoreResponse postFile( return rsp; } + /** + * Upload a file from the test classpath to the distributed file store under the given path, then + * wait until every node in the cluster reports the expected SHA-512 for that path. + */ + public static void postFileAndWait( + MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { + ByteBuffer fileContent = getFileContent(fname); + @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() + String sha512 = DigestUtils.sha512Hex(fileContent.array()); + + postFile(cluster.getSolrClient(), fileContent, path, sig); + + checkAllNodesForFile(cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); + } + /** * Read and return the contents of the file-like resource * diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 950842555669..4c35d7b94d7d 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -60,7 +60,6 @@ import org.apache.solr.pkg.PackageListeners; import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; -import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -294,12 +293,12 @@ public void testApiFromPackage() throws Exception { byte[] derFile = readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.1.jar.bin", FILE1, "pmrmWCDafdNpYle2rueAGnU2J6NYlcAey9mkZYbqh+5RdYo2Ln+llLF9voyRj+DDivK9GV1XdtKvD9rgCxlD7Q=="); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/containerplugin.v.2.jar.bin", FILE2, diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index 324520161a1b..fef6cdb21be6 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -64,7 +64,7 @@ public void testAddAndDeletePackageVersion() throws Exception { // Upload a key and a signed jar file to the filestore byte[] derFile = TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - TestPackages.postFileAndWait( + TestDistribFileStore.postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", FILE1, diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java similarity index 94% rename from solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java rename to solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java index 290943655ca7..f5854b6e296f 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java @@ -39,7 +39,12 @@ import org.junit.Before; import org.junit.Test; -public class PackageStoreSchemaPluginsTest extends SolrCloudTestCase { +/** + * Verifies that a configset which pins a specific package version (via {@code params.json}) + * resolves classes from that exact version's jar — even when other versions of the same package are + * also registered. + */ +public class ConfigsetPinnedPackageVersionTest extends SolrCloudTestCase { private static final KeyPair KEY_PAIR; diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java new file mode 100644 index 000000000000..424cf4fc2f42 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; +import static org.apache.solr.filestore.TestDistribFileStore.readFile; +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.util.Arrays; +import java.util.Map; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.core.SolrCore; +import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.TestDistribFileStore; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Verifies that bumping a package version that backs a schema plugin causes the affected core's + * schema to reload and pick up classes from the new package's classloader. + */ +public class PackageSchemaReloadTest extends SolrCloudTestCase { + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + System.setProperty("solr.packages.enabled", "true"); + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf1", configset("schema-package")) + .configure(); + } + + @After + @Override + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + super.tearDown(); + } + + @Test + public void testSchemaReloadOnPackageVersionBump() throws Exception { + String COLLECTION_NAME = "testSchemaLoadingColl"; + System.setProperty("managed.schema.mutable", "true"); + + IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core + + String FILE1 = "/schemapkg/schema-plugins.jar"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + postFileAndWait( + cluster, + "runtimecode/schema-plugins.jar.bin", + FILE1, + "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); + + String FILE2 = "/schemapkg/payload-component.jar"; + postFileAndWait( + cluster, + "runtimecode/payload-component.jar.bin", + FILE2, + "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); + + // upload package v1.0 + PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); + addReq.setVersion("1.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[0]:version", + "1.0", + ":result:packages:schemapkg[0]:files[0]", + FILE1)); + + CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + // make note of the schema instance for one of the cores + SolrCore.Provider coreProvider = + cluster.getJettySolrRunners().stream() + .flatMap( + jetty -> + jetty.getCoreContainer().getAllCoreNames().stream() + .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) + .findFirst() + .orElseThrow(); + + coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); + + // upload package v2.0 + addReq.setVersion("2.0"); + addReq.setFiles(Arrays.asList(FILE1, FILE2)); + addReq.process(cluster.getSolrClient()); + + TestDistribFileStore.assertResponseValues( + 10, + () -> + new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.GET) + .build() + .process(cluster.getSolrClient()), + Map.of( + ":result:packages:schemapkg[1]:version", + "2.0", + ":result:packages:schemapkg[1]:files[0]", + FILE1)); + + // even though package version 2.0 uses exactly the same files + // as version 1.0, the core schema should still reload, and + // the core should be associated with a different schema instance + TestDistribFileStore.assertResponseValues( + 10, + () -> { + coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); + return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); + }, + Map.of("schemaReloaded", "yes")); + + // after the reload, the custom field type class now comes from package v2.0 + String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; + + FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); + + FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); + assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); + + assertNotEquals( + "my.pkg.MyTextField classes should be from different classloaders", + fieldTypeV1.getClass(), + fieldTypeV2.getClass()); + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 05a8999a0973..3dbcb0fef0fa 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -17,11 +17,10 @@ package org.apache.solr.pkg; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; -import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; import static org.apache.solr.filestore.TestDistribFileStore.checkAllNodesForFile; +import static org.apache.solr.filestore.TestDistribFileStore.postFileAndWait; import static org.apache.solr.filestore.TestDistribFileStore.readFile; import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; @@ -30,15 +29,12 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.Callable; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; @@ -56,7 +52,6 @@ import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.NavigableObject; import org.apache.solr.common.SolrInputDocument; @@ -64,7 +59,6 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrCore; @@ -74,20 +68,31 @@ import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.QParser; import org.apache.solr.search.QParserPlugin; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.util.LogLevel; import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.zookeeper.data.Stat; import org.eclipse.jetty.client.ContentResponse; import org.eclipse.jetty.client.HttpClient; import org.junit.After; import org.junit.Before; import org.junit.Test; +/** + * Multi-node tests covering how Solr consumes packages at runtime: plugin loading via + * {@link SolrPackageLoader} / {@link PackagePluginHolder}, core reload on package version change, + * cross-node refresh fan-out, and package-file download to newly-joined nodes. + * + *

API-contract tests for the {@link ClusterPackage} JAX-RS endpoints live in {@link + * ClusterPackageTest}. Schema-reload-on-version-bump lives in {@link PackageSchemaReloadTest}. + * Configset-pinned-version behavior lives in {@link ConfigsetPinnedPackageVersionTest}. + * + *

Note on the class name: {@code BasePatternReplaceCharFilterFactory} and {@code + * BaseWhitespaceTokenizerFactory} (defined below) are referenced by fully-qualified name from + * pre-compiled test jars in {@code src/test-files/runtimecode/}. Renaming this class would break + * those binary fixtures. + */ @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.ClusterPackage=DEBUG") public class TestPackages extends SolrCloudTestCase { @@ -549,115 +554,6 @@ private void verifyComponent( ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version)); } - @Test - @SuppressWarnings("unchecked") - public void testAPI() throws Exception { - String errPath = "/msg"; - String FILE1 = "/mypkg/v.0.12/jar_a.jar"; - String FILE2 = "/mypkg/v.0.12/jar_b.jar"; - String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - - PackageApi.AddPackageVersion req = new PackageApi.AddPackageVersion("test_pkg"); - req.setVersion("0.12"); - req.setFiles(List.of(FILE1, FILE2)); - - // the files are not yet there. The command should fail with error saying "No such file" - expectError(req, cluster.getSolrClient(), errPath, "No such file:"); - - // post the jar file. No signature is sent - postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null); - - req.setFiles(List.of(FILE1)); - expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); - // now we upload the keys - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - // and upload the same file with a different name, but it has proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs.jar.bin", - FILE2, - "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - // with correct signature - // after uploading the file, let's delete the keys to see if we get proper error message - req.setFiles(List.of(FILE2)); - /*expectError(req, cluster.getSolrClient(), errPath, - "ZooKeeper does not have any public keys");*/ - - // Now lets' put the keys back - - // this time we have a file with proper signature, public keys are in ZK - // so the add {} command should succeed - req.process(cluster.getSolrClient()); - - // Now verify the data in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.12", ":packages:test_pkg[0]:files[0]", FILE2)); - - // post a new jar with a proper signature - postFileAndWait( - cluster, - "runtimecode/runtimelibs_v2.jar.bin", - FILE3, - "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); - - // this time we are adding the second version of the package (0.13) - req.setVersion("0.13"); - req.setFiles(List.of(FILE3)); - - // this request should succeed - req.process(cluster.getSolrClient()); - // no verify the data (/packages.json) in ZK - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); - - // Now we will just delete one version - PackageApi.DeletePackageVersion deleteReq = - new PackageApi.DeletePackageVersion("test_pkg", "0.1"); - - // we are expecting an error - expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - - // correct version. Should succeed - new PackageApi.DeletePackageVersion("test_pkg", "0.12").process(cluster.getSolrClient()); - // Verify with ZK that the data is correct - TestDistribFileStore.assertResponseValues( - 1, - () -> - NavigableObject.wrap( - Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, null, new Stat()))), - Map.of(":packages:test_pkg[0]:version", "0.13", ":packages:test_pkg[0]:files[0]", FILE3)); - - // So far we have been verifying the details with ZK directly - // use the package read API to verify with each node that it has the correct data - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - String path = jetty.getBaseURLV2().toString() + "/cluster/package?wt=javabin"; - TestDistribFileStore.assertResponseValues( - 10, - new Callable() { - @Override - public NavigableObject call() throws Exception { - HttpClient solrClient = jetty.getSolrClient().getHttpClient(); - byte[] bytes = solrClient.GET(path).getContent(); - return (NavigableObject) new JavaBinCodec().unmarshal(bytes); - } - }, - Map.of( - ":result:packages:test_pkg[0]:version", - "0.13", - ":result:packages:test_pkg[0]:files[0]", - FILE3)); - } - } - public static class C extends RequestHandlerBase implements SolrCoreAware { static boolean informCalled = false; @@ -695,121 +591,6 @@ public QParser createParser( } } - @Test - public void testSchemaPlugins() throws Exception { - String COLLECTION_NAME = "testSchemaLoadingColl"; - System.setProperty("managed.schema.mutable", "true"); - - IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core - - String FILE1 = "/schemapkg/schema-plugins.jar"; - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - postFileAndWait( - cluster, - "runtimecode/schema-plugins.jar.bin", - FILE1, - "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); - - String FILE2 = "/schemapkg/payload-component.jar"; - postFileAndWait( - cluster, - "runtimecode/payload-component.jar.bin", - FILE2, - "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); - - // upload package v1.0 - PackageApi.AddPackageVersion addReq = new PackageApi.AddPackageVersion("schemapkg"); - addReq.setVersion("1.0"); - addReq.setFiles(Arrays.asList(FILE1, FILE2)); - addReq.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[0]:version", - "1.0", - ":result:packages:schemapkg[0]:files[0]", - FILE1)); - - CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); - - // make note of the schema instance for one of the cores - SolrCore.Provider coreProvider = - cluster.getJettySolrRunners().stream() - .flatMap( - jetty -> - jetty.getCoreContainer().getAllCoreNames().stream() - .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) - .findFirst() - .orElseThrow(); - - coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); - - // upload package v2.0 - addReq.setVersion("2.0"); - addReq.setFiles(Arrays.asList(FILE1, FILE2)); - addReq.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[1]:version", - "2.0", - ":result:packages:schemapkg[1]:files[0]", - FILE1)); - - // even though package version 2.0 uses exactly the same files - // as version 1.0, the core schema should still reload, and - // the core should be associated with a different schema instance - TestDistribFileStore.assertResponseValues( - 10, - () -> { - coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); - return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); - }, - Map.of("schemaReloaded", "yes")); - - // after the reload, the custom field type class now comes from package v2.0 - String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; - - FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); - - FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); - - assertNotEquals( - "my.pkg.MyTextField classes should be from different classloaders", - fieldTypeV1.getClass(), - fieldTypeV2.getClass()); - } - - public static void postFileAndWait( - MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { - ByteBuffer fileContent = getFileContent(fname); - @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() - String sha512 = DigestUtils.sha512Hex(fileContent.array()); - - TestDistribFileStore.postFile( - cluster.getSolrClient(), fileContent, path, sig); // has file, but no signature - - TestDistribFileStore.checkAllNodesForFile( - cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); - } - private void expectError( SolrRequest req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException { From 1cfc294c91acc83e2f12e64fa3d499c058c2b5e6 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 28 Aug 2026 14:43:38 -0400 Subject: [PATCH 22/31] Bring this API up to date, following our conventions. Better path /api/cluster/packages and errors. --- ...OLR-18212-migrate-packageapi-to-jax-rs.yml | 2 +- .../solr/client/api/endpoint/PackageApis.java | 2 +- .../solr/packagemanager/PackageUtils.java | 2 +- .../packagemanager/RepositoryManager.java | 2 +- .../org/apache/solr/pkg/ClusterPackage.java | 12 +- .../java/org/apache/solr/pkg/PackageAPI.java | 478 ------------------ .../apache/solr/pkg/SolrPackageLoader.java | 2 +- .../solr/handler/TestContainerPlugin.java | 2 +- .../apache/solr/pkg/ClusterPackageTest.java | 14 +- .../solr/pkg/PackageSchemaReloadTest.java | 4 +- .../org/apache/solr/pkg/TestPackages.java | 6 +- .../pages/package-manager-internals.adoc | 16 +- 12 files changed, 32 insertions(+), 510 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageAPI.java diff --git a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml index 67ac97f93f9c..f7dc02a9ef0e 100644 --- a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml +++ b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml @@ -1,4 +1,4 @@ -title: The package management API now has OpenAPI and SolrJ support. +title: The package management API now has OpenAPI and SolrJ support. As part of this migration, the (experimental v2) API path moved from `/api/cluster/package` to `/api/cluster/packages`, and requests for a non-existent package or package version now return HTTP 404 instead of 400. type: changed authors: - name: Eric Pugh diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java index cf3a50f8caff..d48119112bb0 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -30,7 +30,7 @@ import org.apache.solr.client.api.model.SolrJerseyResponse; /** V2 API definitions for managing Solr packages. */ -@Path("/cluster/package") +@Path("/cluster/packages") public interface PackageApis { @GET diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java index 5a8a0125e61d..1901a6120d41 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -60,7 +60,7 @@ public class PackageUtils { /** Represents a version which denotes the latest version available at the moment. */ public static String LATEST = "latest"; - public static String PACKAGE_PATH = "/api/cluster/package"; + public static String PACKAGE_PATH = "/api/cluster/packages"; public static String CLUSTER_PLUGINS_PATH = "/api/cluster/plugin"; public static String REPOSITORIES_ZK_PATH = "/repositories.json"; public static String CLUSTERPROPS_PATH = "/api/cluster/zk/data/clusterprops.json"; diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index f212710b27db..7b3de8e05323 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -154,7 +154,7 @@ private String getRepositoriesJson(SolrZkClient zkClient) /** * Install a given package and version from the available repositories to Solr. The various steps * for doing so are, briefly, a) find upload a manifest to package store, b) download the - * artifacts and upload to package store, c) call {@link PackageManager} to register the package. + * artifacts and upload to package store, c) call the Package API to register the package. */ private boolean installPackage(String packageName, String version) throws SolrException { SolrPackageInstance existingPlugin = packageManager.getPackageInstance(packageName, version); diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index e7bdc0979374..dc7fd09b34b0 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -45,7 +45,7 @@ import org.slf4j.LoggerFactory; /** - * JAX-RS implementation of the package management API ({@code /api/cluster/package}). + * JAX-RS implementation of the package management API ({@code /api/cluster/packages}). * * @see PackageApis */ @@ -196,7 +196,7 @@ public SolrJerseyResponse deletePackageVersion(String packageName, String versio List versions = packages.packages.get(packageName); if (versions == null || versions.isEmpty()) { throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); } int idxToRemove = -1; for (int i = 0; i < versions.size(); i++) { @@ -207,7 +207,7 @@ public SolrJerseyResponse deletePackageVersion(String packageName, String versio } if (idxToRemove == -1) { throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); + SolrException.ErrorCode.NOT_FOUND, "No such version: " + version); } versions.remove(idxToRemove); packages.znodeVersion = stat.getVersion() + 1; @@ -229,7 +229,7 @@ public SolrJerseyResponse refreshPackage(String packageName) { SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); if (pkg == null) { throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); } // first refresh on the current node packageStore.packageLoader.notifyListeners(packageName); @@ -242,7 +242,7 @@ public SolrJerseyResponse refreshPackage(String packageName) { coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); + request.processWithBaseUrl(solrClient, baseUrl, null); } catch (SolrServerException | IOException e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -290,7 +290,7 @@ private void notifyAllNodesToSync(int expectedVersion) { var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); + request.processWithBaseUrl(solrClient, baseUrl, null); } catch (SolrServerException | IOException e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java deleted file mode 100644 index 0fd1e29dc4a9..000000000000 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.pkg; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; -import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import org.apache.solr.api.Command; -import org.apache.solr.api.EndPoint; -import org.apache.solr.api.PayloadObj; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; -import org.apache.solr.client.solrj.response.JavaBinResponseParser; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZooKeeperException; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.ReflectMapWriter; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.SolrJacksonAnnotationInspector; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** This implements the public end points (/api/cluster/package) of package API. */ -public class PackageAPI { - public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; - - final CoreContainer coreContainer; - private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - private final SolrPackageLoader packageLoader; - Packages pkgs; - - public final Edit editAPI = new Edit(); - public final Read readAPI = new Read(); - - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { - this.coreContainer = coreContainer; - this.packageLoader = loader; - pkgs = new Packages(); - SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - pkgs = new Packages(); - // ignore - } - try { - registerListener(zkClient); - } catch (KeeperException | InterruptedException e) { - SolrZkClient.checkInterrupted(e); - } - } - - private void registerListener(SolrZkClient zkClient) - throws KeeperException, InterruptedException { - zkClient.exists( - SOLR_PKGS_PATH, - new Watcher() { - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher - if (Event.EventType.None.equals(event.getType())) { - return; - } - synchronized (this) { - log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); - // remake watch - final Watcher thisWatch = this; - refreshPackages(thisWatch); - } - } - }); - } - - public void refreshPackages(Watcher watcher) { - final Stat stat = new Stat(); - try { - final byte[] data = - coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); - pkgs = readPkgsFromZk(data, stat); - packageLoader.refreshPackageConf(); - } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); - } catch (KeeperException e) { - log.error("A ZK error has occurred", e); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); - } - } - - private Packages readPkgsFromZk(byte[] data, Stat stat) - throws KeeperException, InterruptedException { - - if (data == null || stat == null) { - stat = new Stat(); - data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); - } - Packages packages = null; - if (data == null || data.length == 0) { - packages = new Packages(); - } else { - try { - packages = mapper.readValue(data, Packages.class); - packages.znodeVersion = stat.getVersion(); - } catch (IOException e) { - // invalid data in packages - // TODO handle properly; - return new Packages(); - } - } - return packages; - } - - public static class Packages implements ReflectMapWriter { - @JsonProperty public int znodeVersion = -1; - - @JsonProperty public Map> packages = new LinkedHashMap<>(); - - public Packages copy() { - Packages p = new Packages(); - p.znodeVersion = this.znodeVersion; - p.packages = new LinkedHashMap<>(); - packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); - return p; - } - } - - public static class PkgVersion implements ReflectMapWriter { - - @JsonProperty("package") - public String pkg; - - @JsonProperty public String version; - - @JsonProperty public List files; - - @JsonProperty public String manifest; - - @JsonProperty public String manifestSHA512; - - public PkgVersion() {} - - public PkgVersion(PackagePayload.AddVersion addVersion) { - this.pkg = addVersion.pkg; - this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); - this.manifest = addVersion.manifest; - this.manifestSHA512 = addVersion.manifestSHA512; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof PkgVersion that) { - return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(version); - } - - @Override - public String toString() { - try { - return Utils.writeJson(this, new StringWriter(), false).toString(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public PkgVersion copy() { - PkgVersion result = new PkgVersion(); - result.pkg = this.pkg; - result.version = this.version; - result.files = this.files; - result.manifest = this.manifest; - result.manifestSHA512 = this.manifestSHA512; - return result; - } - } - - @EndPoint( - method = SolrRequest.METHOD.POST, - path = "/cluster/package", - permission = PACKAGE_EDIT_PERM) - public class Edit { - - @Command(name = "refresh") - public void refresh(PayloadObj payload) { - String p = payload.get(); - if (p == null) { - payload.addError("Package null"); - return; - } - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(p); - if (pkg == null) { - payload.addError("No such package: " + p); - return; - } - // first refresh my own - packageLoader.notifyListeners(p); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", p); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - request.processWithBaseUrl(solrClient, baseUrl, null); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } - } - - @Command(name = "add") - public void add(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.AddVersion add = payload.get(); - if (add.files.isEmpty()) { - payload.addError("No files specified"); - return; - } - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); - if (payload.hasError()) return; - Packages[] finalState = new Packages[1]; - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = - bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new Packages(); - } - List list = - packages.packages.computeIfAbsent(add.pkg, o -> new ArrayList<>()); - for (PkgVersion version : list) { - if (Objects.equals(version.version, add.version)) { - payload.addError("Version '" + add.version + "' exists already"); - return null; - } - } - list.add(new PkgVersion(add)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - handleZkErr(e); - } - if (finalState[0] != null) { - // succeeded in updating - pkgs = finalState[0]; - notifyAllNodesToSync(pkgs.znodeVersion); - packageLoader.refreshPackageConf(); - } - } - - @Command(name = "delete") - public void del(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.DelVersion delVersion = payload.get(); - try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new Packages(); - } - - List versions = packages.packages.get(delVersion.pkg); - if (versions == null || versions.isEmpty()) { - payload.addError("No such package: " + delVersion.pkg); - return null; // no change - } - int idxToremove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, delVersion.version)) { - idxToremove = i; - break; - } - } - if (idxToremove == -1) { - payload.addError("No such version: " + delVersion.version); - return null; - } - versions.remove(idxToremove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - - public boolean isEnabled() { - return enablePackages; - } - - private boolean checkEnabled(CommandOperation payload) { - if (!enablePackages) { - payload.addError(ERR_MSG); - return false; - } - return true; - } - - public class Read { - @EndPoint( - method = SolrRequest.METHOD.GET, - path = {"/cluster/package/", "/cluster/package/{name}"}, - permission = PACKAGE_READ_PERM) - public void get(SolrQueryRequest req, SolrQueryResponse rsp) { - String refresh = req.getParams().get("refreshPackage"); - if (refresh != null) { - packageLoader.notifyListeners(refresh); - return; - } - - int expectedVersion = req.getParams().getInt("expectedVersion", -1); - if (expectedVersion != -1) { - syncToVersion(expectedVersion); - } - String name = req.getPathTemplateValues().get("name"); - if (name == null) { - rsp.add("result", pkgs); - } else { - rsp.add("result", Map.of(name, pkgs.packages.get(name))); - } - } - - private void syncToVersion(int expectedVersion) { - int origVersion = pkgs.znodeVersion; - for (int i = 0; i < 10; i++) { - log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion); - if (pkgs.znodeVersion >= expectedVersion) { - if (origVersion < pkgs.znodeVersion) { - packageLoader.refreshPackageConf(); - } - return; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - } - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - } - - void notifyAllNodesToSync(int expected) { - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expected)); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - request.processWithBaseUrl(solrClient, baseUrl, null); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " + liveNode + " to sync expected package version: " + expected, - e); - } - } - } - - public void handleZkErr(Exception e) { - log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); - } - - public boolean isJarInuse(String path) { - Packages pkg = null; - try { - pkg = readPkgsFromZk(null, null); - } catch (KeeperException.NoNodeException nne) { - return false; - } catch (InterruptedException | KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - for (List vers : pkg.packages.values()) { - for (PkgVersion ver : vers) { - if (ver.files.contains(path)) { - return true; - } - } - } - return false; - } -} diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index 6085f3e9ed36..b3e9229181f6 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -106,7 +106,7 @@ public void refreshPackageConf() { } } coreContainer.forEachLoadedCore(core -> core.getPackageListeners().packagesUpdated(updated)); - myCopy = packageAPI.pkgs; + myCopy = packageStore.pkgs; } public Map> getModified( diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 9d2967305c27..1aee5a42e8d1 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -312,7 +312,7 @@ public void testApiFromPackage() throws Exception { listener.waitFor(cluster.getJettySolrRunners().size())); waitForAllNodesToSync( - "/cluster/package", + "/cluster/packages", Map.of( ":result:packages:mypkg[0]:version", "1.0", diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index fef6cdb21be6..7f6b5230bb17 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -52,8 +52,8 @@ public void testListPackagesReturnsResult() throws Exception { new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) .build()) { PackagesResponse response = new PackageApi.ListPackages().process(client); - assertNotNull("Expected non-null response from GET /cluster/package", response); - assertNotNull("Expected 'result' field in GET /cluster/package response", response.result); + assertNotNull("Expected non-null response from GET /cluster/packages", response); + assertNotNull("Expected 'result' field in GET /cluster/packages response", response.result); } } @@ -73,13 +73,13 @@ public void testAddAndDeletePackageVersion() throws Exception { try (HttpJettySolrClient client = new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) .build()) { - // Add a package version via POST /cluster/package/{name}/versions + // Add a package version via POST /cluster/packages/{name}/versions PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); addRequest.setVersion("1.0"); addRequest.setFiles(List.of(FILE1)); addRequest.process(client); - // Verify the package was added via GET /cluster/package + // Verify the package was added via GET /cluster/packages PackagesResponse listResponse = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null list response", listResponse); assertNotNull("Expected non-null result", listResponse.result); @@ -89,7 +89,7 @@ public void testAddAndDeletePackageVersion() throws Exception { "Expected at least one version", listResponse.result.packages.get("pkgapitestpkg").isEmpty()); - // Verify GET /cluster/package/{name} returns only this package + // Verify GET /cluster/packages/{name} returns only this package PackagesResponse getByNameResponse = new PackageApi.GetPackage("pkgapitestpkg").process(client); assertNotNull("Expected non-null get-by-name response", getByNameResponse); @@ -98,7 +98,7 @@ public void testAddAndDeletePackageVersion() throws Exception { "Expected pkgapitestpkg in get-by-name response", getByNameResponse.result.packages.get("pkgapitestpkg")); - // Delete the package version via DELETE /cluster/package/{name}/versions/{version} + // Delete the package version via DELETE /cluster/packages/{name}/versions/{version} new PackageApi.DeletePackageVersion("pkgapitestpkg", "1.0").process(client); // Verify it's deleted @@ -140,7 +140,7 @@ public void testRefreshNonExistentPackage() { expectThrows( RemoteSolrException.class, () -> new PackageApi.RefreshPackage("nonexistentpkg_test").process(client)); - assertEquals("Expected 400 for non-existent package", 400, ex.code()); + assertEquals("Expected 404 for non-existent package", 404, ex.code()); assertTrue( "Expected error message to mention the package: " + ex.getMessage(), ex.getMessage().contains("No such package")); diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java index 424cf4fc2f42..21157d3759e6 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java @@ -94,7 +94,7 @@ public void testSchemaReloadOnPackageVersionBump() throws Exception { TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), @@ -128,7 +128,7 @@ public void testSchemaReloadOnPackageVersionBump() throws Exception { TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 3dbcb0fef0fa..85210bd32dc6 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -145,7 +145,7 @@ public void testCoreReloadingPlugin() throws Exception { TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), @@ -167,7 +167,7 @@ public void testCoreReloadingPlugin() throws Exception { TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), @@ -224,7 +224,7 @@ public void testPluginLoading() throws Exception { TestDistribFileStore.assertResponseValues( 10, () -> - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/packages") .withMethod(SolrRequest.METHOD.GET) .build() .process(cluster.getSolrClient()), diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc index 4814ebb929a4..ff6b6df915e4 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc @@ -178,11 +178,11 @@ For example: == API Endpoints -* `GET /api/cluster/package` Get the list of all packages -* `GET /api/cluster/package/{name}` Get the details of a specific package -* `POST /api/cluster/package/{name}/versions` Add a version of a package -* `DELETE /api/cluster/package/{name}/versions/{version}` Delete a specific version of a package -* `POST /api/cluster/package/{name}/refresh` Refresh a package on all nodes in the cluster +* `GET /api/cluster/packages` Get the list of all packages +* `GET /api/cluster/packages/{name}` Get the details of a specific package +* `POST /api/cluster/packages/{name}/versions` Add a version of a package +* `DELETE /api/cluster/packages/{name}/versions/{version}` Delete a specific version of a package +* `POST /api/cluster/packages/{name}/refresh` Refresh a package on all nodes in the cluster === How to Upgrade? @@ -245,7 +245,7 @@ The plugins loaded from packages cannot depend on core level classes. + [source,bash] ---- -curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ +curl -X POST http://localhost:8983/api/cluster/packages/mypkg/versions \ -H 'Content-type:application/json' \ -d '{"version":"1.0","files":["/mypkg/1.0/myplugins.jar"]}' ---- @@ -254,7 +254,7 @@ curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + [source,bash] ---- -curl http://localhost:8983/api/cluster/package?omitHeader=true +curl http://localhost:8983/api/cluster/packages?omitHeader=true ---- + [source,json] @@ -349,7 +349,7 @@ curl http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=t + [source,bash] ---- -curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ +curl -X POST http://localhost:8983/api/cluster/packages/mypkg/versions \ -H 'Content-type:application/json' \ -d '{"version":"2.0","files":["/mypkg/2.0/myplugins.jar"]}' ---- From b51a15515bd84ca78ab8aa8fedd5bc96ee068d9c Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 28 Aug 2026 14:55:15 -0400 Subject: [PATCH 23/31] add jira --- .../unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml index f7dc02a9ef0e..69ad470fd4ee 100644 --- a/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml +++ b/changelog/unreleased/SOLR-18212-migrate-packageapi-to-jax-rs.yml @@ -5,3 +5,5 @@ authors: links: - name: PR#4178 url: https://github.com/apache/solr/pull/4178 +- name: SOLR-18212 + url: https://issues.apache.org/jira/browse/SOLR-18212 From 742b0ba0e8f41cafd52eb2e383ba895b6d28c0cf Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 28 Aug 2026 15:39:22 -0400 Subject: [PATCH 24/31] tidy --- solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index dc7fd09b34b0..078aee3945ca 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -228,8 +228,7 @@ public SolrJerseyResponse refreshPackage(String packageName) { SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); if (pkg == null) { - throw new SolrException( - SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); } // first refresh on the current node packageStore.packageLoader.notifyListeners(packageName); From 47fd06e321d4bc1341370fd1388260dce2234427 Mon Sep 17 00:00:00 2001 From: jaykay12 Date: Tue, 20 Oct 2026 03:06:48 +0530 Subject: [PATCH 25/31] precommit works now --- .../cli/packagemanager/RepositoryManager.java | 2 - .../solr/handler/SolrConfigHandler.java | 2 - .../solr/handler/admin/api/GetConfig.java | 4 +- .../java/org/apache/solr/pkg/PackageAPI.java | 0 .../org/apache/solr/pkg/TestPackages.java | 132 ------------------ 5 files changed, 2 insertions(+), 138 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageAPI.java diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java index 87e97292ca9c..8ff3fd5b15ec 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/RepositoryManager.java @@ -50,8 +50,6 @@ import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; -import org.apache.solr.packagemanager.SolrPackage.Artifact; -import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index 600c20131524..08614a0a4bc6 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -90,8 +90,6 @@ import org.apache.solr.handler.admin.api.ModifyConfigComponentAPI; import org.apache.solr.handler.admin.api.ModifyParamSetAPI; import org.apache.solr.handler.api.V2ApiUtils; -import org.apache.solr.pkg.PackageListeners; -import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java index 1bc7db362bd9..4068c4fc1e1a 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetConfig.java @@ -46,8 +46,8 @@ import org.apache.solr.handler.SolrConfigHandler; import org.apache.solr.jersey.NullKeyTolerantMap; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.util.SolrPluginUtils; @@ -193,7 +193,7 @@ public static void filterByComponentName( List listeners = req.getCore().getPackageListeners().getListeners(); for (PackageListeners.Listener listener : listeners) { - Map infos = listener.packageDetails(); + Map infos = listener.packageDetails(); if (infos == null || infos.isEmpty()) continue; infos.forEach( (s, mapWriter) -> { diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index b8e66d6c7fd7..85210bd32dc6 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -593,138 +593,6 @@ public QParser createParser( private void expectError( SolrRequest req, SolrClient client, String errPath, String expectErrorMsg) - @Test - @SuppressWarnings("ReferenceEquality") - public void testSchemaPlugins() throws Exception { - String COLLECTION_NAME = "testSchemaLoadingColl"; - System.setProperty("managed.schema.mutable", "true"); - - IndexSchema[] schemas = new IndexSchema[2]; // tracks schemas for a selected core - - String FILE1 = "/schemapkg/schema-plugins.jar"; - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); - postFileAndWait( - cluster, - "runtimecode/schema-plugins.jar.bin", - FILE1, - "U+AdO/jgY3DtMpeFRGoTQk72iA5g/qjPvdQYPGBaXB5+ggcTZk4FoIWiueB0bwGJ8Mg3V/elxOqEbD2JR8R0tA=="); - - String FILE2 = "/schemapkg/payload-component.jar"; - postFileAndWait( - cluster, - "runtimecode/payload-component.jar.bin", - FILE2, - "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); - - // upload package v1.0 - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "schemapkg"; - add.files = Arrays.asList(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - req.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[0]:version", - "1.0", - ":result:packages:schemapkg[0]:files[0]", - FILE1)); - - CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1", 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); - - // make note of the schema instance for one of the cores - SolrCore.Provider coreProvider = - cluster.getJettySolrRunners().stream() - .flatMap( - jetty -> - jetty.getCoreContainer().getAllCoreNames().stream() - .map(name -> new SolrCore.Provider(jetty.getCoreContainer(), name, null))) - .findFirst() - .orElseThrow(); - - coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); - - // upload package v2.0 - add = new PackagePayload.AddVersion(); - add.version = "2.0"; - add.pkg = "schemapkg"; - add.files = Arrays.asList(FILE1, FILE2); - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Map.of("add", add)) - .build(); - req.process(cluster.getSolrClient()); - - TestDistribFileStore.assertResponseValues( - 10, - () -> - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.GET) - .build() - .process(cluster.getSolrClient()), - Map.of( - ":result:packages:schemapkg[1]:version", - "2.0", - ":result:packages:schemapkg[1]:files[0]", - FILE1)); - - // even though package version 2.0 uses exactly the same files - // as version 1.0, the core schema should still reload, and - // the core should be associated with a different schema instance - TestDistribFileStore.assertResponseValues( - 10, - () -> { - coreProvider.withCore(core -> schemas[1] = core.getLatestSchema()); - return params("schemaReloaded", (schemas[0] != schemas[1]) ? "yes" : "no"); - }, - Map.of("schemaReloaded", "yes")); - - // after the reload, the custom field type class now comes from package v2.0 - String fieldTypeName = "myNewTextFieldWithAnalyzerClass"; - - FieldType fieldTypeV1 = schemas[0].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV1.getClass().getCanonicalName()); - - FieldType fieldTypeV2 = schemas[1].getFieldTypeByName(fieldTypeName); - assertEquals("my.pkg.MyTextField", fieldTypeV2.getClass().getCanonicalName()); - - assertNotEquals( - "my.pkg.MyTextField classes should be from different classloaders", - fieldTypeV1.getClass(), - fieldTypeV2.getClass()); - } - - public static void postFileAndWait( - MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { - ByteBuffer fileContent = getFileContent(fname); - @SuppressWarnings("ByteBufferBackingArray") // this is the result of a call to wrap() - String sha512 = DigestUtils.sha512Hex(fileContent.array()); - - TestDistribFileStore.postFile( - cluster.getSolrClient(), fileContent, path, sig); // has file, but no signature - - TestDistribFileStore.checkAllNodesForFile( - cluster, path, Map.of(":files:" + path + ":sha512", sha512), false); - } - - private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException { try { req.process(client); From 2445ef1bb80ab14d24e33de657e4dd43f9570f34 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 28 Oct 2026 09:42:17 -0400 Subject: [PATCH 26/31] fix the pathing and make GET actually not mutate state! --- .../solr/client/api/endpoint/PackageApis.java | 26 +++++++++++---- .../org/apache/solr/pkg/ClusterPackage.java | 33 ++++++++++++------- .../apache/solr/pkg/ClusterPackageTest.java | 25 ++++++++++---- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java index d48119112bb0..948b0c0b824b 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -38,11 +38,6 @@ public interface PackageApis { summary = "List all packages registered in this Solr cluster.", tags = {"package"}) PackagesResponse listPackages( - @Parameter( - description = - "Name of a package to refresh on the receiving node only (reloads its listeners from ZooKeeper). Used internally for inter-node fan-out by the refresh endpoint; not normally invoked directly.") - @QueryParam("refreshPackage") - String refreshPackage, @Parameter( description = "If provided, the node waits until its package data matches this ZooKeeper version.") @@ -97,5 +92,24 @@ SolrJerseyResponse deletePackageVersion( SolrJerseyResponse refreshPackage( @Parameter(description = "The name of the package to refresh.", required = true) @PathParam("packageName") - String packageName); + String packageName, + @Parameter( + description = + "Indicates whether the refresh should only be done on the receiving node. For internal use only.") + @QueryParam("localOnly") + Boolean localOnly); + + @POST + @Path("/sync") + @Operation( + summary = + "Block until the receiving node's package data matches the given ZooKeeper version. " + + "Used internally for inter-node fan-out after a package change; not normally invoked directly.", + tags = {"package"}) + SolrJerseyResponse syncPackages( + @Parameter( + description = "The ZooKeeper version that the receiving node should catch up to.", + required = true) + @QueryParam("expectedVersion") + Integer expectedVersion); } diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 078aee3945ca..989e832d20f4 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -64,14 +64,9 @@ public ClusterPackage(CoreContainer coreContainer) { @Override @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { + public PackagesResponse listPackages(Integer expectedVersion) { PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - if (refreshPackage != null) { - packageStore.packageLoader.notifyListeners(refreshPackage); - return new PackagesResponse(); - } - if (expectedVersion != null) { syncToVersion(packageStore, expectedVersion); } @@ -222,10 +217,15 @@ public SolrJerseyResponse deletePackageVersion(String packageName, String versio @Override @PermissionName(PACKAGE_EDIT_PERM) - public SolrJerseyResponse refreshPackage(String packageName) { + public SolrJerseyResponse refreshPackage(String packageName, Boolean localOnly) { final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + if (Boolean.TRUE.equals(localOnly)) { + packageStore.packageLoader.notifyListeners(packageName); + return response; + } + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); if (pkg == null) { throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); @@ -233,8 +233,8 @@ public SolrJerseyResponse refreshPackage(String packageName) { // first refresh on the current node packageStore.packageLoader.notifyListeners(packageName); - final var request = new PackageApi.ListPackages(); - request.setRefreshPackage(packageName); + final var request = new PackageApi.RefreshPackage(packageName); + request.setLocalOnly(true); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { final var baseUrl = @@ -253,6 +253,18 @@ public SolrJerseyResponse refreshPackage(String packageName) { return response; } + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse syncPackages(Integer expectedVersion) { + if (expectedVersion == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "expectedVersion is required"); + } + final var response = new SolrJerseyResponse(); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + syncToVersion(packageStore, expectedVersion); + return response; + } + private void syncToVersion(PackageStore packageStore, int expectedVersion) { int origVersion = packageStore.pkgs.znodeVersion; for (int i = 0; i < SYNC_MAX_RETRIES; i++) { @@ -282,8 +294,7 @@ private void syncToVersion(PackageStore packageStore, int expectedVersion) { } private void notifyAllNodesToSync(int expectedVersion) { - final var request = new PackageApi.ListPackages(); - request.setExpectedVersion(expectedVersion); + final var request = new PackageApi.SyncPackages(expectedVersion); for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index 7f6b5230bb17..b7daf76e5043 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.PackageApi; @@ -148,15 +149,27 @@ public void testRefreshNonExistentPackage() { } @Test - public void testListPackagesAcceptsRefreshPackageQueryParam() throws Exception { + public void testRefreshPackageLocalOnlyAcceptsUnknownPackage() throws Exception { try (HttpJettySolrClient client = new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) .build()) { - PackageApi.ListPackages req = new PackageApi.ListPackages(); - req.setRefreshPackage("nonexistentpkg_refresh"); - // The inter-node refresh signal calls notifyListeners on this node and returns OK even when - // the package doesn't exist; main's behavior is identical. - PackagesResponse response = req.process(client); + // localOnly=true is the internal inter-node refresh signal: it calls notifyListeners on this + // node only and returns OK even when the package doesn't exist. + PackageApi.RefreshPackage req = new PackageApi.RefreshPackage("nonexistentpkg_refresh"); + req.setLocalOnly(true); + SolrJerseyResponse response = req.process(client); + assertNotNull(response); + } + } + + @Test + public void testSyncPackagesCommandRequiresExpectedVersion() throws Exception { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + // expectedVersion at-or-below current is a no-op in syncToVersion and must return promptly. + PackageApi.SyncPackages req = new PackageApi.SyncPackages(0); + SolrJerseyResponse response = req.process(client); assertNotNull(response); } } From 901d90ec893805743f13d7f1576b347fe76c46e0 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 28 Oct 2026 10:47:46 -0400 Subject: [PATCH 27/31] SOLR-18212: fix GET side effects, 404 handling, and response naming in the Package v2 API - Move the inter-node refresh/sync fan-out off GET /cluster/packages (which was mutating state via refreshPackage/expectedVersion query params) onto POST endpoints: a localOnly flag on the existing /refresh action (mirroring ClusterFileStoreApis.deleteFile's localDelete), and a new /sync action. - getPackage() now returns 404 for an unknown package, matching deletePackageVersion/refreshPackage's existing NOT_FOUND behavior. - Split the single-package GET response into its own flat shape (GetPackageResponse) instead of reusing the list endpoint's map-shaped response filtered to one entry, and renamed both response classes to match the ListResponse / GetResponse convention used elsewhere (aliases, cluster properties, schema fields). - Extracted the duplicated node-broadcast loop in ClusterPackage into a shared helper. - Documented the List/Get response-naming convention in dev-docs/apis-v2-conventions.adoc. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019kzScQszwRcG4S7kA5Pa8b --- dev-docs/apis-v2-conventions.adoc | 10 +++ .../solr/client/api/endpoint/PackageApis.java | 7 +- .../client/api/model/GetPackageResponse.java | 44 ++++++++++++ ...esponse.java => ListPackagesResponse.java} | 2 +- .../org/apache/solr/pkg/ClusterPackage.java | 70 +++++++++---------- .../apache/solr/pkg/ClusterPackageTest.java | 45 ++++++++---- 6 files changed, 126 insertions(+), 52 deletions(-) create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/GetPackageResponse.java rename solr/api/src/java/org/apache/solr/client/api/model/{PackagesResponse.java => ListPackagesResponse.java} (97%) diff --git a/dev-docs/apis-v2-conventions.adoc b/dev-docs/apis-v2-conventions.adoc index 945c94afe725..848f460cedcf 100644 --- a/dev-docs/apis-v2-conventions.adoc +++ b/dev-docs/apis-v2-conventions.adoc @@ -156,6 +156,16 @@ Beyond that base, pick the narrowest existing response type that fits before wri If an API returns specific, known additional data beyond these bases (e.g. timing information, computed ranges, a resource's status fields), extend the appropriate base with typed `@JsonProperty` fields rather than reaching for `FlexibleSolrJerseyResponse` - see `SplitShardResponse.timing`, `SplitCoreResponse.ranges`, or `CollectionStatusResponse` for examples. Don't settle for a bare `SolrJerseyResponse` if the underlying operation actually produces more than a bare success/error - a v2 JSON caller has no other way to get that data, since (unlike v1) nothing else in the response pipeline will surface it (see the async caveat below for why this matters in practice). +=== Listing vs. Single-Resource Responses + +When an API exposes both a "list all" endpoint and a "get one by name" endpoint for the same kind of resource, give each its own response type rather than reusing the list's shape for a single item: + +* `ListResponse` - the collection endpoint's response. Its data is a map or list keyed/indexed across every matching resource (e.g. `ListAliasesResponse.aliases: Map`). +* `GetResponse` (or `GetByNameResponse`) - the single-resource endpoint's response. Its data is a flat object describing just that one resource, with no collection wrapper (e.g. `GetAliasByNameResponse.collections`, `GetAliasByNameResponse.properties`). + +Don't have the single-resource endpoint return the list type filtered down to one entry (e.g. a `Map` with one key) - that forces callers to re-extract by name something they already specified in the request, and conveys no information once only one key is possible. +See `ListAliasesResponse`/`GetAliasByNameResponse`, `ListClusterPropertiesResponse`/`GetClusterPropertyResponse`, and `ListPackagesResponse`/`GetPackageResponse` for examples of the pattern. + == Async Task Handling Solr has two distinct, non-interchangeable mechanisms for handling the `async` request parameter, depending on which base class the API extends: diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java index 948b0c0b824b..e126eb299d71 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -26,7 +26,8 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.QueryParam; import org.apache.solr.client.api.model.AddPackageVersionRequestBody; -import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.GetPackageResponse; +import org.apache.solr.client.api.model.ListPackagesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; /** V2 API definitions for managing Solr packages. */ @@ -37,7 +38,7 @@ public interface PackageApis { @Operation( summary = "List all packages registered in this Solr cluster.", tags = {"package"}) - PackagesResponse listPackages( + ListPackagesResponse listPackages( @Parameter( description = "If provided, the node waits until its package data matches this ZooKeeper version.") @@ -49,7 +50,7 @@ PackagesResponse listPackages( @Operation( summary = "Get information about a specific package in this Solr cluster.", tags = {"package"}) - PackagesResponse getPackage( + GetPackageResponse getPackage( @Parameter(description = "The name of the package.", required = true) @PathParam("packageName") String packageName, diff --git a/solr/api/src/java/org/apache/solr/client/api/model/GetPackageResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/GetPackageResponse.java new file mode 100644 index 000000000000..7e3d590012ec --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/GetPackageResponse.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; + +/** Response for the get-a-specific-package API. */ +public class GetPackageResponse extends SolrJerseyResponse { + + @JsonProperty("result") + @Schema(description = "The requested package's data.") + public PackageInfo result; + + /** Data for a single package, returned by the get-package API. */ + public static class PackageInfo { + @JsonProperty("znodeVersion") + @Schema(description = "The ZooKeeper version of the packages.json node.") + public int znodeVersion; + + @JsonProperty("name") + @Schema(description = "The package name.") + public String name; + + @JsonProperty("versions") + @Schema(description = "The versions registered for this package.") + public List versions; + } +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ListPackagesResponse.java similarity index 97% rename from solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java rename to solr/api/src/java/org/apache/solr/client/api/model/ListPackagesResponse.java index 0dce425c807b..5788b57967fb 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/ListPackagesResponse.java @@ -22,7 +22,7 @@ import java.util.Map; /** Response for the package listing API. */ -public class PackagesResponse extends SolrJerseyResponse { +public class ListPackagesResponse extends SolrJerseyResponse { @JsonProperty("result") @Schema(description = "The package data including znode version and package definitions.") diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 989e832d20f4..3bc8f579ca4e 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -27,12 +27,15 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.solr.api.JerseyResource; import org.apache.solr.client.api.endpoint.PackageApis; import org.apache.solr.client.api.model.AddPackageVersionRequestBody; -import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.GetPackageResponse; +import org.apache.solr.client.api.model.ListPackagesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.common.SolrException; @@ -64,34 +67,38 @@ public ClusterPackage(CoreContainer coreContainer) { @Override @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse listPackages(Integer expectedVersion) { + public ListPackagesResponse listPackages(Integer expectedVersion) { PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); if (expectedVersion != null) { syncToVersion(packageStore, expectedVersion); } - final var response = new PackagesResponse(); + final var response = new ListPackagesResponse(); response.result = toPackageData(packageStore.pkgs); return response; } @Override @PermissionName(PACKAGE_READ_PERM) - public PackagesResponse getPackage(String packageName, Integer expectedVersion) { + public GetPackageResponse getPackage(String packageName, Integer expectedVersion) { PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); if (expectedVersion != null) { syncToVersion(packageStore, expectedVersion); } - final var response = new PackagesResponse(); - response.result = toPackageData(packageStore.pkgs); - // Filter to only the requested package; if absent, return an empty packages map. - if (response.result != null && response.result.packages != null) { - final var pkgVersions = response.result.packages.get(packageName); - response.result.packages = pkgVersions == null ? Map.of() : Map.of(packageName, pkgVersions); + final List pkgVersions = packageStore.pkgs.packages.get(packageName); + if (pkgVersions == null || pkgVersions.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such package: " + packageName); } + + final var response = new GetPackageResponse(); + response.result = new GetPackageResponse.PackageInfo(); + response.result.znodeVersion = packageStore.pkgs.znodeVersion; + response.result.name = packageName; + response.result.versions = + pkgVersions.stream().map(ClusterPackage::toPkgVersionResponse).collect(Collectors.toList()); return response; } @@ -235,20 +242,7 @@ public SolrJerseyResponse refreshPackage(String packageName, Boolean localOnly) final var request = new PackageApi.RefreshPackage(packageName); request.setLocalOnly(true); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - request.processWithBaseUrl(solrClient, baseUrl, null); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); - } - } + broadcastToLiveNodes(request, liveNode -> "Failed to refresh package on node: " + liveNode); return response; } @@ -295,29 +289,35 @@ private void syncToVersion(PackageStore packageStore, int expectedVersion) { private void notifyAllNodesToSync(int expectedVersion) { final var request = new PackageApi.SyncPackages(expectedVersion); + broadcastToLiveNodes( + request, + liveNode -> + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion); + } + private void broadcastToLiveNodes( + SolrRequest request, Function errorMessage) { for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - var baseUrl = coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { var solrClient = coreContainer.getDefaultHttpSolrClient(); request.processWithBaseUrl(solrClient, baseUrl, null); } catch (SolrServerException | IOException e) { throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " - + liveNode - + " to sync expected package version: " - + expectedVersion, - e); + SolrException.ErrorCode.SERVER_ERROR, errorMessage.apply(liveNode), e); } } } - private static PackagesResponse.PackageData toPackageData(PackageStore.Packages packages) { + private static ListPackagesResponse.PackageData toPackageData(PackageStore.Packages packages) { if (packages == null) { return null; } - final var data = new PackagesResponse.PackageData(); + final var data = new ListPackagesResponse.PackageData(); data.znodeVersion = packages.znodeVersion; data.packages = packages.packages.entrySet().stream() @@ -331,9 +331,9 @@ private static PackagesResponse.PackageData toPackageData(PackageStore.Packages return data; } - private static PackagesResponse.PackageVersion toPkgVersionResponse( + private static ListPackagesResponse.PackageVersion toPkgVersionResponse( PackageStore.PkgVersion pkgVersion) { - final var v = new PackagesResponse.PackageVersion(); + final var v = new ListPackagesResponse.PackageVersion(); v.pkg = pkgVersion.pkg; v.version = pkgVersion.version; v.files = pkgVersion.files; diff --git a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java index b7daf76e5043..e8785c7aae9a 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ClusterPackageTest.java @@ -19,7 +19,8 @@ import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; import java.util.List; -import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.GetPackageResponse; +import org.apache.solr.client.api.model.ListPackagesResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; @@ -52,7 +53,7 @@ public void testListPackagesReturnsResult() throws Exception { try (HttpJettySolrClient client = new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) .build()) { - PackagesResponse response = new PackageApi.ListPackages().process(client); + ListPackagesResponse response = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null response from GET /cluster/packages", response); assertNotNull("Expected 'result' field in GET /cluster/packages response", response.result); } @@ -81,7 +82,7 @@ public void testAddAndDeletePackageVersion() throws Exception { addRequest.process(client); // Verify the package was added via GET /cluster/packages - PackagesResponse listResponse = new PackageApi.ListPackages().process(client); + ListPackagesResponse listResponse = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null list response", listResponse); assertNotNull("Expected non-null result", listResponse.result); assertNotNull( @@ -91,19 +92,20 @@ public void testAddAndDeletePackageVersion() throws Exception { listResponse.result.packages.get("pkgapitestpkg").isEmpty()); // Verify GET /cluster/packages/{name} returns only this package - PackagesResponse getByNameResponse = + GetPackageResponse getByNameResponse = new PackageApi.GetPackage("pkgapitestpkg").process(client); assertNotNull("Expected non-null get-by-name response", getByNameResponse); assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); - assertNotNull( - "Expected pkgapitestpkg in get-by-name response", - getByNameResponse.result.packages.get("pkgapitestpkg")); + assertEquals("pkgapitestpkg", getByNameResponse.result.name); + assertFalse( + "Expected at least one version in get-by-name response", + getByNameResponse.result.versions.isEmpty()); // Delete the package version via DELETE /cluster/packages/{name}/versions/{version} new PackageApi.DeletePackageVersion("pkgapitestpkg", "1.0").process(client); // Verify it's deleted - PackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); + ListPackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); assertNotNull("Expected non-null list response after delete", listAfterDelete); assertNotNull("Expected non-null result after delete", listAfterDelete.result); // After deleting the only version, the package entry should be empty or absent @@ -148,6 +150,22 @@ public void testRefreshNonExistentPackage() { } } + @Test + public void testGetNonExistentPackage() { + try (HttpJettySolrClient client = + new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) + .build()) { + RemoteSolrException ex = + expectThrows( + RemoteSolrException.class, + () -> new PackageApi.GetPackage("nonexistentpkg_get").process(client)); + assertEquals("Expected 404 for non-existent package", 404, ex.code()); + assertTrue( + "Expected error message to mention the package: " + ex.getMessage(), + ex.getMessage().contains("No such package")); + } + } + @Test public void testRefreshPackageLocalOnlyAcceptsUnknownPackage() throws Exception { try (HttpJettySolrClient client = @@ -182,15 +200,16 @@ public void testListAndGetPackageAcceptExpectedVersionQueryParam() throws Except // expectedVersion at-or-below current is a no-op in syncToVersion and must return promptly. PackageApi.ListPackages listReq = new PackageApi.ListPackages(); listReq.setExpectedVersion(0); - PackagesResponse listResponse = listReq.process(client); + ListPackagesResponse listResponse = listReq.process(client); assertNotNull(listResponse); assertNotNull(listResponse.result); - PackageApi.GetPackage getReq = new PackageApi.GetPackage("anything"); + // expectedVersion is honored even on the path that ends in a 404 for an unknown package. + PackageApi.GetPackage getReq = new PackageApi.GetPackage("nonexistent_expected_version_pkg"); getReq.setExpectedVersion(0); - PackagesResponse getResponse = getReq.process(client); - assertNotNull(getResponse); - assertNotNull(getResponse.result); + RemoteSolrException ex = + expectThrows(RemoteSolrException.class, () -> getReq.process(client)); + assertEquals("Expected 404 for non-existent package", 404, ex.code()); } } } From 9ff3b2f73ec337774e869190e1d9a02ef82b8f72 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 28 Oct 2026 10:55:30 -0400 Subject: [PATCH 28/31] SOLR-18212: fix CLI package tool to use the new v2 refresh endpoint PackageManager's deploy --update and undeploy flows were still POSTing a v1-style JSON command body ({"refresh": "name"}) to /api/cluster/packages, left over from before the JAX-RS migration. The new endpoint doesn't accept a body at all - it's POST /cluster/packages/{name}/refresh - so these calls were silently broken against the migrated server. Switch both call sites to the generated PackageApi.RefreshPackage client, matching how DeletePackageVersion/AddPackageVersion were already migrated elsewhere in this file. Removes the now-unused PackageUtils.PACKAGE_PATH constant. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019kzScQszwRcG4S7kA5Pa8b --- .../apache/solr/cli/packagemanager/PackageManager.java | 10 +++------- .../apache/solr/cli/packagemanager/PackageUtils.java | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java index d7eb3fe1c816..b82a9d2db11c 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java @@ -447,11 +447,8 @@ private Pair, List> deployCollectionPackage( // If updating, refresh the package version for this to take effect if (isUpdate || pegToLatest) { try { - SolrCLI.postJsonToSolr( - solrClient, - PackageUtils.PACKAGE_PATH, - "{\"refresh\": \"" + packageInstance.name + "\"}"); - } catch (Exception ex) { + new PackageApi.RefreshPackage(packageInstance.name).process(solrClient); + } catch (SolrServerException | IOException ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } } @@ -1065,8 +1062,7 @@ public void undeploy( solrClient, PackageUtils.getCollectionParamsPath(collection), "{set: {PKG_VERSIONS: {" + packageName + ": null}}}"); - SolrCLI.postJsonToSolr( - solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}"); + new PackageApi.RefreshPackage(packageName).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java index 4489a39b91aa..a66a3a28bc34 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java @@ -60,7 +60,6 @@ public class PackageUtils { /** Represents a version which denotes the latest version available at the moment. */ public static String LATEST = "latest"; - public static String PACKAGE_PATH = "/api/cluster/packages"; public static String CLUSTER_PLUGINS_PATH = "/api/cluster/plugin"; public static String REPOSITORIES_ZK_PATH = "/repositories.json"; public static String CLUSTERPROPS_PATH = "/api/cluster/zk/data/clusterprops.json"; From 8a6bdd2bbf7f2bfa0b1970613e6c3f300d124887 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 28 Oct 2026 10:58:34 -0400 Subject: [PATCH 29/31] Use a clearly-fake fixture in TestApiFramework instead of the old Package API shape testPayload()'s @EndPoint/@Command fixture (path /cluster/package, an AddVersion bean with pkg/version/files) mirrored the now-deleted v1 Package API almost exactly, which reads as leftover Package API code rather than a generic framework test. Rename to /foos/FooVersion/myfoo so it's obviously just a test fixture. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019kzScQszwRcG4S7kA5Pa8b --- .../solr/handler/admin/TestApiFramework.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java index e581c574016c..47fd64270c47 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/TestApiFramework.java @@ -119,7 +119,7 @@ public void testFramework() { } public void testPayload() { - String json = "{package:pkg1, version: '0.1', files :[a.jar, b.jar]}"; + String json = "{name:item1, version: '0.1', files :[a.jar, b.jar]}"; Utils.fromJSONString(json); ApiBag apiBag = new ApiBag(false); @@ -129,25 +129,25 @@ public void testPayload() { assertEquals("POST", spec._getStr("/methods[0]")); assertEquals("POST", spec._getStr("/methods[0]")); - assertEquals("/cluster/package", spec._getStr("/url/paths[0]")); - assertEquals("string", spec._getStr("/commands/add/properties/package/type")); + assertEquals("/foos", spec._getStr("/url/paths[0]")); + assertEquals("string", spec._getStr("/commands/add/properties/name/type")); assertEquals("array", spec._getStr("/commands/add/properties/files/type")); assertEquals("string", spec._getStr("/commands/add/properties/files/items/type")); assertEquals("string", spec._getStr("/commands/delete/items/type")); SolrQueryResponse rsp = v2ApiInvoke( apiBag, - "/cluster/package", + "/foos", "POST", new ModifiableSolrParams(), new ByteArrayInputStream( - "{add:{package:mypkg, version: '1.0', files : [a.jar, b.jar]}}".getBytes(UTF_8))); + "{add:{name:myfoo, version: '1.0', files : [a.jar, b.jar]}}".getBytes(UTF_8))); - AddVersion addversion = (AddVersion) rsp.getValues().get("add"); - assertEquals("mypkg", addversion.pkg); - assertEquals("1.0", addversion.version); - assertEquals("a.jar", addversion.files.get(0)); - assertEquals("b.jar", addversion.files.get(1)); + FooVersion fooVersion = (FooVersion) rsp.getValues().get("add"); + assertEquals("myfoo", fooVersion.name); + assertEquals("1.0", fooVersion.version); + assertEquals("a.jar", fooVersion.files.get(0)); + assertEquals("b.jar", fooVersion.files.get(1)); apiBag.registerObject(new C()); rsp = @@ -157,24 +157,24 @@ public void testPayload() { "POST", new ModifiableSolrParams(), new ByteArrayInputStream( - "{\"package\":\"mypkg\", \"version\": \"1.0\", \"files\" : [\"a.jar\", \"b.jar\"]}" + "{\"name\":\"myfoo\", \"version\": \"1.0\", \"files\" : [\"a.jar\", \"b.jar\"]}" .getBytes(UTF_8))); - assertEquals("mypkg", rsp.getValues()._getStr("payload/package")); + assertEquals("myfoo", rsp.getValues()._getStr("payload/name")); assertEquals("1.0", rsp.getValues()._getStr("payload/version")); } public static class C { @EndPoint(path = "/path1", method = POST, permission = PermissionNameProvider.Name.ALL) - public void m1(PayloadObj add) { + public void m1(PayloadObj add) { add.getResponse().add("payload", add.get()); } } - @EndPoint(method = POST, path = "/cluster/package", permission = PermissionNameProvider.Name.ALL) + @EndPoint(method = POST, path = "/foos", permission = PermissionNameProvider.Name.ALL) public static class ApiTest { @Command(name = "add") - public void add(SolrQueryRequest req, SolrQueryResponse rsp, AddVersion addVersion) { - rsp.add("add", addVersion); + public void add(SolrQueryRequest req, SolrQueryResponse rsp, FooVersion fooVersion) { + rsp.add("add", fooVersion); } @Command(name = "delete") @@ -183,9 +183,9 @@ public void del(SolrQueryRequest req, SolrQueryResponse rsp, List names) } } - public static class AddVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; + public static class FooVersion implements ReflectMapWriter { + @JsonProperty(value = "name", required = true) + public String name; @JsonProperty(value = "version", required = true) public String version; From 2690834eeb25ff73fe73f1569b4264dd0b2193fd Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 28 Oct 2026 12:54:59 -0400 Subject: [PATCH 30/31] Fix CI errorprone failure in PackageSchemaReloadTest ReferenceEquality flagged schemas[0] != schemas[1] as a likely mistake, but the test intentionally checks whether the core picked up a new IndexSchema instance after the package version bump - identity, not value equality, is what's being verified. Suppress with the same pattern used elsewhere in the codebase for intentional identity checks (SolrConfig.java, CachingDirectoryFactory.java). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019kzScQszwRcG4S7kA5Pa8b --- .../src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java index 21157d3759e6..7e94025ffa7e 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageSchemaReloadTest.java @@ -63,6 +63,7 @@ public void tearDown() throws Exception { } @Test + @SuppressWarnings("ReferenceEquality") // Identity, not equality, of the schema instance matters public void testSchemaReloadOnPackageVersionBump() throws Exception { String COLLECTION_NAME = "testSchemaLoadingColl"; System.setProperty("managed.schema.mutable", "true"); From f8ed9a0487e8d2617382b57effdd60fed6ac5d9d Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 28 Oct 2026 13:12:49 -0400 Subject: [PATCH 31/31] /simplify pass: dedupe packages.json parsing and enabled-check in ClusterPackage addPackageVersion and deletePackageVersion each inlined the same "deserialize packages.json bytes, fall back to a fresh Packages on missing/invalid data, then copy()" block, and the same isEnabled()/BAD_REQUEST guard. Extracted both into PackageStore (parseForUpdate, requireEnabled) since they operate on PackageStore's own data/mapper, and call them from both atomicUpdate lambdas. Also noted, via a one-line comment, why PackageManager's undeploy refresh call site keeps a broader catch(Exception) than its sibling deploy call site (it still wraps a postJsonToSolr call that declares throws Exception). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019kzScQszwRcG4S7kA5Pa8b --- .../cli/packagemanager/PackageManager.java | 2 +- .../org/apache/solr/pkg/ClusterPackage.java | 29 +++---------------- .../org/apache/solr/pkg/PackageStore.java | 20 +++++++++++++ 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java index b82a9d2db11c..121789ebae36 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageManager.java @@ -1063,7 +1063,7 @@ public void undeploy( PackageUtils.getCollectionParamsPath(collection), "{set: {PKG_VERSIONS: {" + packageName + ": null}}}"); new PackageApi.RefreshPackage(packageName).process(solrClient); - } catch (Exception ex) { + } catch (Exception ex) { // postJsonToSolr above declares a plain throws Exception throw new SolrException(ErrorCode.SERVER_ERROR, ex); } diff --git a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java index 3bc8f579ca4e..8a7dad3bf9a7 100644 --- a/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java +++ b/solr/core/src/java/org/apache/solr/pkg/ClusterPackage.java @@ -109,9 +109,7 @@ public SolrJerseyResponse addPackageVersion( final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - if (!packageStore.isEnabled()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); - } + packageStore.requireEnabled(); if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); } @@ -131,17 +129,7 @@ public SolrJerseyResponse addPackageVersion( .atomicUpdate( SOLR_PKGS_PATH, (stat, bytes) -> { - PackageStore.Packages packages; - try { - packages = - bytes == null - ? new PackageStore.Packages() - : packageStore.mapper.readValue(bytes, PackageStore.Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new PackageStore.Packages(); - } + PackageStore.Packages packages = packageStore.parseForUpdate(bytes); List list = packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); for (PackageStore.PkgVersion pkgVersion : list) { @@ -176,9 +164,7 @@ public SolrJerseyResponse deletePackageVersion(String packageName, String versio final var response = new SolrJerseyResponse(); PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - if (!packageStore.isEnabled()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); - } + packageStore.requireEnabled(); try { coreContainer @@ -187,14 +173,7 @@ public SolrJerseyResponse deletePackageVersion(String packageName, String versio .atomicUpdate( SOLR_PKGS_PATH, (stat, bytes) -> { - PackageStore.Packages packages; - try { - packages = packageStore.mapper.readValue(bytes, PackageStore.Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new PackageStore.Packages(); - } - + PackageStore.Packages packages = packageStore.parseForUpdate(bytes); List versions = packages.packages.get(packageName); if (versions == null || versions.isEmpty()) { throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java index d51c8f56a2a7..99c055c0ea1f 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -216,10 +216,30 @@ public boolean isEnabled() { return enablePackages; } + public void requireEnabled() { + if (!isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ERR_MSG); + } + } + public void handleZkErr(Exception e) { log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); } + /** + * Deserializes {@code packages.json} bytes for a copy-on-write update, defaulting to a fresh + * instance on missing or invalid data. + */ + Packages parseForUpdate(byte[] bytes) { + try { + Packages packages = bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); + return packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + return new Packages(); + } + } + public boolean isJarInuse(String path) { Packages pkg; try {