From e04321fa12aba2ba304549393afe0f27d54db99f Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Wed, 21 Feb 2018 12:03:59 +0100 Subject: [PATCH 0001/2294] [FLINK-8360][checkpointing] Implement file-based local recovery for FsStateBackend This reverts commit 8925b7c --- .../state/heap/HeapKeyedStateBackend.java | 419 +++++++++++------- .../StreamOperatorSnapshotRestoreTest.java | 167 +++++-- 2 files changed, 408 insertions(+), 178 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java index 5d5f7162a46517..d9a5ec11ec1d8b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapKeyedStateBackend.java @@ -41,6 +41,7 @@ import org.apache.flink.runtime.query.TaskKvStateRegistry; import org.apache.flink.runtime.state.AbstractKeyedStateBackend; import org.apache.flink.runtime.state.CheckpointStreamFactory; +import org.apache.flink.runtime.state.CheckpointStreamWithResultProvider; import org.apache.flink.runtime.state.CheckpointedStateScope; import org.apache.flink.runtime.state.DoneFuture; import org.apache.flink.runtime.state.HashMapSerializer; @@ -53,6 +54,7 @@ import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo; import org.apache.flink.runtime.state.SnappyStreamCompressionDecorator; import org.apache.flink.runtime.state.SnapshotResult; +import org.apache.flink.runtime.state.SnapshotStrategy; import org.apache.flink.runtime.state.StreamCompressionDecorator; import org.apache.flink.runtime.state.StreamStateHandle; import org.apache.flink.runtime.state.UncompressedStreamCompressionDecorator; @@ -64,12 +66,15 @@ import org.apache.flink.runtime.state.internal.InternalValueState; import org.apache.flink.util.Preconditions; import org.apache.flink.util.StateMigrationException; +import org.apache.flink.util.function.SupplierWithException; import org.apache.commons.collections.map.HashedMap; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -113,25 +118,34 @@ public class HeapKeyedStateBackend extends AbstractKeyedStateBackend { private final Map> restoredKvStateMetaInfos; /** - * Determines whether or not we run snapshots asynchronously. This impacts the choice of the underlying - * {@link StateTable} implementation. + * The configuration for local recovery. */ - private final boolean asynchronousSnapshots; + private final LocalRecoveryConfig localRecoveryConfig; + + /** + * The snapshot strategy for this backend. This determines, e.g., if snapshots are synchronous or asynchronous. + */ + private final HeapSnapshotStrategy snapshotStrategy; public HeapKeyedStateBackend( - TaskKvStateRegistry kvStateRegistry, - TypeSerializer keySerializer, - ClassLoader userCodeClassLoader, - int numberOfKeyGroups, - KeyGroupRange keyGroupRange, - boolean asynchronousSnapshots, - ExecutionConfig executionConfig, - LocalRecoveryConfig localRecoveryConfig) { + TaskKvStateRegistry kvStateRegistry, + TypeSerializer keySerializer, + ClassLoader userCodeClassLoader, + int numberOfKeyGroups, + KeyGroupRange keyGroupRange, + boolean asynchronousSnapshots, + ExecutionConfig executionConfig, + LocalRecoveryConfig localRecoveryConfig) { super(kvStateRegistry, keySerializer, userCodeClassLoader, numberOfKeyGroups, keyGroupRange, executionConfig); - this.asynchronousSnapshots = asynchronousSnapshots; - LOG.info("Initializing heap keyed state backend with stream factory."); + this.localRecoveryConfig = Preconditions.checkNotNull(localRecoveryConfig); + SnapshotStrategySynchronicityBehavior synchronicityTrait = asynchronousSnapshots ? + new AsyncSnapshotStrategySynchronicityBehavior() : + new SyncSnapshotStrategySynchronicityBehavior(); + + this.snapshotStrategy = new HeapSnapshotStrategy(synchronicityTrait); + LOG.info("Initializing heap keyed state backend with stream factory."); this.restoredKvStateMetaInfos = new HashMap<>(); } @@ -160,7 +174,7 @@ private StateTable tryRegisterStateTable( StateTable stateTable = (StateTable) stateTables.get(stateName); if (stateTable == null) { - stateTable = newStateTable(newMetaInfo); + stateTable = snapshotStrategy.newStateTable(newMetaInfo); stateTables.put(stateName, stateTable); } else { // TODO with eager registration in place, these checks should be moved to restorePartitionedState() @@ -293,139 +307,12 @@ public RunnableFuture> snapshot( final long checkpointId, final long timestamp, final CheckpointStreamFactory streamFactory, - CheckpointOptions checkpointOptions) throws Exception { - - if (!hasRegisteredState()) { - return DoneFuture.of(SnapshotResult.empty()); - } - - long syncStartTime = System.currentTimeMillis(); - - Preconditions.checkState(stateTables.size() <= Short.MAX_VALUE, - "Too many KV-States: " + stateTables.size() + - ". Currently at most " + Short.MAX_VALUE + " states are supported"); - - List> metaInfoSnapshots = new ArrayList<>(stateTables.size()); - - final Map kVStateToId = new HashMap<>(stateTables.size()); - - final Map, StateTableSnapshot> cowStateStableSnapshots = new HashedMap(stateTables.size()); - - for (Map.Entry> kvState : stateTables.entrySet()) { - kVStateToId.put(kvState.getKey(), kVStateToId.size()); - StateTable stateTable = kvState.getValue(); - if (null != stateTable) { - metaInfoSnapshots.add(stateTable.getMetaInfo().snapshot()); - cowStateStableSnapshots.put(stateTable, stateTable.createSnapshot()); - } - } - - final KeyedBackendSerializationProxy serializationProxy = - new KeyedBackendSerializationProxy<>( - keySerializer, - metaInfoSnapshots, - !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator)); - - //--------------------------------------------------- this becomes the end of sync part - - // implementation of the async IO operation, based on FutureTask - final AbstractAsyncCallableWithResources> ioCallable = - new AbstractAsyncCallableWithResources>() { - - CheckpointStreamFactory.CheckpointStateOutputStream stream = null; - - @Override - protected void acquireResources() throws Exception { - stream = streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); - cancelStreamRegistry.registerCloseable(stream); - } - - @Override - protected void releaseResources() throws Exception { - - if (cancelStreamRegistry.unregisterCloseable(stream)) { - IOUtils.closeQuietly(stream); - stream = null; - } - - for (StateTableSnapshot tableSnapshot : cowStateStableSnapshots.values()) { - tableSnapshot.release(); - } - } - - @Override - protected void stopOperation() throws Exception { - if (cancelStreamRegistry.unregisterCloseable(stream)) { - IOUtils.closeQuietly(stream); - stream = null; - } - } - - @Override - public SnapshotResult performOperation() throws Exception { - long asyncStartTime = System.currentTimeMillis(); - - CheckpointStreamFactory.CheckpointStateOutputStream localStream = this.stream; - - DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(localStream); - serializationProxy.write(outView); - - long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; - - for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) { - int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos); - keyGroupRangeOffsets[keyGroupPos] = localStream.getPos(); - outView.writeInt(keyGroupId); - - for (Map.Entry> kvState : stateTables.entrySet()) { - OutputStream kgCompressionOut = keyGroupCompressionDecorator.decorateWithCompression(localStream); - DataOutputViewStreamWrapper kgCompressionView = new DataOutputViewStreamWrapper(kgCompressionOut); - kgCompressionView.writeShort(kVStateToId.get(kvState.getKey())); - cowStateStableSnapshots.get(kvState.getValue()).writeMappingsInKeyGroup(kgCompressionView, keyGroupId); - kgCompressionOut.close(); // this will just close the outer stream - } - } - - if (cancelStreamRegistry.unregisterCloseable(stream)) { - - final StreamStateHandle streamStateHandle = stream.closeAndGetHandle(); - stream = null; - - if (asynchronousSnapshots) { - LOG.info("Heap backend snapshot ({}, asynchronous part) in thread {} took {} ms.", - streamFactory, Thread.currentThread(), (System.currentTimeMillis() - asyncStartTime)); - } - - if (streamStateHandle != null) { - - KeyGroupRangeOffsets offsets = - new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets); - - final KeyGroupsStateHandle keyGroupsStateHandle = - new KeyGroupsStateHandle(offsets, streamStateHandle); - - return SnapshotResult.of(keyGroupsStateHandle); - } - } - - return SnapshotResult.empty(); - } - }; - - AsyncStoppableTaskWithCallback> task = AsyncStoppableTaskWithCallback.from(ioCallable); + CheckpointOptions checkpointOptions) { - if (!asynchronousSnapshots) { - task.run(); - } - - LOG.info("Heap backend snapshot (" + streamFactory + ", synchronous part) in thread " + - Thread.currentThread() + " took " + (System.currentTimeMillis() - syncStartTime) + " ms."); - - return task; + return snapshotStrategy.performSnapshot(checkpointId, timestamp, streamFactory, checkpointOptions); } @SuppressWarnings("deprecation") - @Override public void restore(Collection restoredState) throws Exception { if (restoredState == null || restoredState.isEmpty()) { return; @@ -525,7 +412,7 @@ private void restorePartitionedState(Collection state) throws restoredMetaInfo.getNamespaceSerializer(), restoredMetaInfo.getStateSerializer()); - stateTable = newStateTable(registeredKeyedBackendStateMetaInfo); + stateTable = snapshotStrategy.newStateTable(registeredKeyedBackendStateMetaInfo); stateTables.put(restoredMetaInfo.getName(), stateTable); kvStatesById.put(numRegisteredKvStates, restoredMetaInfo.getName()); ++numRegisteredKvStates; @@ -614,14 +501,244 @@ public int numStateEntries(Object namespace) { return sum; } - public StateTable newStateTable(RegisteredKeyedBackendStateMetaInfo newMetaInfo) { - return asynchronousSnapshots ? - new CopyOnWriteStateTable<>(this, newMetaInfo) : - new NestedMapsStateTable<>(this, newMetaInfo); - } - @Override public boolean supportsAsynchronousSnapshots() { - return asynchronousSnapshots; + return snapshotStrategy.isAsynchronous(); + } + + @VisibleForTesting + public LocalRecoveryConfig getLocalRecoveryConfig() { + return localRecoveryConfig; + } + + private interface SnapshotStrategySynchronicityBehavior { + + default void finalizeSnapshotBeforeReturnHook(Runnable runnable) { + + } + + default void logOperationCompleted(CheckpointStreamFactory streamFactory, long startTime) { + + } + + boolean isAsynchronous(); + + StateTable newStateTable(RegisteredKeyedBackendStateMetaInfo newMetaInfo); + } + + private class AsyncSnapshotStrategySynchronicityBehavior implements SnapshotStrategySynchronicityBehavior { + + @Override + public void logOperationCompleted(CheckpointStreamFactory streamFactory, long startTime) { + LOG.info("Heap backend snapshot ({}, asynchronous part) in thread {} took {} ms.", + streamFactory, Thread.currentThread(), (System.currentTimeMillis() - startTime)); + } + + @Override + public boolean isAsynchronous() { + return true; + } + + @Override + public StateTable newStateTable(RegisteredKeyedBackendStateMetaInfo newMetaInfo) { + return new CopyOnWriteStateTable<>(HeapKeyedStateBackend.this, newMetaInfo); + } + } + + private class SyncSnapshotStrategySynchronicityBehavior implements SnapshotStrategySynchronicityBehavior { + + @Override + public void finalizeSnapshotBeforeReturnHook(Runnable runnable) { + // this triggers a synchronous execution from the main checkpointing thread. + runnable.run(); + } + + @Override + public boolean isAsynchronous() { + return false; + } + + @Override + public StateTable newStateTable(RegisteredKeyedBackendStateMetaInfo newMetaInfo) { + return new NestedMapsStateTable<>(HeapKeyedStateBackend.this, newMetaInfo); + } + } + + /** + * Base class for the snapshots of the heap backend that outlines the algorithm and offers some hooks to realize + * the concrete strategies. Subclasses must be threadsafe. + */ + private class HeapSnapshotStrategy + implements SnapshotStrategy>, SnapshotStrategySynchronicityBehavior { + + private final SnapshotStrategySynchronicityBehavior snapshotStrategySynchronicityTrait; + + public HeapSnapshotStrategy( + SnapshotStrategySynchronicityBehavior snapshotStrategySynchronicityTrait) { + this.snapshotStrategySynchronicityTrait = snapshotStrategySynchronicityTrait; + } + + @Override + public RunnableFuture> performSnapshot( + long checkpointId, + long timestamp, + CheckpointStreamFactory primaryStreamFactory, + CheckpointOptions checkpointOptions) { + + if (!hasRegisteredState()) { + return DoneFuture.of(SnapshotResult.empty()); + } + + long syncStartTime = System.currentTimeMillis(); + + Preconditions.checkState(stateTables.size() <= Short.MAX_VALUE, + "Too many KV-States: " + stateTables.size() + + ". Currently at most " + Short.MAX_VALUE + " states are supported"); + + List> metaInfoSnapshots = + new ArrayList<>(stateTables.size()); + + final Map kVStateToId = new HashMap<>(stateTables.size()); + + final Map, StateTableSnapshot> cowStateStableSnapshots = + new HashedMap(stateTables.size()); + + for (Map.Entry> kvState : stateTables.entrySet()) { + kVStateToId.put(kvState.getKey(), kVStateToId.size()); + StateTable stateTable = kvState.getValue(); + if (null != stateTable) { + metaInfoSnapshots.add(stateTable.getMetaInfo().snapshot()); + cowStateStableSnapshots.put(stateTable, stateTable.createSnapshot()); + } + } + + final KeyedBackendSerializationProxy serializationProxy = + new KeyedBackendSerializationProxy<>( + keySerializer, + metaInfoSnapshots, + !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator)); + + final SupplierWithException checkpointStreamSupplier = + + LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED.equals( + localRecoveryConfig.getLocalRecoveryMode()) ? + + () -> CheckpointStreamWithResultProvider.createDuplicatingStream( + checkpointId, + CheckpointedStateScope.EXCLUSIVE, + primaryStreamFactory, + localRecoveryConfig.getLocalStateDirectoryProvider()) : + + () -> CheckpointStreamWithResultProvider.createSimpleStream( + CheckpointedStateScope.EXCLUSIVE, + primaryStreamFactory); + + //--------------------------------------------------- this becomes the end of sync part + + // implementation of the async IO operation, based on FutureTask + final AbstractAsyncCallableWithResources> ioCallable = + new AbstractAsyncCallableWithResources>() { + + CheckpointStreamWithResultProvider streamAndResultExtractor = null; + + @Override + protected void acquireResources() throws Exception { + streamAndResultExtractor = checkpointStreamSupplier.get(); + cancelStreamRegistry.registerCloseable(streamAndResultExtractor); + } + + @Override + protected void releaseResources() { + + unregisterAndCloseStreamAndResultExtractor(); + + for (StateTableSnapshot tableSnapshot : cowStateStableSnapshots.values()) { + tableSnapshot.release(); + } + } + + @Override + protected void stopOperation() { + unregisterAndCloseStreamAndResultExtractor(); + } + + private void unregisterAndCloseStreamAndResultExtractor() { + if (cancelStreamRegistry.unregisterCloseable(streamAndResultExtractor)) { + IOUtils.closeQuietly(streamAndResultExtractor); + streamAndResultExtractor = null; + } + } + + @Nonnull + @Override + protected SnapshotResult performOperation() throws Exception { + + long startTime = System.currentTimeMillis(); + + CheckpointStreamFactory.CheckpointStateOutputStream localStream = + this.streamAndResultExtractor.getCheckpointOutputStream(); + + DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(localStream); + serializationProxy.write(outView); + + long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; + + for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) { + int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos); + keyGroupRangeOffsets[keyGroupPos] = localStream.getPos(); + outView.writeInt(keyGroupId); + + for (Map.Entry> kvState : stateTables.entrySet()) { + try (OutputStream kgCompressionOut = keyGroupCompressionDecorator.decorateWithCompression(localStream)) { + DataOutputViewStreamWrapper kgCompressionView = new DataOutputViewStreamWrapper(kgCompressionOut); + kgCompressionView.writeShort(kVStateToId.get(kvState.getKey())); + cowStateStableSnapshots.get(kvState.getValue()).writeMappingsInKeyGroup(kgCompressionView, keyGroupId); + } // this will just close the outer compression stream + } + } + + if (cancelStreamRegistry.unregisterCloseable(streamAndResultExtractor)) { + KeyGroupRangeOffsets kgOffs = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets); + SnapshotResult result = + streamAndResultExtractor.closeAndFinalizeCheckpointStreamResult(); + streamAndResultExtractor = null; + logOperationCompleted(primaryStreamFactory, startTime); + return CheckpointStreamWithResultProvider.toKeyedStateHandleSnapshotResult(result, kgOffs); + } + + return SnapshotResult.empty(); + } + }; + + AsyncStoppableTaskWithCallback> task = + AsyncStoppableTaskWithCallback.from(ioCallable); + + finalizeSnapshotBeforeReturnHook(task); + + LOG.info("Heap backend snapshot (" + primaryStreamFactory + ", synchronous part) in thread " + + Thread.currentThread() + " took " + (System.currentTimeMillis() - syncStartTime) + " ms."); + + return task; + } + + @Override + public void finalizeSnapshotBeforeReturnHook(Runnable runnable) { + snapshotStrategySynchronicityTrait.finalizeSnapshotBeforeReturnHook(runnable); + } + + @Override + public void logOperationCompleted(CheckpointStreamFactory streamFactory, long startTime) { + snapshotStrategySynchronicityTrait.logOperationCompleted(streamFactory, startTime); + } + + @Override + public boolean isAsynchronous() { + return snapshotStrategySynchronicityTrait.isAsynchronous(); + } + + @Override + public StateTable newStateTable(RegisteredKeyedBackendStateMetaInfo newMetaInfo) { + return snapshotStrategySynchronicityTrait.newStateTable(newMetaInfo); + } } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamOperatorSnapshotRestoreTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamOperatorSnapshotRestoreTest.java index 9d0b9e2a369477..b2b568e05365ec 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamOperatorSnapshotRestoreTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/StreamOperatorSnapshotRestoreTest.java @@ -18,6 +18,8 @@ package org.apache.flink.streaming.api.operators; +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.state.ValueState; @@ -25,65 +27,159 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.configuration.Configuration; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataInputViewStreamWrapper; import org.apache.flink.core.memory.DataOutputView; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.jobgraph.JobVertexID; +import org.apache.flink.runtime.operators.testutils.MockEnvironment; +import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider; import org.apache.flink.runtime.state.AbstractKeyedStateBackend; import org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider; import org.apache.flink.runtime.state.KeyedStateCheckpointOutputStream; +import org.apache.flink.runtime.state.LocalRecoveryConfig; +import org.apache.flink.runtime.state.LocalRecoveryDirectoryProvider; +import org.apache.flink.runtime.state.LocalRecoveryDirectoryProviderImpl; import org.apache.flink.runtime.state.OperatorStateCheckpointOutputStream; import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StatePartitionStreamProvider; import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.runtime.state.TestTaskStateManager; +import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; +import org.apache.flink.util.TestLogger; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.util.BitSet; /** * Tests for {@link StreamOperator} snapshot restoration. */ -public class StreamOperatorSnapshotRestoreTest { +public class StreamOperatorSnapshotRestoreTest extends TestLogger { + + private static final int ONLY_JM_RECOVERY = 0; + private static final int TM_AND_JM_RECOVERY = 1; + private static final int TM_REMOVE_JM_RECOVERY = 2; + private static final int JM_REMOVE_TM_RECOVERY = 3; private static final int MAX_PARALLELISM = 10; + protected static TemporaryFolder temporaryFolder; + + @BeforeClass + public static void beforeClass() throws IOException { + temporaryFolder = new TemporaryFolder(); + temporaryFolder.create(); + } + + @AfterClass + public static void afterClass() { + temporaryFolder.delete(); + } + + /** + * Test restoring an operator from a snapshot (local recovery deactivated). + */ @Test public void testOperatorStatesSnapshotRestore() throws Exception { + testOperatorStatesSnapshotRestoreInternal(ONLY_JM_RECOVERY); + } + + /** + * Test restoring an operator from a snapshot (local recovery activated). + */ + @Test + public void testOperatorStatesSnapshotRestoreWithLocalState() throws Exception { + testOperatorStatesSnapshotRestoreInternal(TM_AND_JM_RECOVERY); + } + + /** + * Test restoring an operator from a snapshot (local recovery activated, JM snapshot deleted). + * + *

This case does not really simulate a practical scenario, but we make sure that restore happens from the local + * state here because we discard the JM state. + */ + @Test + public void testOperatorStatesSnapshotRestoreWithLocalStateDeletedJM() throws Exception { + testOperatorStatesSnapshotRestoreInternal(TM_REMOVE_JM_RECOVERY); + } + + /** + * Test restoring an operator from a snapshot (local recovery activated, local TM snapshot deleted). + * + *

This tests discards the local state, to simulate corruption and checks that we still recover from the fallback + * JM state. + */ + @Test + public void testOperatorStatesSnapshotRestoreWithLocalStateDeletedTM() throws Exception { + testOperatorStatesSnapshotRestoreInternal(JM_REMOVE_TM_RECOVERY); + } + + private void testOperatorStatesSnapshotRestoreInternal(final int mode) throws Exception { //-------------------------------------------------------------------------- snapshot + StateBackend stateBackend = createStateBackend(); + TestOneInputStreamOperator op = new TestOneInputStreamOperator(false); + JobID jobID = new JobID(); + JobVertexID jobVertexID = new JobVertexID(); + int subtaskIdx = 0; + + LocalRecoveryDirectoryProvider directoryProvider = + new LocalRecoveryDirectoryProviderImpl(temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx); + + LocalRecoveryConfig localRecoveryConfig = + mode != ONLY_JM_RECOVERY ? + new LocalRecoveryConfig(LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED, directoryProvider) : + new LocalRecoveryConfig(LocalRecoveryConfig.LocalRecoveryMode.DISABLED, directoryProvider); + + MockEnvironment mockEnvironment = new MockEnvironment( + jobID, + jobVertexID, + "test", + 1024L * 1024L, + new MockInputSplitProvider(), + 1024 * 1024, + new Configuration(), + new ExecutionConfig(), + new TestTaskStateManager(localRecoveryConfig), + MAX_PARALLELISM, + 1, + subtaskIdx, + getClass().getClassLoader()); + KeyedOneInputStreamOperatorTestHarness testHarness = - new KeyedOneInputStreamOperatorTestHarness<>( - op, - new KeySelector() { - @Override - public Integer getKey(Integer value) throws Exception { - return value; - } - }, - TypeInformation.of(Integer.class), - MAX_PARALLELISM, - 1 /* num subtasks */, - 0 /* subtask index */); + new KeyedOneInputStreamOperatorTestHarness<>( + op, + (KeySelector) value -> value, + TypeInformation.of(Integer.class), + mockEnvironment); + + testHarness.setStateBackend(stateBackend); testHarness.open(); for (int i = 0; i < 10; ++i) { testHarness.processElement(new StreamRecord<>(i)); } - OperatorSubtaskState handles = testHarness.snapshot(1L, 1L); + OperatorSnapshotFinalizer snapshotWithLocalState = testHarness.snapshotWithLocalState(1L, 1L); testHarness.close(); @@ -91,17 +187,12 @@ public Integer getKey(Integer value) throws Exception { op = new TestOneInputStreamOperator(true); testHarness = new KeyedOneInputStreamOperatorTestHarness( - op, - new KeySelector() { - @Override - public Integer getKey(Integer value) throws Exception { - return value; - } - }, - TypeInformation.of(Integer.class), - MAX_PARALLELISM, - 1 /* num subtasks */, - 0 /* subtask index */) { + op, + (KeySelector) value -> value, + TypeInformation.of(Integer.class), + MAX_PARALLELISM, + 1 /* num subtasks */, + 0 /* subtask index */) { @Override protected StreamTaskStateInitializer createStreamTaskStateManager( @@ -122,7 +213,21 @@ protected InternalTimeServiceManager internalTimeServiceManager( } }; - testHarness.initializeState(handles); + testHarness.setStateBackend(stateBackend); + + OperatorSubtaskState jobManagerOwnedState = snapshotWithLocalState.getJobManagerOwnedState(); + OperatorSubtaskState taskLocalState = snapshotWithLocalState.getTaskLocalState(); + + // We check if local state was created when we enabled local recovery + Assert.assertTrue(mode > ONLY_JM_RECOVERY == (taskLocalState != null && taskLocalState.hasState())); + + if (mode == TM_REMOVE_JM_RECOVERY) { + jobManagerOwnedState.getManagedKeyedState().discardState(); + } else if (mode == JM_REMOVE_TM_RECOVERY) { + taskLocalState.getManagedKeyedState().discardState(); + } + + testHarness.initializeState(jobManagerOwnedState, taskLocalState); testHarness.open(); @@ -133,6 +238,15 @@ protected InternalTimeServiceManager internalTimeServiceManager( testHarness.close(); } + protected StateBackend createStateBackend() throws IOException { + return createStateBackendInternal(); + } + + protected final FsStateBackend createStateBackendInternal() throws IOException { + File checkpointDir = temporaryFolder.newFolder(); + return new FsStateBackend(checkpointDir.toURI()); + } + static class TestOneInputStreamOperator extends AbstractStreamOperator implements OneInputStreamOperator { @@ -237,5 +351,4 @@ public void initializeState(StateInitializationContext context) throws Exception } } } - } From 1619fa8abe8d605b14ee0ddabcdc86196c50e24a Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Wed, 21 Feb 2018 11:27:17 +0100 Subject: [PATCH 0002/2294] [FLINK-8360][checkpointing] Implement file-based local recovery for RocksDBStateBackend --- .../state/RocksDBKeyedStateBackend.java | 3038 +++++++++-------- ...ocksStreamOperatorSnapshotRestoreTest.java | 37 + .../RocksDBRocksIteratorWrapperTest.java | 15 +- ...ocksStreamOperatorSnapshotRestoreTest.java | 37 + 4 files changed, 1769 insertions(+), 1358 deletions(-) create mode 100644 flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/IncrementalRocksStreamOperatorSnapshotRestoreTest.java create mode 100644 flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksStreamOperatorSnapshotRestoreTest.java diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 344255fe959255..0cb2792f0b2e5c 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -52,19 +52,25 @@ import org.apache.flink.runtime.query.TaskKvStateRegistry; import org.apache.flink.runtime.state.AbstractKeyedStateBackend; import org.apache.flink.runtime.state.CheckpointStreamFactory; +import org.apache.flink.runtime.state.CheckpointStreamWithResultProvider; import org.apache.flink.runtime.state.CheckpointedStateScope; +import org.apache.flink.runtime.state.DirectoryStateHandle; import org.apache.flink.runtime.state.DoneFuture; import org.apache.flink.runtime.state.IncrementalKeyedStateHandle; +import org.apache.flink.runtime.state.IncrementalLocalKeyedStateHandle; import org.apache.flink.runtime.state.KeyGroupRange; import org.apache.flink.runtime.state.KeyGroupRangeOffsets; import org.apache.flink.runtime.state.KeyGroupsStateHandle; import org.apache.flink.runtime.state.KeyedBackendSerializationProxy; import org.apache.flink.runtime.state.KeyedStateHandle; import org.apache.flink.runtime.state.LocalRecoveryConfig; +import org.apache.flink.runtime.state.LocalRecoveryDirectoryProvider; import org.apache.flink.runtime.state.PlaceholderStreamStateHandle; import org.apache.flink.runtime.state.RegisteredKeyedBackendStateMetaInfo; import org.apache.flink.runtime.state.SnappyStreamCompressionDecorator; +import org.apache.flink.runtime.state.SnapshotDirectory; import org.apache.flink.runtime.state.SnapshotResult; +import org.apache.flink.runtime.state.SnapshotStrategy; import org.apache.flink.runtime.state.StateHandleID; import org.apache.flink.runtime.state.StateObject; import org.apache.flink.runtime.state.StateUtil; @@ -77,12 +83,14 @@ import org.apache.flink.runtime.state.internal.InternalMapState; import org.apache.flink.runtime.state.internal.InternalReducingState; import org.apache.flink.runtime.state.internal.InternalValueState; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FileUtils; import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.IOUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.util.ResourceGuard; import org.apache.flink.util.StateMigrationException; +import org.apache.flink.util.function.SupplierWithException; import org.rocksdb.Checkpoint; import org.rocksdb.ColumnFamilyDescriptor; @@ -104,6 +112,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -122,7 +131,6 @@ import java.util.Spliterators; import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.Callable; import java.util.concurrent.FutureTask; import java.util.concurrent.RunnableFuture; import java.util.stream.Stream; @@ -217,6 +225,9 @@ public class RocksDBKeyedStateBackend extends AbstractKeyedStateBackend { /** The configuration of local recovery. */ private final LocalRecoveryConfig localRecoveryConfig; + /** The snapshot strategy, e.g., if we use full or incremental checkpoints, local state, and so on. */ + private final SnapshotStrategy> snapshotStrategy; + public RocksDBKeyedStateBackend( String operatorIdentifier, ClassLoader userCodeClassLoader, @@ -248,26 +259,41 @@ public RocksDBKeyedStateBackend( this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath); this.instanceRocksDBPath = new File(instanceBasePath, "db"); - if (instanceBasePath.exists()) { + checkAndCreateDirectory(instanceBasePath); + + if (instanceRocksDBPath.exists()) { // Clear the base directory when the backend is created // in case something crashed and the backend never reached dispose() cleanInstanceBasePath(); } - if (!instanceBasePath.mkdirs()) { - throw new IOException( - String.format("Could not create RocksDB data directory at %s.", instanceBasePath.getAbsolutePath())); - } - this.localRecoveryConfig = Preconditions.checkNotNull(localRecoveryConfig); this.keyGroupPrefixBytes = getNumberOfKeyGroups() > (Byte.MAX_VALUE + 1) ? 2 : 1; this.kvStateInformation = new HashMap<>(); this.restoredKvStateMetaInfos = new HashMap<>(); this.materializedSstFiles = new TreeMap<>(); this.backendUID = UUID.randomUUID(); + + this.snapshotStrategy = enableIncrementalCheckpointing ? + new IncrementalSnapshotStrategy() : + new FullSnapshotStrategy(); + LOG.debug("Setting initial keyed backend uid for operator {} to {}.", this.operatorIdentifier, this.backendUID); } + private static void checkAndCreateDirectory(File directory) throws IOException { + if (directory.exists()) { + if (!directory.isDirectory()) { + throw new IOException("Not a directory: " + directory); + } + } else { + if (!directory.mkdirs()) { + throw new IOException( + String.format("Could not create RocksDB data directory at %s.", directory)); + } + } + } + @Override public Stream getKeys(String state, N namespace) { Tuple2> columnInfo = kvStateInformation.get(state); @@ -294,7 +320,7 @@ public Stream getKeys(String state, N namespace) { RocksIterator iterator = db.newIterator(columnInfo.f0); iterator.seekToFirst(); - final RocksIteratorWrapper iteratorWrapper = new RocksIteratorWrapper<>(iterator, state, keySerializer, keyGroupPrefixBytes, + final RocksIteratorForKeysWrapper iteratorWrapper = new RocksIteratorForKeysWrapper<>(iterator, state, keySerializer, keyGroupPrefixBytes, ambiguousKeyPossible, nameSpaceBytes); Stream targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false); @@ -381,1743 +407,2053 @@ public RunnableFuture> snapshot( final CheckpointStreamFactory streamFactory, CheckpointOptions checkpointOptions) throws Exception { - if (checkpointOptions.getCheckpointType() != CheckpointType.SAVEPOINT && - enableIncrementalCheckpointing) { - return snapshotIncrementally(checkpointId, timestamp, streamFactory); - } else { - return snapshotFully(checkpointId, timestamp, streamFactory); - } + return snapshotStrategy.performSnapshot(checkpointId, timestamp, streamFactory, checkpointOptions); } - private RunnableFuture> snapshotIncrementally( - final long checkpointId, - final long checkpointTimestamp, - final CheckpointStreamFactory checkpointStreamFactory) throws Exception { - - if (db == null) { - throw new IOException("RocksDB closed."); - } + @Override + public void restore(Collection restoreState) throws Exception { + LOG.info("Initializing RocksDB keyed state backend."); - if (kvStateInformation.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Asynchronous RocksDB snapshot performed on empty keyed state at {}. Returning null.", - checkpointTimestamp); - } - return DoneFuture.of(SnapshotResult.empty()); + if (LOG.isDebugEnabled()) { + LOG.debug("Restoring snapshot from state handles: {}.", restoreState); } - final RocksDBIncrementalSnapshotOperation snapshotOperation = - new RocksDBIncrementalSnapshotOperation<>( - this, - checkpointStreamFactory, - checkpointId, - checkpointTimestamp); + // clear all meta data + kvStateInformation.clear(); + restoredKvStateMetaInfos.clear(); try { - snapshotOperation.takeSnapshot(); - } catch (Exception e) { - snapshotOperation.stop(); - snapshotOperation.releaseResources(true); - throw e; - } - - return new FutureTask>( - new Callable>() { - @Override - public SnapshotResult call() throws Exception { - KeyedStateHandle keyedStateHandle = snapshotOperation.materializeSnapshot(); - return SnapshotResult.of(keyedStateHandle); + if (restoreState == null || restoreState.isEmpty()) { + createDB(); + } else { + KeyedStateHandle firstStateHandle = restoreState.iterator().next(); + if (firstStateHandle instanceof IncrementalKeyedStateHandle + || firstStateHandle instanceof IncrementalLocalKeyedStateHandle) { + RocksDBIncrementalRestoreOperation restoreOperation = new RocksDBIncrementalRestoreOperation<>(this); + restoreOperation.restore(restoreState); + } else { + RocksDBFullRestoreOperation restoreOperation = new RocksDBFullRestoreOperation<>(this); + restoreOperation.doRestore(restoreState); } } - ) { - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - snapshotOperation.stop(); - return super.cancel(mayInterruptIfRunning); - } - - @Override - protected void done() { - snapshotOperation.releaseResources(isCancelled()); - } - }; + } catch (Exception ex) { + dispose(); + throw ex; + } } - private RunnableFuture> snapshotFully( - final long checkpointId, - final long timestamp, - final CheckpointStreamFactory streamFactory) throws Exception { - - long startTime = System.currentTimeMillis(); - final CloseableRegistry snapshotCloseableRegistry = new CloseableRegistry(); - - final RocksDBFullSnapshotOperation snapshotOperation; - - if (kvStateInformation.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Asynchronous RocksDB snapshot performed on empty keyed state at {}. Returning null.", timestamp); - } + @Override + public void notifyCheckpointComplete(long completedCheckpointId) { - return DoneFuture.of(SnapshotResult.empty()); + if (!enableIncrementalCheckpointing) { + return; } - snapshotOperation = new RocksDBFullSnapshotOperation<>(this, streamFactory, snapshotCloseableRegistry); - snapshotOperation.takeDBSnapShot(checkpointId, timestamp); - - // implementation of the async IO operation, based on FutureTask - AbstractAsyncCallableWithResources> ioCallable = - new AbstractAsyncCallableWithResources>() { + synchronized (materializedSstFiles) { - @Override - protected void acquireResources() throws Exception { - cancelStreamRegistry.registerCloseable(snapshotCloseableRegistry); - snapshotOperation.openCheckpointStream(); - } + if (completedCheckpointId < lastCompletedCheckpointId) { + return; + } - @Override - protected void releaseResources() throws Exception { - closeLocalRegistry(); - releaseSnapshotOperationResources(); - } + materializedSstFiles.keySet().removeIf(checkpointId -> checkpointId < completedCheckpointId); - private void releaseSnapshotOperationResources() { - // hold the db lock while operation on the db to guard us against async db disposal - snapshotOperation.releaseSnapshotResources(); - } + lastCompletedCheckpointId = completedCheckpointId; + } + } - @Override - protected void stopOperation() throws Exception { - closeLocalRegistry(); - } + private void createDB() throws IOException { + List columnFamilyHandles = new ArrayList<>(1); + this.db = openDB(instanceRocksDBPath.getAbsolutePath(), Collections.emptyList(), columnFamilyHandles); + this.defaultColumnFamily = columnFamilyHandles.get(0); + } - private void closeLocalRegistry() { - if (cancelStreamRegistry.unregisterCloseable(snapshotCloseableRegistry)) { - try { - snapshotCloseableRegistry.close(); - } catch (Exception ex) { - LOG.warn("Error closing local registry", ex); - } - } - } + private RocksDB openDB( + String path, + List stateColumnFamilyDescriptors, + List stateColumnFamilyHandles) throws IOException { - @Override - public SnapshotResult performOperation() throws Exception { - long startTime = System.currentTimeMillis(); + List columnFamilyDescriptors = + new ArrayList<>(1 + stateColumnFamilyDescriptors.size()); - if (isStopped()) { - throw new IOException("RocksDB closed."); - } + columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors); - snapshotOperation.writeDBSnapshot(); + // we add the required descriptor for the default CF in last position. + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_COLUMN_FAMILY_NAME_BYTES, columnOptions)); - LOG.info("Asynchronous RocksDB snapshot ({}, asynchronous part) in thread {} took {} ms.", - streamFactory, Thread.currentThread(), (System.currentTimeMillis() - startTime)); + RocksDB dbRef; - KeyGroupsStateHandle snapshotResultStateHandle = snapshotOperation.getSnapshotResultStateHandle(); - return SnapshotResult.of(snapshotResultStateHandle); - } - }; + try { + dbRef = RocksDB.open( + Preconditions.checkNotNull(dbOptions), + Preconditions.checkNotNull(path), + columnFamilyDescriptors, + stateColumnFamilyHandles); + } catch (RocksDBException e) { + throw new IOException("Error while opening RocksDB instance.", e); + } - LOG.info("Asynchronous RocksDB snapshot ({}, synchronous part) in thread {} took {} ms.", - streamFactory, Thread.currentThread(), (System.currentTimeMillis() - startTime)); + // requested + default CF + Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(), + "Not all requested column family handles have been created"); - return AsyncStoppableTaskWithCallback.from(ioCallable); + return dbRef; } /** - * Encapsulates the process to perform a snapshot of a RocksDBKeyedStateBackend. + * Encapsulates the process of restoring a RocksDBKeyedStateBackend from a full snapshot. */ - static final class RocksDBFullSnapshotOperation { - - static final int FIRST_BIT_IN_BYTE_MASK = 0x80; - static final int END_OF_KEY_GROUP_MARK = 0xFFFF; - - private final RocksDBKeyedStateBackend stateBackend; - private final KeyGroupRangeOffsets keyGroupRangeOffsets; - private final CheckpointStreamFactory checkpointStreamFactory; - private final CloseableRegistry snapshotCloseableRegistry; - private final ResourceGuard.Lease dbLease; - - private long checkpointId; - private long checkpointTimeStamp; + private static final class RocksDBFullRestoreOperation { - private Snapshot snapshot; - private ReadOptions readOptions; - private List> kvStateIterators; - - private CheckpointStreamFactory.CheckpointStateOutputStream outStream; - private DataOutputView outputView; - - RocksDBFullSnapshotOperation( - RocksDBKeyedStateBackend stateBackend, - CheckpointStreamFactory checkpointStreamFactory, - CloseableRegistry registry) throws IOException { + private final RocksDBKeyedStateBackend rocksDBKeyedStateBackend; - this.stateBackend = stateBackend; - this.checkpointStreamFactory = checkpointStreamFactory; - this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(stateBackend.keyGroupRange); - this.snapshotCloseableRegistry = registry; - this.dbLease = this.stateBackend.rocksDBResourceGuard.acquireResource(); - } + /** Current key-groups state handle from which we restore key-groups. */ + private KeyGroupsStateHandle currentKeyGroupsStateHandle; + /** Current input stream we obtained from currentKeyGroupsStateHandle. */ + private FSDataInputStream currentStateHandleInStream; + /** Current data input view that wraps currentStateHandleInStream. */ + private DataInputView currentStateHandleInView; + /** Current list of ColumnFamilyHandles for all column families we restore from currentKeyGroupsStateHandle. */ + private List currentStateHandleKVStateColumnFamilies; + /** The compression decorator that was used for writing the state, as determined by the meta data. */ + private StreamCompressionDecorator keygroupStreamCompressionDecorator; /** - * 1) Create a snapshot object from RocksDB. + * Creates a restore operation object for the given state backend instance. * - * @param checkpointId id of the checkpoint for which we take the snapshot - * @param checkpointTimeStamp timestamp of the checkpoint for which we take the snapshot + * @param rocksDBKeyedStateBackend the state backend into which we restore */ - public void takeDBSnapShot(long checkpointId, long checkpointTimeStamp) { - Preconditions.checkArgument(snapshot == null, "Only one ongoing snapshot allowed!"); - this.kvStateIterators = new ArrayList<>(stateBackend.kvStateInformation.size()); - this.checkpointId = checkpointId; - this.checkpointTimeStamp = checkpointTimeStamp; - this.snapshot = stateBackend.db.getSnapshot(); + public RocksDBFullRestoreOperation(RocksDBKeyedStateBackend rocksDBKeyedStateBackend) { + this.rocksDBKeyedStateBackend = Preconditions.checkNotNull(rocksDBKeyedStateBackend); } /** - * 2) Open CheckpointStateOutputStream through the checkpointStreamFactory into which we will write. + * Restores all key-groups data that is referenced by the passed state handles. * - * @throws Exception + * @param keyedStateHandles List of all key groups state handles that shall be restored. */ - public void openCheckpointStream() throws Exception { - Preconditions.checkArgument(outStream == null, "Output stream for snapshot is already set."); - outStream = checkpointStreamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); - snapshotCloseableRegistry.registerCloseable(outStream); - outputView = new DataOutputViewStreamWrapper(outStream); - } + public void doRestore(Collection keyedStateHandles) + throws IOException, StateMigrationException, RocksDBException { - /** - * 3) Write the actual data from RocksDB from the time we took the snapshot object in (1). - * - * @throws IOException - */ - public void writeDBSnapshot() throws IOException, InterruptedException { + rocksDBKeyedStateBackend.createDB(); - if (null == snapshot) { - throw new IOException("No snapshot available. Might be released due to cancellation."); - } + for (KeyedStateHandle keyedStateHandle : keyedStateHandles) { + if (keyedStateHandle != null) { - Preconditions.checkNotNull(outStream, "No output stream to write snapshot."); - writeKVStateMetaData(); - writeKVStateData(); + if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { + throw new IllegalStateException("Unexpected state handle type, " + + "expected: " + KeyGroupsStateHandle.class + + ", but found: " + keyedStateHandle.getClass()); + } + this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; + restoreKeyGroupsInStateHandle(); + } + } } /** - * 4) Returns a state handle to the snapshot after the snapshot procedure is completed and null before. - * - * @return state handle to the completed snapshot + * Restore one key groups state handle. */ - public KeyGroupsStateHandle getSnapshotResultStateHandle() throws IOException { - - if (snapshotCloseableRegistry.unregisterCloseable(outStream)) { - - StreamStateHandle stateHandle = outStream.closeAndGetHandle(); - outStream = null; - - if (stateHandle != null) { - return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle); + private void restoreKeyGroupsInStateHandle() + throws IOException, StateMigrationException, RocksDBException { + try { + currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream(); + rocksDBKeyedStateBackend.cancelStreamRegistry.registerCloseable(currentStateHandleInStream); + currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream); + restoreKVStateMetaData(); + restoreKVStateData(); + } finally { + if (rocksDBKeyedStateBackend.cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) { + IOUtils.closeQuietly(currentStateHandleInStream); } } - return null; } /** - * 5) Release the snapshot object for RocksDB and clean up. + * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws RocksDBException */ - public void releaseSnapshotResources() { + private void restoreKVStateMetaData() throws IOException, StateMigrationException, RocksDBException { - outStream = null; + KeyedBackendSerializationProxy serializationProxy = + new KeyedBackendSerializationProxy<>(rocksDBKeyedStateBackend.userCodeClassLoader); - if (null != kvStateIterators) { - for (Tuple2 kvStateIterator : kvStateIterators) { - IOUtils.closeQuietly(kvStateIterator.f0); - } - kvStateIterators = null; - } + serializationProxy.read(currentStateHandleInView); - if (null != snapshot) { - if (null != stateBackend.db) { - stateBackend.db.releaseSnapshot(snapshot); - } - IOUtils.closeQuietly(snapshot); - snapshot = null; - } + // check for key serializer compatibility; this also reconfigures the + // key serializer to be compatible, if it is required and is possible + if (CompatibilityUtil.resolveCompatibilityResult( + serializationProxy.getKeySerializer(), + UnloadableDummyTypeSerializer.class, + serializationProxy.getKeySerializerConfigSnapshot(), + rocksDBKeyedStateBackend.keySerializer) + .isRequiresMigration()) { - if (null != readOptions) { - IOUtils.closeQuietly(readOptions); - readOptions = null; + // TODO replace with state migration; note that key hash codes need to remain the same after migration + throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + + "Aborting now since state migration is currently not available"); } - this.dbLease.close(); - } + this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ? + SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; - private void writeKVStateMetaData() throws IOException { + List> restoredMetaInfos = + serializationProxy.getStateMetaInfoSnapshots(); + currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size()); + //rocksDBKeyedStateBackend.restoredKvStateMetaInfos = new HashMap<>(restoredMetaInfos.size()); - List> metaInfoSnapshots = - new ArrayList<>(stateBackend.kvStateInformation.size()); + for (RegisteredKeyedBackendStateMetaInfo.Snapshot restoredMetaInfo : restoredMetaInfos) { - int kvStateId = 0; - for (Map.Entry>> column : - stateBackend.kvStateInformation.entrySet()) { + Tuple2> registeredColumn = + rocksDBKeyedStateBackend.kvStateInformation.get(restoredMetaInfo.getName()); - metaInfoSnapshots.add(column.getValue().f1.snapshot()); + if (registeredColumn == null) { + byte[] nameBytes = restoredMetaInfo.getName().getBytes(ConfigConstants.DEFAULT_CHARSET); - //retrieve iterator for this k/v states - readOptions = new ReadOptions(); - readOptions.setSnapshot(snapshot); - - kvStateIterators.add( - new Tuple2<>(stateBackend.db.newIterator(column.getValue().f0, readOptions), kvStateId)); + ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( + nameBytes, + rocksDBKeyedStateBackend.columnOptions); - ++kvStateId; - } + RegisteredKeyedBackendStateMetaInfo stateMetaInfo = + new RegisteredKeyedBackendStateMetaInfo<>( + restoredMetaInfo.getStateType(), + restoredMetaInfo.getName(), + restoredMetaInfo.getNamespaceSerializer(), + restoredMetaInfo.getStateSerializer()); - KeyedBackendSerializationProxy serializationProxy = - new KeyedBackendSerializationProxy<>( - stateBackend.getKeySerializer(), - metaInfoSnapshots, - !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, stateBackend.keyGroupCompressionDecorator)); + rocksDBKeyedStateBackend.restoredKvStateMetaInfos.put(restoredMetaInfo.getName(), restoredMetaInfo); - serializationProxy.write(outputView); - } + ColumnFamilyHandle columnFamily = rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor); - private void writeKVStateData() throws IOException, InterruptedException { + registeredColumn = new Tuple2<>(columnFamily, stateMetaInfo); + rocksDBKeyedStateBackend.kvStateInformation.put(stateMetaInfo.getName(), registeredColumn); - byte[] previousKey = null; - byte[] previousValue = null; - OutputStream kgOutStream = null; - DataOutputView kgOutView = null; + } else { + // TODO with eager state registration in place, check here for serializer migration strategies + } + currentStateHandleKVStateColumnFamilies.add(registeredColumn.f0); + } + } - try { - // Here we transfer ownership of RocksIterators to the RocksDBMergeIterator - try (RocksDBMergeIterator mergeIterator = new RocksDBMergeIterator( - kvStateIterators, stateBackend.keyGroupPrefixBytes)) { + /** + * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state handle. + * + * @throws IOException + * @throws RocksDBException + */ + private void restoreKVStateData() throws IOException, RocksDBException { + //for all key-groups in the current state handle... + for (Tuple2 keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) { + int keyGroup = keyGroupOffset.f0; - // handover complete, null out to prevent double close - kvStateIterators = null; + // Check that restored key groups all belong to the backend + Preconditions.checkState(rocksDBKeyedStateBackend.getKeyGroupRange().contains(keyGroup), + "The key group must belong to the backend"); - //preamble: setup with first key-group as our lookahead - if (mergeIterator.isValid()) { - //begin first key-group by recording the offset - keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos()); - //write the k/v-state id as metadata - kgOutStream = stateBackend.keyGroupCompressionDecorator.decorateWithCompression(outStream); - kgOutView = new DataOutputViewStreamWrapper(kgOutStream); + long offset = keyGroupOffset.f1; + //not empty key-group? + if (0L != offset) { + currentStateHandleInStream.seek(offset); + try (InputStream compressedKgIn = keygroupStreamCompressionDecorator.decorateWithCompression(currentStateHandleInStream)) { + DataInputViewStreamWrapper compressedKgInputView = new DataInputViewStreamWrapper(compressedKgIn); //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible - kgOutView.writeShort(mergeIterator.kvStateId()); - previousKey = mergeIterator.key(); - previousValue = mergeIterator.value(); - mergeIterator.next(); + int kvStateId = compressedKgInputView.readShort(); + ColumnFamilyHandle handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); + //insert all k/v pairs into DB + boolean keyGroupHasMoreKeys = true; + while (keyGroupHasMoreKeys) { + byte[] key = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView); + byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView); + if (RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(key)) { + //clear the signal bit in the key to make it ready for insertion again + RocksDBFullSnapshotOperation.clearMetaDataFollowsFlag(key); + rocksDBKeyedStateBackend.db.put(handle, key, value); + //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible + kvStateId = RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK + & compressedKgInputView.readShort(); + if (RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK == kvStateId) { + keyGroupHasMoreKeys = false; + } else { + handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); + } + } else { + rocksDBKeyedStateBackend.db.put(handle, key, value); + } + } } + } + } + } + } - //main loop: write k/v pairs ordered by (key-group, kv-state), thereby tracking key-group offsets. - while (mergeIterator.isValid()) { - - assert (!hasMetaDataFollowsFlag(previousKey)); + /** + * Encapsulates the process of restoring a RocksDBKeyedStateBackend from an incremental snapshot. + */ + private static class RocksDBIncrementalRestoreOperation { - //set signal in first key byte that meta data will follow in the stream after this k/v pair - if (mergeIterator.isNewKeyGroup() || mergeIterator.isNewKeyValueState()) { + private final RocksDBKeyedStateBackend stateBackend; - //be cooperative and check for interruption from time to time in the hot loop - checkInterrupted(); + private RocksDBIncrementalRestoreOperation(RocksDBKeyedStateBackend stateBackend) { + this.stateBackend = stateBackend; + } - setMetaDataFollowsFlagInKey(previousKey); - } + /** + * Root method that branches for different implementations of {@link KeyedStateHandle}. + */ + void restore(Collection restoreStateHandles) throws Exception { - writeKeyValuePair(previousKey, previousValue, kgOutView); + boolean hasExtraKeys = (restoreStateHandles.size() > 1 || + !Objects.equals(restoreStateHandles.iterator().next().getKeyGroupRange(), stateBackend.keyGroupRange)); - //write meta data if we have to - if (mergeIterator.isNewKeyGroup()) { - //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible - kgOutView.writeShort(END_OF_KEY_GROUP_MARK); - // this will just close the outer stream - kgOutStream.close(); - //begin new key-group - keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos()); - //write the kev-state - //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible - kgOutStream = stateBackend.keyGroupCompressionDecorator.decorateWithCompression(outStream); - kgOutView = new DataOutputViewStreamWrapper(kgOutStream); - kgOutView.writeShort(mergeIterator.kvStateId()); - } else if (mergeIterator.isNewKeyValueState()) { - //write the k/v-state - //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible - kgOutView.writeShort(mergeIterator.kvStateId()); - } + if (hasExtraKeys) { + stateBackend.createDB(); + } - //request next k/v pair - previousKey = mergeIterator.key(); - previousValue = mergeIterator.value(); - mergeIterator.next(); - } - } + for (KeyedStateHandle rawStateHandle : restoreStateHandles) { - //epilogue: write last key-group - if (previousKey != null) { - assert (!hasMetaDataFollowsFlag(previousKey)); - setMetaDataFollowsFlagInKey(previousKey); - writeKeyValuePair(previousKey, previousValue, kgOutView); - //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible - kgOutView.writeShort(END_OF_KEY_GROUP_MARK); - // this will just close the outer stream - kgOutStream.close(); - kgOutStream = null; + if (rawStateHandle instanceof IncrementalKeyedStateHandle) { + restoreInstance((IncrementalKeyedStateHandle) rawStateHandle, hasExtraKeys); + } else if (rawStateHandle instanceof IncrementalLocalKeyedStateHandle) { + Preconditions.checkState(!hasExtraKeys, "Cannot recover from local state after rescaling."); + restoreInstance((IncrementalLocalKeyedStateHandle) rawStateHandle); + } else { + throw new IllegalStateException("Unexpected state handle type, " + + "expected " + IncrementalKeyedStateHandle.class + + ", but found " + rawStateHandle.getClass()); } - - } finally { - // this will just close the outer stream - IOUtils.closeQuietly(kgOutStream); } } - private void writeKeyValuePair(byte[] key, byte[] value, DataOutputView out) throws IOException { - BytePrimitiveArraySerializer.INSTANCE.serialize(key, out); - BytePrimitiveArraySerializer.INSTANCE.serialize(value, out); - } + /** + * Recovery from remote incremental state. + */ + private void restoreInstance( + IncrementalKeyedStateHandle restoreStateHandle, + boolean hasExtraKeys) throws Exception { - static void setMetaDataFollowsFlagInKey(byte[] key) { - key[0] |= FIRST_BIT_IN_BYTE_MASK; - } + // read state data + Path temporaryRestoreInstancePath = new Path( + stateBackend.instanceBasePath.getAbsolutePath(), + UUID.randomUUID().toString()); - static void clearMetaDataFollowsFlag(byte[] key) { - key[0] &= (~RocksDBFullSnapshotOperation.FIRST_BIT_IN_BYTE_MASK); - } + try { - static boolean hasMetaDataFollowsFlag(byte[] key) { - return 0 != (key[0] & RocksDBFullSnapshotOperation.FIRST_BIT_IN_BYTE_MASK); - } + transferAllStateDataToDirectory(restoreStateHandle, temporaryRestoreInstancePath); - private static void checkInterrupted() throws InterruptedException { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException("RocksDB snapshot interrupted."); + // read meta data + List> stateMetaInfoSnapshots = + readMetaData(restoreStateHandle.getMetaStateHandle()); + + List columnFamilyDescriptors = + createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots); + + if (hasExtraKeys) { + restoreKeyGroupsShardWithTemporaryHelperInstance( + temporaryRestoreInstancePath, + columnFamilyDescriptors, + stateMetaInfoSnapshots); + } else { + + // since we transferred all remote state to a local directory, we can use the same code as for + // local recovery. + IncrementalLocalKeyedStateHandle localKeyedStateHandle = new IncrementalLocalKeyedStateHandle( + restoreStateHandle.getBackendIdentifier(), + restoreStateHandle.getCheckpointId(), + new DirectoryStateHandle(temporaryRestoreInstancePath), + restoreStateHandle.getKeyGroupRange(), + restoreStateHandle.getMetaStateHandle(), + restoreStateHandle.getSharedState().keySet()); + + restoreLocalStateIntoFullInstance( + localKeyedStateHandle, + columnFamilyDescriptors, + stateMetaInfoSnapshots); + } + } finally { + FileSystem restoreFileSystem = temporaryRestoreInstancePath.getFileSystem(); + if (restoreFileSystem.exists(temporaryRestoreInstancePath)) { + restoreFileSystem.delete(temporaryRestoreInstancePath, true); + } } } - } - private static final class RocksDBIncrementalSnapshotOperation { + /** + * Recovery from local incremental state. + */ + private void restoreInstance(IncrementalLocalKeyedStateHandle localKeyedStateHandle) throws Exception { + // read meta data + List> stateMetaInfoSnapshots = + readMetaData(localKeyedStateHandle.getMetaDataState()); - /** The backend which we snapshot. */ - private final RocksDBKeyedStateBackend stateBackend; + List columnFamilyDescriptors = + createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots); - /** Stream factory that creates the outpus streams to DFS. */ - private final CheckpointStreamFactory checkpointStreamFactory; + restoreLocalStateIntoFullInstance( + localKeyedStateHandle, + columnFamilyDescriptors, + stateMetaInfoSnapshots); + } - /** Id for the current checkpoint. */ - private final long checkpointId; + /** + * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot. + */ + private List createAndRegisterColumnFamilyDescriptors( + List> stateMetaInfoSnapshots) { - /** Timestamp for the current checkpoint. */ - private final long checkpointTimestamp; + List columnFamilyDescriptors = + new ArrayList<>(1 + stateMetaInfoSnapshots.size()); - /** All sst files that were part of the last previously completed checkpoint. */ - private Set baseSstFiles; + for (RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) { - /** The state meta data. */ - private final List> stateMetaInfoSnapshots = new ArrayList<>(); + ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( + stateMetaInfoSnapshot.getName().getBytes(ConfigConstants.DEFAULT_CHARSET), + stateBackend.columnOptions); + + columnFamilyDescriptors.add(columnFamilyDescriptor); + stateBackend.restoredKvStateMetaInfos.put(stateMetaInfoSnapshot.getName(), stateMetaInfoSnapshot); + } + return columnFamilyDescriptors; + } + + /** + * This method implements the core of the restore logic that unifies how local and remote state are recovered. + */ + private void restoreLocalStateIntoFullInstance( + IncrementalLocalKeyedStateHandle restoreStateHandle, + List columnFamilyDescriptors, + List> stateMetaInfoSnapshots) throws Exception { + // pick up again the old backend id, so the we can reference existing state + stateBackend.backendUID = restoreStateHandle.getBackendIdentifier(); + + LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.", + stateBackend.operatorIdentifier, stateBackend.backendUID); + + // create hard links in the instance directory + if (!stateBackend.instanceRocksDBPath.mkdirs()) { + throw new IOException("Could not create RocksDB data directory."); + } - /** Local filesystem for the RocksDB backup. */ - private FileSystem backupFileSystem; + Path restoreSourcePath = restoreStateHandle.getDirectoryStateHandle().getDirectory(); + restoreInstanceDirectoryFromPath(restoreSourcePath); - /** Local path for the RocksDB backup. */ - private Path backupPath; + List columnFamilyHandles = + new ArrayList<>(1 + columnFamilyDescriptors.size()); - // Registry for all opened i/o streams - private final CloseableRegistry closeableRegistry = new CloseableRegistry(); + stateBackend.db = stateBackend.openDB( + stateBackend.instanceRocksDBPath.getAbsolutePath(), + columnFamilyDescriptors, columnFamilyHandles); - // new sst files since the last completed checkpoint - private final Map sstFiles = new HashMap<>(); + // extract and store the default column family which is located at the last index + stateBackend.defaultColumnFamily = columnFamilyHandles.remove(columnFamilyHandles.size() - 1); - // handles to the misc files in the current snapshot - private final Map miscFiles = new HashMap<>(); + for (int i = 0; i < columnFamilyDescriptors.size(); ++i) { + RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i); - // This lease protects from concurrent disposal of the native rocksdb instance. - private final ResourceGuard.Lease dbLease; + ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i); + RegisteredKeyedBackendStateMetaInfo stateMetaInfo = + new RegisteredKeyedBackendStateMetaInfo<>( + stateMetaInfoSnapshot.getStateType(), + stateMetaInfoSnapshot.getName(), + stateMetaInfoSnapshot.getNamespaceSerializer(), + stateMetaInfoSnapshot.getStateSerializer()); - private StreamStateHandle metaStateHandle = null; + stateBackend.kvStateInformation.put( + stateMetaInfoSnapshot.getName(), + new Tuple2<>(columnFamilyHandle, stateMetaInfo)); + } - private RocksDBIncrementalSnapshotOperation( - RocksDBKeyedStateBackend stateBackend, - CheckpointStreamFactory checkpointStreamFactory, - long checkpointId, - long checkpointTimestamp) throws IOException { + // use the restore sst files as the base for succeeding checkpoints + synchronized (stateBackend.materializedSstFiles) { + stateBackend.materializedSstFiles.put( + restoreStateHandle.getCheckpointId(), + restoreStateHandle.getSharedStateHandleIDs()); + } - this.stateBackend = stateBackend; - this.checkpointStreamFactory = checkpointStreamFactory; - this.checkpointId = checkpointId; - this.checkpointTimestamp = checkpointTimestamp; - this.dbLease = this.stateBackend.rocksDBResourceGuard.acquireResource(); + stateBackend.lastCompletedCheckpointId = restoreStateHandle.getCheckpointId(); } - private StreamStateHandle materializeStateData(Path filePath) throws Exception { - FSDataInputStream inputStream = null; - CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null; + /** + * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from + * a local state. + */ + private void restoreInstanceDirectoryFromPath(Path source) throws IOException { - try { - final byte[] buffer = new byte[8 * 1024]; + FileSystem fileSystem = source.getFileSystem(); - FileSystem backupFileSystem = backupPath.getFileSystem(); - inputStream = backupFileSystem.open(filePath); - closeableRegistry.registerCloseable(inputStream); + final FileStatus[] fileStatuses = fileSystem.listStatus(source); - outputStream = checkpointStreamFactory - .createCheckpointStateOutputStream(CheckpointedStateScope.SHARED); - closeableRegistry.registerCloseable(outputStream); - - while (true) { - int numBytes = inputStream.read(buffer); - - if (numBytes == -1) { - break; - } - - outputStream.write(buffer, 0, numBytes); - } - - StreamStateHandle result = null; - if (closeableRegistry.unregisterCloseable(outputStream)) { - result = outputStream.closeAndGetHandle(); - outputStream = null; - } - return result; - - } finally { - - if (closeableRegistry.unregisterCloseable(inputStream)) { - inputStream.close(); - } + if (fileStatuses == null) { + throw new IOException("Cannot list file statues. Directory " + source + " does not exist."); + } - if (closeableRegistry.unregisterCloseable(outputStream)) { - outputStream.close(); + for (FileStatus fileStatus : fileStatuses) { + final Path filePath = fileStatus.getPath(); + final String fileName = filePath.getName(); + File restoreFile = new File(source.getPath(), fileName); + File targetFile = new File(stateBackend.instanceRocksDBPath.getPath(), fileName); + if (fileName.endsWith(SST_FILE_SUFFIX)) { + // hardlink'ing the immutable sst-files. + Files.createLink(targetFile.toPath(), restoreFile.toPath()); + } else { + // true copy for all other files. + Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } } } - private StreamStateHandle materializeMetaData() throws Exception { - CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null; + /** + * Reads Flink's state meta data file from the state handle. + */ + private List> readMetaData( + StreamStateHandle metaStateHandle) throws Exception { - try { - outputStream = checkpointStreamFactory - .createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); - closeableRegistry.registerCloseable(outputStream); + FSDataInputStream inputStream = null; - //no need for compression scheme support because sst-files are already compressed - KeyedBackendSerializationProxy serializationProxy = - new KeyedBackendSerializationProxy<>( - stateBackend.keySerializer, - stateMetaInfoSnapshots, - false); + try { + inputStream = metaStateHandle.openInputStream(); + stateBackend.cancelStreamRegistry.registerCloseable(inputStream); - DataOutputView out = new DataOutputViewStreamWrapper(outputStream); + KeyedBackendSerializationProxy serializationProxy = + new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); + DataInputView in = new DataInputViewStreamWrapper(inputStream); + serializationProxy.read(in); - serializationProxy.write(out); + // check for key serializer compatibility; this also reconfigures the + // key serializer to be compatible, if it is required and is possible + if (CompatibilityUtil.resolveCompatibilityResult( + serializationProxy.getKeySerializer(), + UnloadableDummyTypeSerializer.class, + serializationProxy.getKeySerializerConfigSnapshot(), + stateBackend.keySerializer) + .isRequiresMigration()) { - StreamStateHandle result = null; - if (closeableRegistry.unregisterCloseable(outputStream)) { - result = outputStream.closeAndGetHandle(); - outputStream = null; + // TODO replace with state migration; note that key hash codes need to remain the same after migration + throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + + "Aborting now since state migration is currently not available"); } - return result; + + return serializationProxy.getStateMetaInfoSnapshots(); } finally { - if (outputStream != null) { - if (closeableRegistry.unregisterCloseable(outputStream)) { - outputStream.close(); - } + if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) { + inputStream.close(); } } } - void takeSnapshot() throws Exception { - - final long lastCompletedCheckpoint; - - // use the last completed checkpoint as the comparison base. - synchronized (stateBackend.materializedSstFiles) { - lastCompletedCheckpoint = stateBackend.lastCompletedCheckpointId; - baseSstFiles = stateBackend.materializedSstFiles.get(lastCompletedCheckpoint); - } - - LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " + - "assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles); + private void transferAllStateDataToDirectory( + IncrementalKeyedStateHandle restoreStateHandle, + Path dest) throws IOException { - // save meta data - for (Map.Entry>> stateMetaInfoEntry - : stateBackend.kvStateInformation.entrySet()) { - stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().f1.snapshot()); - } + final Map sstFiles = + restoreStateHandle.getSharedState(); + final Map miscFiles = + restoreStateHandle.getPrivateState(); - // save state data - backupPath = new Path(stateBackend.instanceBasePath.getAbsolutePath(), "chk-" + checkpointId); + transferAllDataFromStateHandles(sstFiles, dest); + transferAllDataFromStateHandles(miscFiles, dest); + } - LOG.trace("Local RocksDB checkpoint goes to backup path {}.", backupPath); + /** + * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their + * {@link StateHandleID}. + */ + private void transferAllDataFromStateHandles( + Map stateHandleMap, + Path restoreInstancePath) throws IOException { - backupFileSystem = backupPath.getFileSystem(); - if (backupFileSystem.exists(backupPath)) { - throw new IllegalStateException("Unexpected existence of the backup directory."); + for (Map.Entry entry : stateHandleMap.entrySet()) { + StateHandleID stateHandleID = entry.getKey(); + StreamStateHandle remoteFileHandle = entry.getValue(); + copyStateDataHandleData(new Path(restoreInstancePath, stateHandleID.toString()), remoteFileHandle); } - - // create hard links of living files in the checkpoint path - Checkpoint checkpoint = Checkpoint.create(stateBackend.db); - checkpoint.createCheckpoint(backupPath.getPath()); } - KeyedStateHandle materializeSnapshot() throws Exception { - - stateBackend.cancelStreamRegistry.registerCloseable(closeableRegistry); + /** + * Copies the file from a single state handle to the given path. + */ + private void copyStateDataHandleData( + Path restoreFilePath, + StreamStateHandle remoteFileHandle) throws IOException { - // write meta data - metaStateHandle = materializeMetaData(); + FileSystem restoreFileSystem = restoreFilePath.getFileSystem(); - // write state data - Preconditions.checkState(backupFileSystem.exists(backupPath)); + FSDataInputStream inputStream = null; + FSDataOutputStream outputStream = null; - FileStatus[] fileStatuses = backupFileSystem.listStatus(backupPath); - if (fileStatuses != null) { - for (FileStatus fileStatus : fileStatuses) { - final Path filePath = fileStatus.getPath(); - final String fileName = filePath.getName(); - final StateHandleID stateHandleID = new StateHandleID(fileName); + try { + inputStream = remoteFileHandle.openInputStream(); + stateBackend.cancelStreamRegistry.registerCloseable(inputStream); - if (fileName.endsWith(SST_FILE_SUFFIX)) { - final boolean existsAlready = - baseSstFiles != null && baseSstFiles.contains(stateHandleID); + outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE); + stateBackend.cancelStreamRegistry.registerCloseable(outputStream); - if (existsAlready) { - // we introduce a placeholder state handle, that is replaced with the - // original from the shared state registry (created from a previous checkpoint) - sstFiles.put( - stateHandleID, - new PlaceholderStreamStateHandle()); - } else { - sstFiles.put(stateHandleID, materializeStateData(filePath)); - } - } else { - StreamStateHandle fileHandle = materializeStateData(filePath); - miscFiles.put(stateHandleID, fileHandle); + byte[] buffer = new byte[8 * 1024]; + while (true) { + int numBytes = inputStream.read(buffer); + if (numBytes == -1) { + break; } - } - } - - synchronized (stateBackend.materializedSstFiles) { - stateBackend.materializedSstFiles.put(checkpointId, sstFiles.keySet()); - } - - return new IncrementalKeyedStateHandle( - stateBackend.backendUID, - stateBackend.keyGroupRange, - checkpointId, - sstFiles, - miscFiles, - metaStateHandle); - } - void stop() { + outputStream.write(buffer, 0, numBytes); + } + } finally { + if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) { + inputStream.close(); + } - if (stateBackend.cancelStreamRegistry.unregisterCloseable(closeableRegistry)) { - try { - closeableRegistry.close(); - } catch (IOException e) { - LOG.warn("Could not properly close io streams.", e); + if (stateBackend.cancelStreamRegistry.unregisterCloseable(outputStream)) { + outputStream.close(); } } } - void releaseResources(boolean canceled) { + /** + * In case of rescaling, this method creates a temporary RocksDB instance for a key-groups shard. All contents + * from the temporary instance are copied into the real restore instance and then the temporary instance is + * discarded. + */ + private void restoreKeyGroupsShardWithTemporaryHelperInstance( + Path restoreInstancePath, + List columnFamilyDescriptors, + List> stateMetaInfoSnapshots) throws Exception { - dbLease.close(); + List columnFamilyHandles = + new ArrayList<>(1 + columnFamilyDescriptors.size()); - if (stateBackend.cancelStreamRegistry.unregisterCloseable(closeableRegistry)) { - try { - closeableRegistry.close(); - } catch (IOException e) { - LOG.warn("Exception on closing registry.", e); - } - } + try (RocksDB restoreDb = stateBackend.openDB( + restoreInstancePath.getPath(), + columnFamilyDescriptors, + columnFamilyHandles)) { - if (backupPath != null) { try { - if (backupFileSystem.exists(backupPath)) { - - LOG.trace("Deleting local RocksDB backup path {}.", backupPath); - backupFileSystem.delete(backupPath, true); - } - } catch (Exception e) { - LOG.warn("Could not properly delete the checkpoint directory.", e); - } - } + // iterating only the requested descriptors automatically skips the default column family handle + for (int i = 0; i < columnFamilyDescriptors.size(); ++i) { + ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i); + ColumnFamilyDescriptor columnFamilyDescriptor = columnFamilyDescriptors.get(i); + RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i); - if (canceled) { - Collection statesToDiscard = - new ArrayList<>(1 + miscFiles.size() + sstFiles.size()); + Tuple2> registeredStateMetaInfoEntry = + stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName()); - statesToDiscard.add(metaStateHandle); - statesToDiscard.addAll(miscFiles.values()); - statesToDiscard.addAll(sstFiles.values()); + if (null == registeredStateMetaInfoEntry) { - try { - StateUtil.bestEffortDiscardAllStateObjects(statesToDiscard); - } catch (Exception e) { - LOG.warn("Could not properly discard states.", e); - } - } - } - } + RegisteredKeyedBackendStateMetaInfo stateMetaInfo = + new RegisteredKeyedBackendStateMetaInfo<>( + stateMetaInfoSnapshot.getStateType(), + stateMetaInfoSnapshot.getName(), + stateMetaInfoSnapshot.getNamespaceSerializer(), + stateMetaInfoSnapshot.getStateSerializer()); - @Override - public void restore(Collection restoreState) throws Exception { - LOG.info("Initializing RocksDB keyed state backend from snapshot."); + registeredStateMetaInfoEntry = + new Tuple2<>( + stateBackend.db.createColumnFamily(columnFamilyDescriptor), + stateMetaInfo); - if (LOG.isDebugEnabled()) { - LOG.debug("Restoring snapshot from state handles: {}.", restoreState); - } + stateBackend.kvStateInformation.put( + stateMetaInfoSnapshot.getName(), + registeredStateMetaInfoEntry); + } - // clear all meta data - kvStateInformation.clear(); - restoredKvStateMetaInfos.clear(); + ColumnFamilyHandle targetColumnFamilyHandle = registeredStateMetaInfoEntry.f0; - try { - if (restoreState == null || restoreState.isEmpty()) { - createDB(); - } else if (restoreState.iterator().next() instanceof IncrementalKeyedStateHandle) { - RocksDBIncrementalRestoreOperation restoreOperation = new RocksDBIncrementalRestoreOperation<>(this); - restoreOperation.restore(restoreState); - } else { - RocksDBFullRestoreOperation restoreOperation = new RocksDBFullRestoreOperation<>(this); - restoreOperation.doRestore(restoreState); - } - } catch (Exception ex) { - dispose(); - throw ex; - } - } + try (RocksIterator iterator = restoreDb.newIterator(columnFamilyHandle)) { - @Override - public void notifyCheckpointComplete(long completedCheckpointId) { + int startKeyGroup = stateBackend.getKeyGroupRange().getStartKeyGroup(); + byte[] startKeyGroupPrefixBytes = new byte[stateBackend.keyGroupPrefixBytes]; + for (int j = 0; j < stateBackend.keyGroupPrefixBytes; ++j) { + startKeyGroupPrefixBytes[j] = (byte) (startKeyGroup >>> ((stateBackend.keyGroupPrefixBytes - j - 1) * Byte.SIZE)); + } - if (!enableIncrementalCheckpointing) { - return; - } + iterator.seek(startKeyGroupPrefixBytes); - synchronized (materializedSstFiles) { + while (iterator.isValid()) { - if (completedCheckpointId < lastCompletedCheckpointId) { - return; - } + int keyGroup = 0; + for (int j = 0; j < stateBackend.keyGroupPrefixBytes; ++j) { + keyGroup = (keyGroup << Byte.SIZE) + iterator.key()[j]; + } - materializedSstFiles.keySet().removeIf(checkpointId -> checkpointId < completedCheckpointId); + if (stateBackend.keyGroupRange.contains(keyGroup)) { + stateBackend.db.put(targetColumnFamilyHandle, + iterator.key(), iterator.value()); + } - lastCompletedCheckpointId = completedCheckpointId; + iterator.next(); + } + } // releases native iterator resources + } + } finally { + //release native tmp db column family resources + for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { + IOUtils.closeQuietly(columnFamilyHandle); + } + } + } // releases native tmp db resources } } - private void createDB() throws IOException { - List columnFamilyHandles = new ArrayList<>(1); - this.db = openDB(instanceRocksDBPath.getAbsolutePath(), Collections.emptyList(), columnFamilyHandles); - this.defaultColumnFamily = columnFamilyHandles.get(0); - } - - private RocksDB openDB( - String path, - List stateColumnFamilyDescriptors, - List stateColumnFamilyHandles) throws IOException { + // ------------------------------------------------------------------------ + // State factories + // ------------------------------------------------------------------------ - List columnFamilyDescriptors = - new ArrayList<>(1 + stateColumnFamilyDescriptors.size()); + /** + * Creates a column family handle for use with a k/v state. When restoring from a snapshot + * we don't restore the individual k/v states, just the global RocksDB database and the + * list of column families. When a k/v state is first requested we check here whether we + * already have a column family for that and return it or create a new one if it doesn't exist. + * + *

This also checks whether the {@link StateDescriptor} for a state matches the one + * that we checkpointed, i.e. is already in the map of column families. + */ + @SuppressWarnings("rawtypes, unchecked") + protected ColumnFamilyHandle getColumnFamily( + StateDescriptor descriptor, TypeSerializer namespaceSerializer) throws IOException, StateMigrationException { - columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors); + Tuple2> stateInfo = + kvStateInformation.get(descriptor.getName()); - // we add the required descriptor for the default CF in last position. - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_COLUMN_FAMILY_NAME_BYTES, columnOptions)); + RegisteredKeyedBackendStateMetaInfo newMetaInfo = new RegisteredKeyedBackendStateMetaInfo<>( + descriptor.getType(), + descriptor.getName(), + namespaceSerializer, + descriptor.getSerializer()); - RocksDB dbRef; + if (stateInfo != null) { + // TODO with eager registration in place, these checks should be moved to restore() + + RegisteredKeyedBackendStateMetaInfo.Snapshot restoredMetaInfo = + (RegisteredKeyedBackendStateMetaInfo.Snapshot) restoredKvStateMetaInfos.get(descriptor.getName()); + + Preconditions.checkState( + Objects.equals(newMetaInfo.getName(), restoredMetaInfo.getName()), + "Incompatible state names. " + + "Was [" + restoredMetaInfo.getName() + "], " + + "registered with [" + newMetaInfo.getName() + "]."); + + if (!Objects.equals(newMetaInfo.getStateType(), StateDescriptor.Type.UNKNOWN) + && !Objects.equals(restoredMetaInfo.getStateType(), StateDescriptor.Type.UNKNOWN)) { + + Preconditions.checkState( + newMetaInfo.getStateType() == restoredMetaInfo.getStateType(), + "Incompatible state types. " + + "Was [" + restoredMetaInfo.getStateType() + "], " + + "registered with [" + newMetaInfo.getStateType() + "]."); + } + + // check compatibility results to determine if state migration is required + CompatibilityResult namespaceCompatibility = CompatibilityUtil.resolveCompatibilityResult( + restoredMetaInfo.getNamespaceSerializer(), + null, + restoredMetaInfo.getNamespaceSerializerConfigSnapshot(), + newMetaInfo.getNamespaceSerializer()); + + CompatibilityResult stateCompatibility = CompatibilityUtil.resolveCompatibilityResult( + restoredMetaInfo.getStateSerializer(), + UnloadableDummyTypeSerializer.class, + restoredMetaInfo.getStateSerializerConfigSnapshot(), + newMetaInfo.getStateSerializer()); + + if (namespaceCompatibility.isRequiresMigration() || stateCompatibility.isRequiresMigration()) { + // TODO state migration currently isn't possible. + throw new StateMigrationException("State migration isn't supported, yet."); + } else { + stateInfo.f1 = newMetaInfo; + return stateInfo.f0; + } + } + + byte[] nameBytes = descriptor.getName().getBytes(ConfigConstants.DEFAULT_CHARSET); + Preconditions.checkState(!Arrays.equals(DEFAULT_COLUMN_FAMILY_NAME_BYTES, nameBytes), + "The chosen state name 'default' collides with the name of the default column family!"); + + ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); + + final ColumnFamilyHandle columnFamily; try { - dbRef = RocksDB.open( - Preconditions.checkNotNull(dbOptions), - Preconditions.checkNotNull(path), - columnFamilyDescriptors, - stateColumnFamilyHandles); + columnFamily = db.createColumnFamily(columnDescriptor); } catch (RocksDBException e) { - throw new IOException("Error while opening RocksDB instance.", e); + throw new IOException("Error creating ColumnFamilyHandle.", e); } - // requested + default CF - Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(), - "Not all requested column family handles have been created"); + Tuple2> tuple = + new Tuple2<>(columnFamily, newMetaInfo); + Map rawAccess = kvStateInformation; + rawAccess.put(descriptor.getName(), tuple); + return columnFamily; + } - return dbRef; + @Override + protected InternalValueState createValueState( + TypeSerializer namespaceSerializer, + ValueStateDescriptor stateDesc) throws Exception { + + ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + + return new RocksDBValueState<>(columnFamily, namespaceSerializer, stateDesc, this); + } + + @Override + protected InternalListState createListState( + TypeSerializer namespaceSerializer, + ListStateDescriptor stateDesc) throws Exception { + + ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + + return new RocksDBListState<>(columnFamily, namespaceSerializer, stateDesc, this); + } + + @Override + protected InternalReducingState createReducingState( + TypeSerializer namespaceSerializer, + ReducingStateDescriptor stateDesc) throws Exception { + + ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + + return new RocksDBReducingState<>(columnFamily, namespaceSerializer, stateDesc, this); + } + + @Override + protected InternalAggregatingState createAggregatingState( + TypeSerializer namespaceSerializer, + AggregatingStateDescriptor stateDesc) throws Exception { + + ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + return new RocksDBAggregatingState<>(columnFamily, namespaceSerializer, stateDesc, this); + } + + @Override + protected InternalFoldingState createFoldingState( + TypeSerializer namespaceSerializer, + FoldingStateDescriptor stateDesc) throws Exception { + + ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + + return new RocksDBFoldingState<>(columnFamily, namespaceSerializer, stateDesc, this); + } + + @Override + protected InternalMapState createMapState( + TypeSerializer namespaceSerializer, + MapStateDescriptor stateDesc) throws Exception { + + ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + + return new RocksDBMapState<>(columnFamily, namespaceSerializer, stateDesc, this); } /** - * Encapsulates the process of restoring a RocksDBKeyedStateBackend from a snapshot. + * Only visible for testing, DO NOT USE. */ - static final class RocksDBFullRestoreOperation { + public File getInstanceBasePath() { + return instanceBasePath; + } - private final RocksDBKeyedStateBackend rocksDBKeyedStateBackend; + @Override + public boolean supportsAsynchronousSnapshots() { + return true; + } - /** Current key-groups state handle from which we restore key-groups. */ - private KeyGroupsStateHandle currentKeyGroupsStateHandle; - /** Current input stream we obtained from currentKeyGroupsStateHandle. */ - private FSDataInputStream currentStateHandleInStream; - /** Current data input view that wraps currentStateHandleInStream. */ - private DataInputView currentStateHandleInView; - /** Current list of ColumnFamilyHandles for all column families we restore from currentKeyGroupsStateHandle. */ - private List currentStateHandleKVStateColumnFamilies; - /** The compression decorator that was used for writing the state, as determined by the meta data. */ - private StreamCompressionDecorator keygroupStreamCompressionDecorator; + @VisibleForTesting + @SuppressWarnings("unchecked") + @Override + public int numStateEntries() { + int count = 0; + + for (Tuple2> column : kvStateInformation.values()) { + try (RocksIterator rocksIterator = db.newIterator(column.f0)) { + rocksIterator.seekToFirst(); + + while (rocksIterator.isValid()) { + count++; + rocksIterator.next(); + } + } + } + + return count; + } + + + + /** + * Iterator that merges multiple RocksDB iterators to partition all states into contiguous key-groups. + * The resulting iteration sequence is ordered by (key-group, kv-state). + */ + @VisibleForTesting + static final class RocksDBMergeIterator implements AutoCloseable { + + private final PriorityQueue heap; + private final int keyGroupPrefixByteCount; + private boolean newKeyGroup; + private boolean newKVState; + private boolean valid; + + private MergeIterator currentSubIterator; + + private static final List> COMPARATORS; + + static { + int maxBytes = 4; + COMPARATORS = new ArrayList<>(maxBytes); + for (int i = 0; i < maxBytes; ++i) { + final int currentBytes = i; + COMPARATORS.add(new Comparator() { + @Override + public int compare(MergeIterator o1, MergeIterator o2) { + int arrayCmpRes = compareKeyGroupsForByteArrays( + o1.currentKey, o2.currentKey, currentBytes); + return arrayCmpRes == 0 ? o1.getKvStateId() - o2.getKvStateId() : arrayCmpRes; + } + }); + } + } + + RocksDBMergeIterator(List> kvStateIterators, final int keyGroupPrefixByteCount) { + Preconditions.checkNotNull(kvStateIterators); + this.keyGroupPrefixByteCount = keyGroupPrefixByteCount; + + Comparator iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount); + + if (kvStateIterators.size() > 0) { + PriorityQueue iteratorPriorityQueue = + new PriorityQueue<>(kvStateIterators.size(), iteratorComparator); + + for (Tuple2 rocksIteratorWithKVStateId : kvStateIterators) { + final RocksIterator rocksIterator = rocksIteratorWithKVStateId.f0; + rocksIterator.seekToFirst(); + if (rocksIterator.isValid()) { + iteratorPriorityQueue.offer(new MergeIterator(rocksIterator, rocksIteratorWithKVStateId.f1)); + } else { + IOUtils.closeQuietly(rocksIterator); + } + } + + kvStateIterators.clear(); + + this.heap = iteratorPriorityQueue; + this.valid = !heap.isEmpty(); + this.currentSubIterator = heap.poll(); + } else { + // creating a PriorityQueue of size 0 results in an exception. + this.heap = null; + this.valid = false; + } + + this.newKeyGroup = true; + this.newKVState = true; + } /** - * Creates a restore operation object for the given state backend instance. - * - * @param rocksDBKeyedStateBackend the state backend into which we restore + * Advance the iterator. Should only be called if {@link #isValid()} returned true. Valid can only chance after + * calls to {@link #next()}. */ - public RocksDBFullRestoreOperation(RocksDBKeyedStateBackend rocksDBKeyedStateBackend) { - this.rocksDBKeyedStateBackend = Preconditions.checkNotNull(rocksDBKeyedStateBackend); + public void next() { + newKeyGroup = false; + newKVState = false; + + final RocksIterator rocksIterator = currentSubIterator.getIterator(); + rocksIterator.next(); + + byte[] oldKey = currentSubIterator.getCurrentKey(); + if (rocksIterator.isValid()) { + currentSubIterator.currentKey = rocksIterator.key(); + + if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) { + heap.offer(currentSubIterator); + currentSubIterator = heap.poll(); + newKVState = currentSubIterator.getIterator() != rocksIterator; + detectNewKeyGroup(oldKey); + } + } else { + IOUtils.closeQuietly(rocksIterator); + + if (heap.isEmpty()) { + currentSubIterator = null; + valid = false; + } else { + currentSubIterator = heap.poll(); + newKVState = true; + detectNewKeyGroup(oldKey); + } + } + } + + private boolean isDifferentKeyGroup(byte[] a, byte[] b) { + return 0 != compareKeyGroupsForByteArrays(a, b, keyGroupPrefixByteCount); + } + + private void detectNewKeyGroup(byte[] oldKey) { + if (isDifferentKeyGroup(oldKey, currentSubIterator.currentKey)) { + newKeyGroup = true; + } } /** - * Restores all key-groups data that is referenced by the passed state handles. - * - * @param keyedStateHandles List of all key groups state handles that shall be restored. + * @return key-group for the current key */ - public void doRestore(Collection keyedStateHandles) - throws IOException, StateMigrationException, RocksDBException { + public int keyGroup() { + int result = 0; + //big endian decode + for (int i = 0; i < keyGroupPrefixByteCount; ++i) { + result <<= 8; + result |= (currentSubIterator.currentKey[i] & 0xFF); + } + return result; + } - rocksDBKeyedStateBackend.createDB(); + public byte[] key() { + return currentSubIterator.getCurrentKey(); + } + + public byte[] value() { + return currentSubIterator.getIterator().value(); + } + + /** + * @return Id of K/V state to which the current key belongs. + */ + public int kvStateId() { + return currentSubIterator.getKvStateId(); + } + + /** + * Indicates if current key starts a new k/v-state, i.e. belong to a different k/v-state than it's predecessor. + * @return true iff the current key belong to a different k/v-state than it's predecessor. + */ + public boolean isNewKeyValueState() { + return newKVState; + } + + /** + * Indicates if current key starts a new key-group, i.e. belong to a different key-group than it's predecessor. + * @return true iff the current key belong to a different key-group than it's predecessor. + */ + public boolean isNewKeyGroup() { + return newKeyGroup; + } + + /** + * Check if the iterator is still valid. Getters like {@link #key()}, {@link #value()}, etc. as well as + * {@link #next()} should only be called if valid returned true. Should be checked after each call to + * {@link #next()} before accessing iterator state. + * @return True iff this iterator is valid. + */ + public boolean isValid() { + return valid; + } + + private static int compareKeyGroupsForByteArrays(byte[] a, byte[] b, int len) { + for (int i = 0; i < len; ++i) { + int diff = (a[i] & 0xFF) - (b[i] & 0xFF); + if (diff != 0) { + return diff; + } + } + return 0; + } + + @Override + public void close() { + IOUtils.closeQuietly(currentSubIterator); + currentSubIterator = null; + + IOUtils.closeAllQuietly(heap); + heap.clear(); + } + } + + /** + * Wraps a RocksDB iterator to cache it's current key and assigns an id for the key/value state to the iterator. + * Used by #MergeIterator. + */ + private static final class MergeIterator implements AutoCloseable { + + /** + * @param iterator The #RocksIterator to wrap . + * @param kvStateId Id of the K/V state to which this iterator belongs. + */ + MergeIterator(RocksIterator iterator, int kvStateId) { + this.iterator = Preconditions.checkNotNull(iterator); + this.currentKey = iterator.key(); + this.kvStateId = kvStateId; + } + + private final RocksIterator iterator; + private byte[] currentKey; + private final int kvStateId; + + public byte[] getCurrentKey() { + return currentKey; + } + + public void setCurrentKey(byte[] currentKey) { + this.currentKey = currentKey; + } + + public RocksIterator getIterator() { + return iterator; + } + + public int getKvStateId() { + return kvStateId; + } + + @Override + public void close() { + IOUtils.closeQuietly(iterator); + } + } + + /** + * Adapter class to bridge between {@link RocksIterator} and {@link Iterator} to iterate over the keys. This class + * is not thread safe. + * + * @param the type of the iterated objects, which are keys in RocksDB. + */ + static class RocksIteratorForKeysWrapper implements Iterator, AutoCloseable { + private final RocksIterator iterator; + private final String state; + private final TypeSerializer keySerializer; + private final int keyGroupPrefixBytes; + private final byte[] namespaceBytes; + private final boolean ambiguousKeyPossible; + private K nextKey; + + RocksIteratorForKeysWrapper( + RocksIterator iterator, + String state, + TypeSerializer keySerializer, + int keyGroupPrefixBytes, + boolean ambiguousKeyPossible, + byte[] namespaceBytes) { + this.iterator = Preconditions.checkNotNull(iterator); + this.state = Preconditions.checkNotNull(state); + this.keySerializer = Preconditions.checkNotNull(keySerializer); + this.keyGroupPrefixBytes = Preconditions.checkNotNull(keyGroupPrefixBytes); + this.namespaceBytes = Preconditions.checkNotNull(namespaceBytes); + this.nextKey = null; + this.ambiguousKeyPossible = ambiguousKeyPossible; + } + + @Override + public boolean hasNext() { + while (nextKey == null && iterator.isValid()) { + try { + byte[] key = iterator.key(); + if (isMatchingNameSpace(key)) { + ByteArrayInputStreamWithPos inputStream = + new ByteArrayInputStreamWithPos(key, keyGroupPrefixBytes, key.length - keyGroupPrefixBytes); + DataInputViewStreamWrapper dataInput = new DataInputViewStreamWrapper(inputStream); + K value = RocksDBKeySerializationUtils.readKey( + keySerializer, + inputStream, + dataInput, + ambiguousKeyPossible); + nextKey = value; + } + iterator.next(); + } catch (IOException e) { + throw new FlinkRuntimeException("Failed to access state [" + state + "]", e); + } + } + return nextKey != null; + } + + @Override + public K next() { + if (!hasNext()) { + throw new NoSuchElementException("Failed to access state [" + state + "]"); + } - for (KeyedStateHandle keyedStateHandle : keyedStateHandles) { - if (keyedStateHandle != null) { + K tmpKey = nextKey; + nextKey = null; + return tmpKey; + } - if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) { - throw new IllegalStateException("Unexpected state handle type, " + - "expected: " + KeyGroupsStateHandle.class + - ", but found: " + keyedStateHandle.getClass()); + private boolean isMatchingNameSpace(@Nonnull byte[] key) { + final int namespaceBytesLength = namespaceBytes.length; + final int basicLength = namespaceBytesLength + keyGroupPrefixBytes; + if (key.length >= basicLength) { + for (int i = 1; i <= namespaceBytesLength; ++i) { + if (key[key.length - i] != namespaceBytes[namespaceBytesLength - i]) { + return false; } - this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle; - restoreKeyGroupsInStateHandle(); } + return true; } + return false; } - /** - * Restore one key groups state handle. - */ - private void restoreKeyGroupsInStateHandle() - throws IOException, StateMigrationException, RocksDBException { - try { - currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream(); - rocksDBKeyedStateBackend.cancelStreamRegistry.registerCloseable(currentStateHandleInStream); - currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream); - restoreKVStateMetaData(); - restoreKVStateData(); - } finally { - if (rocksDBKeyedStateBackend.cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) { - IOUtils.closeQuietly(currentStateHandleInStream); - } - } + @Override + public void close() { + iterator.close(); } + } - /** - * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle. - * - * @throws IOException - * @throws ClassNotFoundException - * @throws RocksDBException - */ - private void restoreKVStateMetaData() throws IOException, StateMigrationException, RocksDBException { + private class FullSnapshotStrategy implements SnapshotStrategy> { - KeyedBackendSerializationProxy serializationProxy = - new KeyedBackendSerializationProxy<>(rocksDBKeyedStateBackend.userCodeClassLoader); + @Override + public RunnableFuture> performSnapshot( + long checkpointId, + long timestamp, + CheckpointStreamFactory primaryStreamFactory, + CheckpointOptions checkpointOptions) throws Exception { - serializationProxy.read(currentStateHandleInView); + long startTime = System.currentTimeMillis(); + final CloseableRegistry snapshotCloseableRegistry = new CloseableRegistry(); - // check for key serializer compatibility; this also reconfigures the - // key serializer to be compatible, if it is required and is possible - if (CompatibilityUtil.resolveCompatibilityResult( - serializationProxy.getKeySerializer(), - UnloadableDummyTypeSerializer.class, - serializationProxy.getKeySerializerConfigSnapshot(), - rocksDBKeyedStateBackend.keySerializer) - .isRequiresMigration()) { + if (kvStateInformation.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Asynchronous RocksDB snapshot performed on empty keyed state at {}. Returning null.", + timestamp); + } - // TODO replace with state migration; note that key hash codes need to remain the same after migration - throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + - "Aborting now since state migration is currently not available"); + return DoneFuture.of(SnapshotResult.empty()); } - this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ? - SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; - - List> restoredMetaInfos = - serializationProxy.getStateMetaInfoSnapshots(); - currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size()); - //rocksDBKeyedStateBackend.restoredKvStateMetaInfos = new HashMap<>(restoredMetaInfos.size()); - - for (RegisteredKeyedBackendStateMetaInfo.Snapshot restoredMetaInfo : restoredMetaInfos) { + final SupplierWithException supplier = - Tuple2> registeredColumn = - rocksDBKeyedStateBackend.kvStateInformation.get(restoredMetaInfo.getName()); + isWithLocalRecovery( + checkpointOptions.getCheckpointType(), + localRecoveryConfig.getLocalRecoveryMode()) ? - if (registeredColumn == null) { - byte[] nameBytes = restoredMetaInfo.getName().getBytes(ConfigConstants.DEFAULT_CHARSET); + () -> CheckpointStreamWithResultProvider.createDuplicatingStream( + checkpointId, + CheckpointedStateScope.EXCLUSIVE, + primaryStreamFactory, + localRecoveryConfig.getLocalStateDirectoryProvider()) : - ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( - nameBytes, - rocksDBKeyedStateBackend.columnOptions); + () -> CheckpointStreamWithResultProvider.createSimpleStream( + CheckpointedStateScope.EXCLUSIVE, + primaryStreamFactory); - RegisteredKeyedBackendStateMetaInfo stateMetaInfo = - new RegisteredKeyedBackendStateMetaInfo<>( - restoredMetaInfo.getStateType(), - restoredMetaInfo.getName(), - restoredMetaInfo.getNamespaceSerializer(), - restoredMetaInfo.getStateSerializer()); + final RocksDBFullSnapshotOperation snapshotOperation = + new RocksDBFullSnapshotOperation<>( + RocksDBKeyedStateBackend.this, + supplier, + snapshotCloseableRegistry); - rocksDBKeyedStateBackend.restoredKvStateMetaInfos.put(restoredMetaInfo.getName(), restoredMetaInfo); + snapshotOperation.takeDBSnapShot(); - ColumnFamilyHandle columnFamily = rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor); + // implementation of the async IO operation, based on FutureTask + AbstractAsyncCallableWithResources> ioCallable = + new AbstractAsyncCallableWithResources>() { - registeredColumn = new Tuple2<>(columnFamily, stateMetaInfo); - rocksDBKeyedStateBackend.kvStateInformation.put(stateMetaInfo.getName(), registeredColumn); + @Override + protected void acquireResources() throws Exception { + cancelStreamRegistry.registerCloseable(snapshotCloseableRegistry); + snapshotOperation.openCheckpointStream(); + } - } else { - // TODO with eager state registration in place, check here for serializer migration strategies - } - currentStateHandleKVStateColumnFamilies.add(registeredColumn.f0); - } - } + @Override + protected void releaseResources() throws Exception { + closeLocalRegistry(); + releaseSnapshotOperationResources(); + } - /** - * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state handle. - * - * @throws IOException - * @throws RocksDBException - */ - private void restoreKVStateData() throws IOException, RocksDBException { - //for all key-groups in the current state handle... - for (Tuple2 keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) { - int keyGroup = keyGroupOffset.f0; + private void releaseSnapshotOperationResources() { + // hold the db lock while operation on the db to guard us against async db disposal + snapshotOperation.releaseSnapshotResources(); + } - // Check that restored key groups all belong to the backend - Preconditions.checkState(rocksDBKeyedStateBackend.getKeyGroupRange().contains(keyGroup), - "The key group must belong to the backend"); + @Override + protected void stopOperation() throws Exception { + closeLocalRegistry(); + } - long offset = keyGroupOffset.f1; - //not empty key-group? - if (0L != offset) { - currentStateHandleInStream.seek(offset); - try (InputStream compressedKgIn = keygroupStreamCompressionDecorator.decorateWithCompression(currentStateHandleInStream)) { - DataInputViewStreamWrapper compressedKgInputView = new DataInputViewStreamWrapper(compressedKgIn); - //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible - int kvStateId = compressedKgInputView.readShort(); - ColumnFamilyHandle handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); - //insert all k/v pairs into DB - boolean keyGroupHasMoreKeys = true; - while (keyGroupHasMoreKeys) { - byte[] key = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView); - byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(compressedKgInputView); - if (RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(key)) { - //clear the signal bit in the key to make it ready for insertion again - RocksDBFullSnapshotOperation.clearMetaDataFollowsFlag(key); - rocksDBKeyedStateBackend.db.put(handle, key, value); - //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible - kvStateId = RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK - & compressedKgInputView.readShort(); - if (RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK == kvStateId) { - keyGroupHasMoreKeys = false; - } else { - handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); - } - } else { - rocksDBKeyedStateBackend.db.put(handle, key, value); + private void closeLocalRegistry() { + if (cancelStreamRegistry.unregisterCloseable(snapshotCloseableRegistry)) { + try { + snapshotCloseableRegistry.close(); + } catch (Exception ex) { + LOG.warn("Error closing local registry", ex); } } } - } - } - } - } - private static class RocksDBIncrementalRestoreOperation { + @Nonnull + @Override + public SnapshotResult performOperation() throws Exception { + long startTime = System.currentTimeMillis(); - private final RocksDBKeyedStateBackend stateBackend; + if (isStopped()) { + throw new IOException("RocksDB closed."); + } - private RocksDBIncrementalRestoreOperation(RocksDBKeyedStateBackend stateBackend) { - this.stateBackend = stateBackend; - } + snapshotOperation.writeDBSnapshot(); - private List> readMetaData( - StreamStateHandle metaStateHandle) throws Exception { + LOG.info("Asynchronous RocksDB snapshot ({}, asynchronous part) in thread {} took {} ms.", + primaryStreamFactory, Thread.currentThread(), (System.currentTimeMillis() - startTime)); - FSDataInputStream inputStream = null; + return snapshotOperation.getSnapshotResultStateHandle(); + } + }; - try { - inputStream = metaStateHandle.openInputStream(); - stateBackend.cancelStreamRegistry.registerCloseable(inputStream); + LOG.info("Asynchronous RocksDB snapshot ({}, synchronous part) in thread {} took {} ms.", + primaryStreamFactory, Thread.currentThread(), (System.currentTimeMillis() - startTime)); + return AsyncStoppableTaskWithCallback.from(ioCallable); + } - KeyedBackendSerializationProxy serializationProxy = - new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); - DataInputView in = new DataInputViewStreamWrapper(inputStream); - serializationProxy.read(in); + private boolean isWithLocalRecovery( + CheckpointType checkpointType, + LocalRecoveryConfig.LocalRecoveryMode recoveryMode) { + // we use local recovery when it is activated and we are not taking a savepoint. + return LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED == recoveryMode + && CheckpointType.SAVEPOINT != checkpointType; + } + } - // check for key serializer compatibility; this also reconfigures the - // key serializer to be compatible, if it is required and is possible - if (CompatibilityUtil.resolveCompatibilityResult( - serializationProxy.getKeySerializer(), - UnloadableDummyTypeSerializer.class, - serializationProxy.getKeySerializerConfigSnapshot(), - stateBackend.keySerializer) - .isRequiresMigration()) { + private class IncrementalSnapshotStrategy implements SnapshotStrategy> { - // TODO replace with state migration; note that key hash codes need to remain the same after migration - throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + - "Aborting now since state migration is currently not available"); - } + private final SnapshotStrategy> savepointDelegate; - return serializationProxy.getStateMetaInfoSnapshots(); - } finally { - if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) { - inputStream.close(); - } - } + public IncrementalSnapshotStrategy() { + this.savepointDelegate = new FullSnapshotStrategy(); } - private void readStateData( - Path restoreFilePath, - StreamStateHandle remoteFileHandle) throws IOException { - - FileSystem restoreFileSystem = restoreFilePath.getFileSystem(); + @Override + public RunnableFuture> performSnapshot( + long checkpointId, + long checkpointTimestamp, + CheckpointStreamFactory checkpointStreamFactory, + CheckpointOptions checkpointOptions) throws Exception { + + // for savepoints, we delegate to the full snapshot strategy because savepoints are always self-contained. + if (CheckpointType.SAVEPOINT == checkpointOptions.getCheckpointType()) { + return savepointDelegate.performSnapshot( + checkpointId, + checkpointTimestamp, + checkpointStreamFactory, + checkpointOptions); + } - FSDataInputStream inputStream = null; - FSDataOutputStream outputStream = null; + if (db == null) { + throw new IOException("RocksDB closed."); + } - try { - inputStream = remoteFileHandle.openInputStream(); - stateBackend.cancelStreamRegistry.registerCloseable(inputStream); + if (kvStateInformation.isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Asynchronous RocksDB snapshot performed on empty keyed state at {}. Returning null.", checkpointTimestamp); + } + return DoneFuture.of(SnapshotResult.empty()); + } - outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE); - stateBackend.cancelStreamRegistry.registerCloseable(outputStream); + SnapshotDirectory snapshotDirectory; - byte[] buffer = new byte[8 * 1024]; - while (true) { - int numBytes = inputStream.read(buffer); - if (numBytes == -1) { - break; - } + if (LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED == localRecoveryConfig.getLocalRecoveryMode()) { + // create a "permanent" snapshot directory for local recovery. + LocalRecoveryDirectoryProvider directoryProvider = localRecoveryConfig.getLocalStateDirectoryProvider(); + File directory = directoryProvider.subtaskSpecificCheckpointDirectory(checkpointId); - outputStream.write(buffer, 0, numBytes); - } - } finally { - if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) { - inputStream.close(); + if (directory.exists()) { + FileUtils.deleteDirectory(directory); } - if (stateBackend.cancelStreamRegistry.unregisterCloseable(outputStream)) { - outputStream.close(); + if (!directory.mkdirs()) { + throw new IOException("Local state base directory for checkpoint " + checkpointId + + " already exists: " + directory); } - } - } - private void restoreInstance( - IncrementalKeyedStateHandle restoreStateHandle, - boolean hasExtraKeys) throws Exception { + // introduces an extra directory because RocksDB wants a non-existing directory for native checkpoints. + File rdbSnapshotDir = new File(directory, "rocks_db"); + Path path = new Path(rdbSnapshotDir.toURI()); + // create a "permanent" snapshot directory because local recovery is active. + snapshotDirectory = SnapshotDirectory.permanent(path); + } else { + // create a "temporary" snapshot directory because local recovery is inactive. + Path path = new Path(instanceBasePath.getAbsolutePath(), "chk-" + checkpointId); + snapshotDirectory = SnapshotDirectory.temporary(path); + } - // read state data - Path restoreInstancePath = new Path( - stateBackend.instanceBasePath.getAbsolutePath(), - UUID.randomUUID().toString()); + final RocksDBIncrementalSnapshotOperation snapshotOperation = + new RocksDBIncrementalSnapshotOperation<>( + RocksDBKeyedStateBackend.this, + checkpointStreamFactory, + snapshotDirectory, + checkpointId); try { - final Map sstFiles = - restoreStateHandle.getSharedState(); - final Map miscFiles = - restoreStateHandle.getPrivateState(); - - readAllStateData(sstFiles, restoreInstancePath); - readAllStateData(miscFiles, restoreInstancePath); - - // read meta data - List> stateMetaInfoSnapshots = - readMetaData(restoreStateHandle.getMetaStateHandle()); - - List columnFamilyDescriptors = - new ArrayList<>(1 + stateMetaInfoSnapshots.size()); - - for (RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) { - - ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( - stateMetaInfoSnapshot.getName().getBytes(ConfigConstants.DEFAULT_CHARSET), - stateBackend.columnOptions); + snapshotOperation.takeSnapshot(); + } catch (Exception e) { + snapshotOperation.stop(); + snapshotOperation.releaseResources(true); + throw e; + } - columnFamilyDescriptors.add(columnFamilyDescriptor); - stateBackend.restoredKvStateMetaInfos.put(stateMetaInfoSnapshot.getName(), stateMetaInfoSnapshot); + return new FutureTask>( + snapshotOperation::runSnapshot + ) { + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + snapshotOperation.stop(); + return super.cancel(mayInterruptIfRunning); } - if (hasExtraKeys) { - - List columnFamilyHandles = - new ArrayList<>(1 + columnFamilyDescriptors.size()); + @Override + protected void done() { + snapshotOperation.releaseResources(isCancelled()); + } + }; + } + } - try (RocksDB restoreDb = stateBackend.openDB( - restoreInstancePath.getPath(), - columnFamilyDescriptors, - columnFamilyHandles)) { - - try { - // iterating only the requested descriptors automatically skips the default column family handle - for (int i = 0; i < columnFamilyDescriptors.size(); ++i) { - ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i); - ColumnFamilyDescriptor columnFamilyDescriptor = columnFamilyDescriptors.get(i); - RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i); - - Tuple2> registeredStateMetaInfoEntry = - stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName()); - - if (null == registeredStateMetaInfoEntry) { - - RegisteredKeyedBackendStateMetaInfo stateMetaInfo = - new RegisteredKeyedBackendStateMetaInfo<>( - stateMetaInfoSnapshot.getStateType(), - stateMetaInfoSnapshot.getName(), - stateMetaInfoSnapshot.getNamespaceSerializer(), - stateMetaInfoSnapshot.getStateSerializer()); - - registeredStateMetaInfoEntry = - new Tuple2<>( - stateBackend.db.createColumnFamily(columnFamilyDescriptor), - stateMetaInfo); - - stateBackend.kvStateInformation.put( - stateMetaInfoSnapshot.getName(), - registeredStateMetaInfoEntry); - } + /** + * Encapsulates the process to perform a full snapshot of a RocksDBKeyedStateBackend. + */ + @VisibleForTesting + static class RocksDBFullSnapshotOperation + extends AbstractAsyncCallableWithResources> { - ColumnFamilyHandle targetColumnFamilyHandle = registeredStateMetaInfoEntry.f0; + static final int FIRST_BIT_IN_BYTE_MASK = 0x80; + static final int END_OF_KEY_GROUP_MARK = 0xFFFF; - try (RocksIterator iterator = restoreDb.newIterator(columnFamilyHandle)) { + private final RocksDBKeyedStateBackend stateBackend; + private final KeyGroupRangeOffsets keyGroupRangeOffsets; + private final SupplierWithException checkpointStreamSupplier; + private final CloseableRegistry snapshotCloseableRegistry; + private final ResourceGuard.Lease dbLease; - int startKeyGroup = stateBackend.getKeyGroupRange().getStartKeyGroup(); - byte[] startKeyGroupPrefixBytes = new byte[stateBackend.keyGroupPrefixBytes]; - for (int j = 0; j < stateBackend.keyGroupPrefixBytes; ++j) { - startKeyGroupPrefixBytes[j] = (byte) (startKeyGroup >>> ((stateBackend.keyGroupPrefixBytes - j - 1) * Byte.SIZE)); - } + private Snapshot snapshot; + private ReadOptions readOptions; + private List> kvStateIterators; - iterator.seek(startKeyGroupPrefixBytes); + private CheckpointStreamWithResultProvider checkpointStreamWithResultProvider; + private DataOutputView outputView; - while (iterator.isValid()) { + RocksDBFullSnapshotOperation( + RocksDBKeyedStateBackend stateBackend, + SupplierWithException checkpointStreamSupplier, + CloseableRegistry registry) throws IOException { - int keyGroup = 0; - for (int j = 0; j < stateBackend.keyGroupPrefixBytes; ++j) { - keyGroup = (keyGroup << Byte.SIZE) + iterator.key()[j]; - } + this.stateBackend = stateBackend; + this.checkpointStreamSupplier = checkpointStreamSupplier; + this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(stateBackend.keyGroupRange); + this.snapshotCloseableRegistry = registry; + this.dbLease = this.stateBackend.rocksDBResourceGuard.acquireResource(); + } - if (stateBackend.keyGroupRange.contains(keyGroup)) { - stateBackend.db.put(targetColumnFamilyHandle, - iterator.key(), iterator.value()); - } + /** + * 1) Create a snapshot object from RocksDB. + * + */ + public void takeDBSnapShot() { + Preconditions.checkArgument(snapshot == null, "Only one ongoing snapshot allowed!"); + this.kvStateIterators = new ArrayList<>(stateBackend.kvStateInformation.size()); + this.snapshot = stateBackend.db.getSnapshot(); + } - iterator.next(); - } - } // releases native iterator resources - } - } finally { - //release native tmp db column family resources - for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { - IOUtils.closeQuietly(columnFamilyHandle); - } - } - } // releases native tmp db resources - } else { - // pick up again the old backend id, so the we can reference existing state - stateBackend.backendUID = restoreStateHandle.getBackendIdentifier(); + /** + * 2) Open CheckpointStateOutputStream through the checkpointStreamFactory into which we will write. + * + * @throws Exception + */ + public void openCheckpointStream() throws Exception { + Preconditions.checkArgument(checkpointStreamWithResultProvider == null, + "Output stream for snapshot is already set."); - LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.", - stateBackend.operatorIdentifier, stateBackend.backendUID); + checkpointStreamWithResultProvider = checkpointStreamSupplier.get(); + snapshotCloseableRegistry.registerCloseable(checkpointStreamWithResultProvider); + outputView = new DataOutputViewStreamWrapper( + checkpointStreamWithResultProvider.getCheckpointOutputStream()); + } - // create hard links in the instance directory - if (!stateBackend.instanceRocksDBPath.mkdirs()) { - throw new IOException("Could not create RocksDB data directory."); - } + /** + * 3) Write the actual data from RocksDB from the time we took the snapshot object in (1). + * + * @throws IOException + */ + public void writeDBSnapshot() throws IOException, InterruptedException { - createFileHardLinksInRestorePath(sstFiles, restoreInstancePath); - createFileHardLinksInRestorePath(miscFiles, restoreInstancePath); + if (null == snapshot) { + throw new IOException("No snapshot available. Might be released due to cancellation."); + } - List columnFamilyHandles = - new ArrayList<>(1 + columnFamilyDescriptors.size()); + Preconditions.checkNotNull(checkpointStreamWithResultProvider, "No output stream to write snapshot."); + writeKVStateMetaData(); + writeKVStateData(); + } - stateBackend.db = stateBackend.openDB( - stateBackend.instanceRocksDBPath.getAbsolutePath(), - columnFamilyDescriptors, columnFamilyHandles); + /** + * 4) Returns a snapshot result for the completed snapshot. + * + * @return snapshot result for the completed snapshot. + */ + @Nonnull + public SnapshotResult getSnapshotResultStateHandle() throws IOException { - // extract and store the default column family which is located at the last index - stateBackend.defaultColumnFamily = columnFamilyHandles.remove(columnFamilyHandles.size() - 1); + if (snapshotCloseableRegistry.unregisterCloseable(checkpointStreamWithResultProvider)) { - for (int i = 0; i < columnFamilyDescriptors.size(); ++i) { - RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i); + SnapshotResult res = + checkpointStreamWithResultProvider.closeAndFinalizeCheckpointStreamResult(); + checkpointStreamWithResultProvider = null; + return CheckpointStreamWithResultProvider.toKeyedStateHandleSnapshotResult(res, keyGroupRangeOffsets); + } - ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i); - RegisteredKeyedBackendStateMetaInfo stateMetaInfo = - new RegisteredKeyedBackendStateMetaInfo<>( - stateMetaInfoSnapshot.getStateType(), - stateMetaInfoSnapshot.getName(), - stateMetaInfoSnapshot.getNamespaceSerializer(), - stateMetaInfoSnapshot.getStateSerializer()); + return SnapshotResult.empty(); + } - stateBackend.kvStateInformation.put( - stateMetaInfoSnapshot.getName(), - new Tuple2<>(columnFamilyHandle, stateMetaInfo)); - } + /** + * 5) Release the snapshot object for RocksDB and clean up. + */ + public void releaseSnapshotResources() { - // use the restore sst files as the base for succeeding checkpoints - synchronized (stateBackend.materializedSstFiles) { - stateBackend.materializedSstFiles.put(restoreStateHandle.getCheckpointId(), sstFiles.keySet()); - } + checkpointStreamWithResultProvider = null; - stateBackend.lastCompletedCheckpointId = restoreStateHandle.getCheckpointId(); - } - } finally { - FileSystem restoreFileSystem = restoreInstancePath.getFileSystem(); - if (restoreFileSystem.exists(restoreInstancePath)) { - restoreFileSystem.delete(restoreInstancePath, true); + if (null != kvStateIterators) { + for (Tuple2 kvStateIterator : kvStateIterators) { + IOUtils.closeQuietly(kvStateIterator.f0); } + kvStateIterators = null; } - } - - private void readAllStateData( - Map stateHandleMap, - Path restoreInstancePath) throws IOException { - for (Map.Entry entry : stateHandleMap.entrySet()) { - StateHandleID stateHandleID = entry.getKey(); - StreamStateHandle remoteFileHandle = entry.getValue(); - readStateData(new Path(restoreInstancePath, stateHandleID.toString()), remoteFileHandle); + if (null != snapshot) { + if (null != stateBackend.db) { + stateBackend.db.releaseSnapshot(snapshot); + } + IOUtils.closeQuietly(snapshot); + snapshot = null; } - } - - private void createFileHardLinksInRestorePath( - Map stateHandleMap, - Path restoreInstancePath) throws IOException { - for (StateHandleID stateHandleID : stateHandleMap.keySet()) { - String newSstFileName = stateHandleID.toString(); - File restoreFile = new File(restoreInstancePath.getPath(), newSstFileName); - File targetFile = new File(stateBackend.instanceRocksDBPath, newSstFileName); - Files.createLink(targetFile.toPath(), restoreFile.toPath()); + if (null != readOptions) { + IOUtils.closeQuietly(readOptions); + readOptions = null; } + + this.dbLease.close(); } - void restore(Collection restoreStateHandles) throws Exception { + private void writeKVStateMetaData() throws IOException { - boolean hasExtraKeys = (restoreStateHandles.size() > 1 || - !Objects.equals(restoreStateHandles.iterator().next().getKeyGroupRange(), stateBackend.keyGroupRange)); + List> metaInfoSnapshots = + new ArrayList<>(stateBackend.kvStateInformation.size()); - if (hasExtraKeys) { - stateBackend.createDB(); - } + int kvStateId = 0; + for (Map.Entry>> column : + stateBackend.kvStateInformation.entrySet()) { - for (KeyedStateHandle rawStateHandle : restoreStateHandles) { + metaInfoSnapshots.add(column.getValue().f1.snapshot()); - if (!(rawStateHandle instanceof IncrementalKeyedStateHandle)) { - throw new IllegalStateException("Unexpected state handle type, " + - "expected " + IncrementalKeyedStateHandle.class + - ", but found " + rawStateHandle.getClass()); - } + //retrieve iterator for this k/v states + readOptions = new ReadOptions(); + readOptions.setSnapshot(snapshot); - IncrementalKeyedStateHandle keyedStateHandle = (IncrementalKeyedStateHandle) rawStateHandle; + kvStateIterators.add( + new Tuple2<>(stateBackend.db.newIterator(column.getValue().f0, readOptions), kvStateId)); - restoreInstance(keyedStateHandle, hasExtraKeys); + ++kvStateId; } + + KeyedBackendSerializationProxy serializationProxy = + new KeyedBackendSerializationProxy<>( + stateBackend.getKeySerializer(), + metaInfoSnapshots, + !Objects.equals( + UncompressedStreamCompressionDecorator.INSTANCE, + stateBackend.keyGroupCompressionDecorator)); + + serializationProxy.write(outputView); } - } - // ------------------------------------------------------------------------ - // State factories - // ------------------------------------------------------------------------ + private void writeKVStateData() throws IOException, InterruptedException { - /** - * Creates a column family handle for use with a k/v state. When restoring from a snapshot - * we don't restore the individual k/v states, just the global RocksDB database and the - * list of column families. When a k/v state is first requested we check here whether we - * already have a column family for that and return it or create a new one if it doesn't exist. - * - *

This also checks whether the {@link StateDescriptor} for a state matches the one - * that we checkpointed, i.e. is already in the map of column families. - */ - @SuppressWarnings("rawtypes, unchecked") - protected ColumnFamilyHandle getColumnFamily( - StateDescriptor descriptor, TypeSerializer namespaceSerializer) throws IOException, StateMigrationException { + byte[] previousKey = null; + byte[] previousValue = null; + DataOutputView kgOutView = null; + OutputStream kgOutStream = null; + CheckpointStreamFactory.CheckpointStateOutputStream checkpointOutputStream = + checkpointStreamWithResultProvider.getCheckpointOutputStream(); - Tuple2> stateInfo = - kvStateInformation.get(descriptor.getName()); + try { + // Here we transfer ownership of RocksIterators to the RocksDBMergeIterator + try (RocksDBMergeIterator mergeIterator = new RocksDBMergeIterator( + kvStateIterators, stateBackend.keyGroupPrefixBytes)) { - RegisteredKeyedBackendStateMetaInfo newMetaInfo = new RegisteredKeyedBackendStateMetaInfo<>( - descriptor.getType(), - descriptor.getName(), - namespaceSerializer, - descriptor.getSerializer()); + // handover complete, null out to prevent double close + kvStateIterators = null; - if (stateInfo != null) { - // TODO with eager registration in place, these checks should be moved to restore() + //preamble: setup with first key-group as our lookahead + if (mergeIterator.isValid()) { + //begin first key-group by recording the offset + keyGroupRangeOffsets.setKeyGroupOffset( + mergeIterator.keyGroup(), + checkpointOutputStream.getPos()); + //write the k/v-state id as metadata + kgOutStream = stateBackend.keyGroupCompressionDecorator. + decorateWithCompression(checkpointOutputStream); + kgOutView = new DataOutputViewStreamWrapper(kgOutStream); + //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible + kgOutView.writeShort(mergeIterator.kvStateId()); + previousKey = mergeIterator.key(); + previousValue = mergeIterator.value(); + mergeIterator.next(); + } - RegisteredKeyedBackendStateMetaInfo.Snapshot restoredMetaInfo = - (RegisteredKeyedBackendStateMetaInfo.Snapshot) restoredKvStateMetaInfos.get(descriptor.getName()); + //main loop: write k/v pairs ordered by (key-group, kv-state), thereby tracking key-group offsets. + while (mergeIterator.isValid()) { - Preconditions.checkState( - Objects.equals(newMetaInfo.getName(), restoredMetaInfo.getName()), - "Incompatible state names. " + - "Was [" + restoredMetaInfo.getName() + "], " + - "registered with [" + newMetaInfo.getName() + "]."); + assert (!hasMetaDataFollowsFlag(previousKey)); - if (!Objects.equals(newMetaInfo.getStateType(), StateDescriptor.Type.UNKNOWN) - && !Objects.equals(restoredMetaInfo.getStateType(), StateDescriptor.Type.UNKNOWN)) { + //set signal in first key byte that meta data will follow in the stream after this k/v pair + if (mergeIterator.isNewKeyGroup() || mergeIterator.isNewKeyValueState()) { - Preconditions.checkState( - newMetaInfo.getStateType() == restoredMetaInfo.getStateType(), - "Incompatible state types. " + - "Was [" + restoredMetaInfo.getStateType() + "], " + - "registered with [" + newMetaInfo.getStateType() + "]."); - } + //be cooperative and check for interruption from time to time in the hot loop + checkInterrupted(); - // check compatibility results to determine if state migration is required - CompatibilityResult namespaceCompatibility = CompatibilityUtil.resolveCompatibilityResult( - restoredMetaInfo.getNamespaceSerializer(), - null, - restoredMetaInfo.getNamespaceSerializerConfigSnapshot(), - newMetaInfo.getNamespaceSerializer()); + setMetaDataFollowsFlagInKey(previousKey); + } - CompatibilityResult stateCompatibility = CompatibilityUtil.resolveCompatibilityResult( - restoredMetaInfo.getStateSerializer(), - UnloadableDummyTypeSerializer.class, - restoredMetaInfo.getStateSerializerConfigSnapshot(), - newMetaInfo.getStateSerializer()); + writeKeyValuePair(previousKey, previousValue, kgOutView); - if (namespaceCompatibility.isRequiresMigration() || stateCompatibility.isRequiresMigration()) { - // TODO state migration currently isn't possible. - throw new StateMigrationException("State migration isn't supported, yet."); - } else { - stateInfo.f1 = newMetaInfo; - return stateInfo.f0; + //write meta data if we have to + if (mergeIterator.isNewKeyGroup()) { + //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible + kgOutView.writeShort(END_OF_KEY_GROUP_MARK); + // this will just close the outer stream + kgOutStream.close(); + //begin new key-group + keyGroupRangeOffsets.setKeyGroupOffset( + mergeIterator.keyGroup(), + checkpointOutputStream.getPos()); + //write the kev-state + //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible + kgOutStream = stateBackend.keyGroupCompressionDecorator. + decorateWithCompression(checkpointOutputStream); + kgOutView = new DataOutputViewStreamWrapper(kgOutStream); + kgOutView.writeShort(mergeIterator.kvStateId()); + } else if (mergeIterator.isNewKeyValueState()) { + //write the k/v-state + //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible + kgOutView.writeShort(mergeIterator.kvStateId()); + } + + //request next k/v pair + previousKey = mergeIterator.key(); + previousValue = mergeIterator.value(); + mergeIterator.next(); + } + } + + //epilogue: write last key-group + if (previousKey != null) { + assert (!hasMetaDataFollowsFlag(previousKey)); + setMetaDataFollowsFlagInKey(previousKey); + writeKeyValuePair(previousKey, previousValue, kgOutView); + //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible + kgOutView.writeShort(END_OF_KEY_GROUP_MARK); + // this will just close the outer stream + kgOutStream.close(); + kgOutStream = null; + } + + } finally { + // this will just close the outer stream + IOUtils.closeQuietly(kgOutStream); } } - byte[] nameBytes = descriptor.getName().getBytes(ConfigConstants.DEFAULT_CHARSET); - Preconditions.checkState(!Arrays.equals(DEFAULT_COLUMN_FAMILY_NAME_BYTES, nameBytes), - "The chosen state name 'default' collides with the name of the default column family!"); + private void writeKeyValuePair(byte[] key, byte[] value, DataOutputView out) throws IOException { + BytePrimitiveArraySerializer.INSTANCE.serialize(key, out); + BytePrimitiveArraySerializer.INSTANCE.serialize(value, out); + } + + static void setMetaDataFollowsFlagInKey(byte[] key) { + key[0] |= FIRST_BIT_IN_BYTE_MASK; + } - ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); + static void clearMetaDataFollowsFlag(byte[] key) { + key[0] &= (~RocksDBFullSnapshotOperation.FIRST_BIT_IN_BYTE_MASK); + } - final ColumnFamilyHandle columnFamily; + static boolean hasMetaDataFollowsFlag(byte[] key) { + return 0 != (key[0] & RocksDBFullSnapshotOperation.FIRST_BIT_IN_BYTE_MASK); + } - try { - columnFamily = db.createColumnFamily(columnDescriptor); - } catch (RocksDBException e) { - throw new IOException("Error creating ColumnFamilyHandle.", e); + private static void checkInterrupted() throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("RocksDB snapshot interrupted."); + } } - Tuple2> tuple = - new Tuple2<>(columnFamily, newMetaInfo); - Map rawAccess = kvStateInformation; - rawAccess.put(descriptor.getName(), tuple); - return columnFamily; - } + @Override + protected void acquireResources() throws Exception { + stateBackend.cancelStreamRegistry.registerCloseable(snapshotCloseableRegistry); + openCheckpointStream(); + } - @Override - protected InternalValueState createValueState( - TypeSerializer namespaceSerializer, - ValueStateDescriptor stateDesc) throws Exception { + @Override + protected void releaseResources() { + closeLocalRegistry(); + releaseSnapshotOperationResources(); + } - ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + private void releaseSnapshotOperationResources() { + // hold the db lock while operation on the db to guard us against async db disposal + releaseSnapshotResources(); + } - return new RocksDBValueState<>(columnFamily, namespaceSerializer, stateDesc, this); - } + @Override + protected void stopOperation() { + closeLocalRegistry(); + } - @Override - protected InternalListState createListState( - TypeSerializer namespaceSerializer, - ListStateDescriptor stateDesc) throws Exception { + private void closeLocalRegistry() { + if (stateBackend.cancelStreamRegistry.unregisterCloseable(snapshotCloseableRegistry)) { + try { + snapshotCloseableRegistry.close(); + } catch (Exception ex) { + LOG.warn("Error closing local registry", ex); + } + } + } - ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + @Nonnull + @Override + public SnapshotResult performOperation() throws Exception { + long startTime = System.currentTimeMillis(); - return new RocksDBListState<>(columnFamily, namespaceSerializer, stateDesc, this); - } + if (isStopped()) { + throw new IOException("RocksDB closed."); + } - @Override - protected InternalReducingState createReducingState( - TypeSerializer namespaceSerializer, - ReducingStateDescriptor stateDesc) throws Exception { + writeDBSnapshot(); - ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + LOG.info("Asynchronous RocksDB snapshot ({}, asynchronous part) in thread {} took {} ms.", + checkpointStreamSupplier, Thread.currentThread(), (System.currentTimeMillis() - startTime)); - return new RocksDBReducingState<>(columnFamily, namespaceSerializer, stateDesc, this); + return getSnapshotResultStateHandle(); + } } - @Override - protected InternalAggregatingState createAggregatingState( - TypeSerializer namespaceSerializer, - AggregatingStateDescriptor stateDesc) throws Exception { - - ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); - return new RocksDBAggregatingState<>(columnFamily, namespaceSerializer, stateDesc, this); - } + /** + * Encapsulates the process to perform an incremental snapshot of a RocksDBKeyedStateBackend. + */ + private static final class RocksDBIncrementalSnapshotOperation { - @Override - protected InternalFoldingState createFoldingState( - TypeSerializer namespaceSerializer, - FoldingStateDescriptor stateDesc) throws Exception { + /** The backend which we snapshot. */ + private final RocksDBKeyedStateBackend stateBackend; - ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + /** Stream factory that creates the outpus streams to DFS. */ + private final CheckpointStreamFactory checkpointStreamFactory; - return new RocksDBFoldingState<>(columnFamily, namespaceSerializer, stateDesc, this); - } + /** Id for the current checkpoint. */ + private final long checkpointId; - @Override - protected InternalMapState createMapState( - TypeSerializer namespaceSerializer, - MapStateDescriptor stateDesc) throws Exception { + /** All sst files that were part of the last previously completed checkpoint. */ + private Set baseSstFiles; - ColumnFamilyHandle columnFamily = getColumnFamily(stateDesc, namespaceSerializer); + /** The state meta data. */ + private final List> stateMetaInfoSnapshots = new ArrayList<>(); - return new RocksDBMapState<>(columnFamily, namespaceSerializer, stateDesc, this); - } + /** Local directory for the RocksDB native backup. */ + private SnapshotDirectory localBackupDirectory; - /** - * Wraps a RocksDB iterator to cache it's current key and assigns an id for the key/value state to the iterator. - * Used by #MergeIterator. - */ - static final class MergeIterator implements AutoCloseable { + // Registry for all opened i/o streams + private final CloseableRegistry closeableRegistry = new CloseableRegistry(); - /** - * @param iterator The #RocksIterator to wrap . - * @param kvStateId Id of the K/V state to which this iterator belongs. - */ - MergeIterator(RocksIterator iterator, int kvStateId) { - this.iterator = Preconditions.checkNotNull(iterator); - this.currentKey = iterator.key(); - this.kvStateId = kvStateId; - } + // new sst files since the last completed checkpoint + private final Map sstFiles = new HashMap<>(); - private final RocksIterator iterator; - private byte[] currentKey; - private final int kvStateId; + // handles to the misc files in the current snapshot + private final Map miscFiles = new HashMap<>(); - public byte[] getCurrentKey() { - return currentKey; - } + // This lease protects from concurrent disposal of the native rocksdb instance. + private final ResourceGuard.Lease dbLease; - public void setCurrentKey(byte[] currentKey) { - this.currentKey = currentKey; - } + private SnapshotResult metaStateHandle = null; - public RocksIterator getIterator() { - return iterator; - } + private RocksDBIncrementalSnapshotOperation( + RocksDBKeyedStateBackend stateBackend, + CheckpointStreamFactory checkpointStreamFactory, + SnapshotDirectory localBackupDirectory, + long checkpointId) throws IOException { - public int getKvStateId() { - return kvStateId; + this.stateBackend = stateBackend; + this.checkpointStreamFactory = checkpointStreamFactory; + this.checkpointId = checkpointId; + this.dbLease = this.stateBackend.rocksDBResourceGuard.acquireResource(); + this.localBackupDirectory = localBackupDirectory; } - @Override - public void close() { - IOUtils.closeQuietly(iterator); - } - } + private StreamStateHandle materializeStateData(Path filePath) throws Exception { + FSDataInputStream inputStream = null; + CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null; - /** - * Iterator that merges multiple RocksDB iterators to partition all states into contiguous key-groups. - * The resulting iteration sequence is ordered by (key-group, kv-state). - */ - static final class RocksDBMergeIterator implements AutoCloseable { + try { + final byte[] buffer = new byte[8 * 1024]; - private final PriorityQueue heap; - private final int keyGroupPrefixByteCount; - private boolean newKeyGroup; - private boolean newKVState; - private boolean valid; + FileSystem backupFileSystem = localBackupDirectory.getFileSystem(); + inputStream = backupFileSystem.open(filePath); + closeableRegistry.registerCloseable(inputStream); - private MergeIterator currentSubIterator; + outputStream = checkpointStreamFactory + .createCheckpointStateOutputStream(CheckpointedStateScope.SHARED); + closeableRegistry.registerCloseable(outputStream); - private static final List> COMPARATORS; + while (true) { + int numBytes = inputStream.read(buffer); - static { - int maxBytes = 4; - COMPARATORS = new ArrayList<>(maxBytes); - for (int i = 0; i < maxBytes; ++i) { - final int currentBytes = i; - COMPARATORS.add(new Comparator() { - @Override - public int compare(MergeIterator o1, MergeIterator o2) { - int arrayCmpRes = compareKeyGroupsForByteArrays( - o1.currentKey, o2.currentKey, currentBytes); - return arrayCmpRes == 0 ? o1.getKvStateId() - o2.getKvStateId() : arrayCmpRes; + if (numBytes == -1) { + break; } - }); - } - } - RocksDBMergeIterator(List> kvStateIterators, final int keyGroupPrefixByteCount) { - Preconditions.checkNotNull(kvStateIterators); - this.keyGroupPrefixByteCount = keyGroupPrefixByteCount; + outputStream.write(buffer, 0, numBytes); + } - Comparator iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount); + StreamStateHandle result = null; + if (closeableRegistry.unregisterCloseable(outputStream)) { + result = outputStream.closeAndGetHandle(); + outputStream = null; + } + return result; - if (kvStateIterators.size() > 0) { - PriorityQueue iteratorPriorityQueue = - new PriorityQueue<>(kvStateIterators.size(), iteratorComparator); + } finally { - for (Tuple2 rocksIteratorWithKVStateId : kvStateIterators) { - final RocksIterator rocksIterator = rocksIteratorWithKVStateId.f0; - rocksIterator.seekToFirst(); - if (rocksIterator.isValid()) { - iteratorPriorityQueue.offer(new MergeIterator(rocksIterator, rocksIteratorWithKVStateId.f1)); - } else { - IOUtils.closeQuietly(rocksIterator); - } + if (closeableRegistry.unregisterCloseable(inputStream)) { + inputStream.close(); } - kvStateIterators.clear(); - - this.heap = iteratorPriorityQueue; - this.valid = !heap.isEmpty(); - this.currentSubIterator = heap.poll(); - } else { - // creating a PriorityQueue of size 0 results in an exception. - this.heap = null; - this.valid = false; + if (closeableRegistry.unregisterCloseable(outputStream)) { + outputStream.close(); + } } - - this.newKeyGroup = true; - this.newKVState = true; } - /** - * Advance the iterator. Should only be called if {@link #isValid()} returned true. Valid can only chance after - * calls to {@link #next()}. - */ - public void next() { - newKeyGroup = false; - newKVState = false; + @Nonnull + private SnapshotResult materializeMetaData() throws Exception { - final RocksIterator rocksIterator = currentSubIterator.getIterator(); - rocksIterator.next(); + LocalRecoveryConfig localRecoveryConfig = stateBackend.localRecoveryConfig; - byte[] oldKey = currentSubIterator.getCurrentKey(); - if (rocksIterator.isValid()) { - currentSubIterator.currentKey = rocksIterator.key(); + CheckpointStreamWithResultProvider streamWithResultProvider = - if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) { - heap.offer(currentSubIterator); - currentSubIterator = heap.poll(); - newKVState = currentSubIterator.getIterator() != rocksIterator; - detectNewKeyGroup(oldKey); - } - } else { - IOUtils.closeQuietly(rocksIterator); + LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED == localRecoveryConfig.getLocalRecoveryMode() ? + + CheckpointStreamWithResultProvider.createDuplicatingStream( + checkpointId, + CheckpointedStateScope.EXCLUSIVE, + checkpointStreamFactory, + localRecoveryConfig.getLocalStateDirectoryProvider()) : + + CheckpointStreamWithResultProvider.createSimpleStream( + CheckpointedStateScope.EXCLUSIVE, + checkpointStreamFactory); + + try { + closeableRegistry.registerCloseable(streamWithResultProvider); + + //no need for compression scheme support because sst-files are already compressed + KeyedBackendSerializationProxy serializationProxy = + new KeyedBackendSerializationProxy<>( + stateBackend.keySerializer, + stateMetaInfoSnapshots, + false); - if (heap.isEmpty()) { - currentSubIterator = null; - valid = false; + DataOutputView out = + new DataOutputViewStreamWrapper(streamWithResultProvider.getCheckpointOutputStream()); + + serializationProxy.write(out); + + if (closeableRegistry.unregisterCloseable(streamWithResultProvider)) { + SnapshotResult result = + streamWithResultProvider.closeAndFinalizeCheckpointStreamResult(); + streamWithResultProvider = null; + return result; } else { - currentSubIterator = heap.poll(); - newKVState = true; - detectNewKeyGroup(oldKey); + throw new IOException("Stream already closed and cannot return a handle."); + } + } finally { + if (streamWithResultProvider != null) { + if (closeableRegistry.unregisterCloseable(streamWithResultProvider)) { + IOUtils.closeQuietly(streamWithResultProvider); + } } } } - private boolean isDifferentKeyGroup(byte[] a, byte[] b) { - return 0 != compareKeyGroupsForByteArrays(a, b, keyGroupPrefixByteCount); - } + void takeSnapshot() throws Exception { - private void detectNewKeyGroup(byte[] oldKey) { - if (isDifferentKeyGroup(oldKey, currentSubIterator.currentKey)) { - newKeyGroup = true; + final long lastCompletedCheckpoint; + + // use the last completed checkpoint as the comparison base. + synchronized (stateBackend.materializedSstFiles) { + lastCompletedCheckpoint = stateBackend.lastCompletedCheckpointId; + baseSstFiles = stateBackend.materializedSstFiles.get(lastCompletedCheckpoint); } - } - /** - * @return key-group for the current key - */ - public int keyGroup() { - int result = 0; - //big endian decode - for (int i = 0; i < keyGroupPrefixByteCount; ++i) { - result <<= 8; - result |= (currentSubIterator.currentKey[i] & 0xFF); + LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " + + "assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles); + + // save meta data + for (Map.Entry>> stateMetaInfoEntry + : stateBackend.kvStateInformation.entrySet()) { + stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().f1.snapshot()); } - return result; - } - public byte[] key() { - return currentSubIterator.getCurrentKey(); - } + LOG.trace("Local RocksDB checkpoint goes to backup path {}.", localBackupDirectory); - public byte[] value() { - return currentSubIterator.getIterator().value(); - } + if (localBackupDirectory.exists()) { + throw new IllegalStateException("Unexpected existence of the backup directory."); + } - /** - * @return Id of K/V state to which the current key belongs. - */ - public int kvStateId() { - return currentSubIterator.getKvStateId(); + // create hard links of living files in the snapshot path + Checkpoint checkpoint = Checkpoint.create(stateBackend.db); + checkpoint.createCheckpoint(localBackupDirectory.getDirectory().getPath()); } - /** - * Indicates if current key starts a new k/v-state, i.e. belong to a different k/v-state than it's predecessor. - * @return true iff the current key belong to a different k/v-state than it's predecessor. - */ - public boolean isNewKeyValueState() { - return newKVState; - } + @Nonnull + SnapshotResult runSnapshot() throws Exception { - /** - * Indicates if current key starts a new key-group, i.e. belong to a different key-group than it's predecessor. - * @return true iff the current key belong to a different key-group than it's predecessor. - */ - public boolean isNewKeyGroup() { - return newKeyGroup; - } + stateBackend.cancelStreamRegistry.registerCloseable(closeableRegistry); - /** - * Check if the iterator is still valid. Getters like {@link #key()}, {@link #value()}, etc. as well as - * {@link #next()} should only be called if valid returned true. Should be checked after each call to - * {@link #next()} before accessing iterator state. - * @return True iff this iterator is valid. - */ - public boolean isValid() { - return valid; - } + // write meta data + metaStateHandle = materializeMetaData(); - private static int compareKeyGroupsForByteArrays(byte[] a, byte[] b, int len) { - for (int i = 0; i < len; ++i) { - int diff = (a[i] & 0xFF) - (b[i] & 0xFF); - if (diff != 0) { - return diff; + // sanity checks - they should never fail + Preconditions.checkNotNull(metaStateHandle, + "Metadata was not properly created."); + Preconditions.checkNotNull(metaStateHandle.getJobManagerOwnedSnapshot(), + "Metadata for job manager was not properly created."); + + // write state data + Preconditions.checkState(localBackupDirectory.exists()); + + FileStatus[] fileStatuses = localBackupDirectory.listStatus(); + if (fileStatuses != null) { + for (FileStatus fileStatus : fileStatuses) { + final Path filePath = fileStatus.getPath(); + final String fileName = filePath.getName(); + final StateHandleID stateHandleID = new StateHandleID(fileName); + + if (fileName.endsWith(SST_FILE_SUFFIX)) { + final boolean existsAlready = + baseSstFiles != null && baseSstFiles.contains(stateHandleID); + + if (existsAlready) { + // we introduce a placeholder state handle, that is replaced with the + // original from the shared state registry (created from a previous checkpoint) + sstFiles.put( + stateHandleID, + new PlaceholderStreamStateHandle()); + } else { + sstFiles.put(stateHandleID, materializeStateData(filePath)); + } + } else { + StreamStateHandle fileHandle = materializeStateData(filePath); + miscFiles.put(stateHandleID, fileHandle); + } } } - return 0; - } - @Override - public void close() { - IOUtils.closeQuietly(currentSubIterator); - currentSubIterator = null; + synchronized (stateBackend.materializedSstFiles) { + stateBackend.materializedSstFiles.put(checkpointId, sstFiles.keySet()); + } - IOUtils.closeAllQuietly(heap); - heap.clear(); - } - } + IncrementalKeyedStateHandle jmIncrementalKeyedStateHandle = new IncrementalKeyedStateHandle( + stateBackend.backendUID, + stateBackend.keyGroupRange, + checkpointId, + sstFiles, + miscFiles, + metaStateHandle.getJobManagerOwnedSnapshot()); - /** - * Only visible for testing, DO NOT USE. - */ - @VisibleForTesting - public File getInstanceBasePath() { - return instanceBasePath; - } + StreamStateHandle taskLocalSnapshotMetaDataStateHandle = metaStateHandle.getTaskLocalSnapshot(); + DirectoryStateHandle directoryStateHandle = null; - @Override - public boolean supportsAsynchronousSnapshots() { - return true; - } + try { - @VisibleForTesting - @SuppressWarnings("unchecked") - @Override - public int numStateEntries() { - int count = 0; + directoryStateHandle = localBackupDirectory.completeSnapshotAndGetHandle(); + } catch (IOException ex) { - for (Tuple2> column : kvStateInformation.values()) { - try (RocksIterator rocksIterator = db.newIterator(column.f0)) { - rocksIterator.seekToFirst(); + Exception collector = ex; - while (rocksIterator.isValid()) { - count++; - rocksIterator.next(); + try { + taskLocalSnapshotMetaDataStateHandle.discardState(); + } catch (Exception discardEx) { + collector = ExceptionUtils.firstOrSuppressed(discardEx, collector); } - } - } - - return count; - } - /** - * This class is not thread safe. - */ - static class RocksIteratorWrapper implements Iterator, AutoCloseable { - private final RocksIterator iterator; - private final String state; - private final TypeSerializer keySerializer; - private final int keyGroupPrefixBytes; - private final byte[] namespaceBytes; - private final boolean ambiguousKeyPossible; - private K nextKey; + LOG.warn("Problem with local state snapshot.", collector); + } - public RocksIteratorWrapper( - RocksIterator iterator, - String state, - TypeSerializer keySerializer, - int keyGroupPrefixBytes, - boolean ambiguousKeyPossible, - byte[] namespaceBytes) { - this.iterator = Preconditions.checkNotNull(iterator); - this.state = Preconditions.checkNotNull(state); - this.keySerializer = Preconditions.checkNotNull(keySerializer); - this.keyGroupPrefixBytes = Preconditions.checkNotNull(keyGroupPrefixBytes); - this.namespaceBytes = Preconditions.checkNotNull(namespaceBytes); - this.nextKey = null; - this.ambiguousKeyPossible = ambiguousKeyPossible; + if (directoryStateHandle != null && taskLocalSnapshotMetaDataStateHandle != null) { + + IncrementalLocalKeyedStateHandle localDirKeyedStateHandle = + new IncrementalLocalKeyedStateHandle( + stateBackend.backendUID, + checkpointId, + directoryStateHandle, + stateBackend.keyGroupRange, + taskLocalSnapshotMetaDataStateHandle, + sstFiles.keySet()); + return SnapshotResult.withLocalState(jmIncrementalKeyedStateHandle, localDirKeyedStateHandle); + } else { + return SnapshotResult.of(jmIncrementalKeyedStateHandle); + } } - @Override - public boolean hasNext() { - while (nextKey == null && iterator.isValid()) { + void stop() { + + if (stateBackend.cancelStreamRegistry.unregisterCloseable(closeableRegistry)) { try { - byte[] key = iterator.key(); - if (isMatchingNameSpace(key)) { - ByteArrayInputStreamWithPos inputStream = - new ByteArrayInputStreamWithPos(key, keyGroupPrefixBytes, key.length - keyGroupPrefixBytes); - DataInputViewStreamWrapper dataInput = new DataInputViewStreamWrapper(inputStream); - K value = RocksDBKeySerializationUtils.readKey( - keySerializer, - inputStream, - dataInput, - ambiguousKeyPossible); - nextKey = value; - } - iterator.next(); + closeableRegistry.close(); } catch (IOException e) { - throw new FlinkRuntimeException("Failed to access state [" + state + "]", e); + LOG.warn("Could not properly close io streams.", e); } } - return nextKey != null; } - @Override - public K next() { - if (!hasNext()) { - throw new NoSuchElementException("Failed to access state [" + state + "]"); + void releaseResources(boolean canceled) { + + dbLease.close(); + + if (stateBackend.cancelStreamRegistry.unregisterCloseable(closeableRegistry)) { + try { + closeableRegistry.close(); + } catch (IOException e) { + LOG.warn("Exception on closing registry.", e); + } } - K tmpKey = nextKey; - nextKey = null; - return tmpKey; - } + try { + if (localBackupDirectory.exists()) { + LOG.trace("Running cleanup for local RocksDB backup directory {}.", localBackupDirectory); + boolean cleanupOk = localBackupDirectory.cleanup(); - private boolean isMatchingNameSpace(@Nonnull byte[] key) { - final int namespaceBytesLength = namespaceBytes.length; - final int basicLength = namespaceBytesLength + keyGroupPrefixBytes; - if (key.length >= basicLength) { - for (int i = 1; i <= namespaceBytesLength; ++i) { - if (key[key.length - i] != namespaceBytes[namespaceBytesLength - i]) { - return false; + if (!cleanupOk) { + LOG.debug("Could not properly cleanup local RocksDB backup directory."); } } - return true; + } catch (IOException e) { + LOG.warn("Could not properly cleanup local RocksDB backup directory.", e); } - return false; - } - @Override - public void close() { - iterator.close(); + if (canceled) { + Collection statesToDiscard = + new ArrayList<>(1 + miscFiles.size() + sstFiles.size()); + + statesToDiscard.add(metaStateHandle); + statesToDiscard.addAll(miscFiles.values()); + statesToDiscard.addAll(sstFiles.values()); + + try { + StateUtil.bestEffortDiscardAllStateObjects(statesToDiscard); + } catch (Exception e) { + LOG.warn("Could not properly discard states.", e); + } + + if (localBackupDirectory.isSnapshotCompleted()) { + try { + DirectoryStateHandle directoryStateHandle = localBackupDirectory.completeSnapshotAndGetHandle(); + if (directoryStateHandle != null) { + directoryStateHandle.discardState(); + } + } catch (Exception e) { + LOG.warn("Could not properly discard local state.", e); + } + } + } } } } diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/IncrementalRocksStreamOperatorSnapshotRestoreTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/IncrementalRocksStreamOperatorSnapshotRestoreTest.java new file mode 100644 index 00000000000000..37536339028e5a --- /dev/null +++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/IncrementalRocksStreamOperatorSnapshotRestoreTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.contrib.streaming.state; + +import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.streaming.api.operators.StreamOperatorSnapshotRestoreTest; + +import java.io.IOException; + +/** + * Test snapshot/restore of stream operators for RocksDB (incremental snapshots). + */ +public class IncrementalRocksStreamOperatorSnapshotRestoreTest extends StreamOperatorSnapshotRestoreTest { + + @Override + protected StateBackend createStateBackend() throws IOException { + FsStateBackend stateBackend = createStateBackendInternal(); + return new RocksDBStateBackend(stateBackend, true); + } +} diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksIteratorWrapperTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksIteratorWrapperTest.java index 98f937a66e6f95..0cdda4b83d65d1 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksIteratorWrapperTest.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBRocksIteratorWrapperTest.java @@ -120,13 +120,14 @@ void testIteratorHelper( try ( ColumnFamilyHandle handle = keyedStateBackend.getColumnFamilyHandle(testStateName); RocksIterator iterator = keyedStateBackend.db.newIterator(handle); - RocksDBKeyedStateBackend.RocksIteratorWrapper iteratorWrapper = new RocksDBKeyedStateBackend.RocksIteratorWrapper( - iterator, - testStateName, - keySerializer, - keyedStateBackend.getKeyGroupPrefixBytes(), - ambiguousKeyPossible, - nameSpaceBytes)) { + RocksDBKeyedStateBackend.RocksIteratorForKeysWrapper iteratorWrapper = + new RocksDBKeyedStateBackend.RocksIteratorForKeysWrapper<>( + iterator, + testStateName, + keySerializer, + keyedStateBackend.getKeyGroupPrefixBytes(), + ambiguousKeyPossible, + nameSpaceBytes)) { iterator.seekToFirst(); diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksStreamOperatorSnapshotRestoreTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksStreamOperatorSnapshotRestoreTest.java new file mode 100644 index 00000000000000..0ff32117e09e2a --- /dev/null +++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksStreamOperatorSnapshotRestoreTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.contrib.streaming.state; + +import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.streaming.api.operators.StreamOperatorSnapshotRestoreTest; + +import java.io.IOException; + +/** + * Test snapshot/restore of stream operators for RocksDB (full snapshots). + */ +public class RocksStreamOperatorSnapshotRestoreTest extends StreamOperatorSnapshotRestoreTest { + + @Override + protected StateBackend createStateBackend() throws IOException { + FsStateBackend stateBackend = createStateBackendInternal(); + return new RocksDBStateBackend(stateBackend, false); + } +} From 32e25eb6426081703bc925e765ce3d9f5ab892e4 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Thu, 8 Feb 2018 17:14:08 +0100 Subject: [PATCH 0003/2294] [FLINK-8360][checkpointing] Documentation for local recovery This closes #5239. --- docs/fig/local_recovery.png | Bin 0 -> 426255 bytes docs/ops/state/large_state_tuning.md | 93 +++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 docs/fig/local_recovery.png diff --git a/docs/fig/local_recovery.png b/docs/fig/local_recovery.png new file mode 100644 index 0000000000000000000000000000000000000000..01c429535cf077274da664c3dd66ba9ef6a2a2ee GIT binary patch literal 426255 zcmeFZXE>aD_cokJgs9nx1krc4h)(n_>_no6AdDIbFFo*^;TU~ft=(f$+>gq$R9oY zQ{&t@B8PM5h`Wd{0aq*&p`Xv4JI`h%FRyMRuORPW>)@>EXku!9&&=VOxrTxqznGZl zxpQ}-O^uB;9&+9JVsh2k_)9k@JBhQ0#+x@`8pi&g8$UNTHMSUcXCx-*>rc~MnQl3E zk?-bp-G2vZ$ zOzyeOx}Z!5B=GiIKW1V@4NgVbG8eyl=8rABAH02ei!V##y5!co9apL_GBR!&8rd0& z$A@B9=+nZUSzdWY{NhUTt)|B5hM0hWIZC;%toQ(v0E!D2j-R|vNMgD0>V@=$mU^+T zko)H9rZkCRv^2yU=XGu=x~+UV2dv1s%Z?9qUCy23xpne)-V-9@e(s#?xkrEAd*XS1 zb&@DWUk7|=x|D+x=8froXP&3`&Zo7~m=EefaQ^DU7q71pOA@d|#m65!47jL#^`Ttw zIvCOT83(RMx75?zA+btxcdf_6uVJIBXToPq3|S>$+I^gM zoMRKgdgmhX6=vBN=gt%SzyIlj5rC_A7AaK!`z?Px%}wwUu6psmyyVCGzst(53lqJ{ zxcvYA)afX$)Vfjp?U-bliGZ(((OgvdyOGPjcrgug`5%TYcI8TKcN6n-g8w%Ee+}xQ zL+XXUAM}@i7t?kWm+oEr`{iWzT%rH_LCaEHyy)PdegS znf&Tr-~X^H#8cu#e?RD`3+I~<^kgq@|NXAWt`Cd-{h$NriHWD?jjsp>{Qb~h!h8OH zKZySkC17*^e=bVfo%iU@BuTxjv5g_Pnc7}obKB4Tmot-L6QE-#eyb~0_`?dh-Mv$N zpso$bbcu6r*uNz4Q(R-cyvhvpIw;pow4LQljMX&>;(M5QJwckc3QGT_0b~s?W~uaa zK73B}w=yck&Fl#pXer$KTfOo9KA;kLZ)Gz4?OlUqfO1ixZ0Nc6%fwD+ayHf|YG6vW zVXxi)_L!;<0J>913NNnqf4$)BO|mb5NnUi=x_fo}Y{7o{@$B%*s*n;-4R5cnNdE0o zc>M+_*?$%c*t>rg?5FJfBVnfp@Sh`jrd>Awc=VB1%7=&dZr~{YZ!B%S4(4 z;}nl4=*85K((OQM15wxuZneqp!5;QkYNcYc1{y9=t}-c#rAy@bVgo+1q+%wu4Bf`8 zW>~4O`%}0LO(}@1U8_Ztd|ml>!;L4_{ETkQCE6SRjMNkeU82gI*h;JvtE3WLTbhM%Pl7#SGNyX z+h_eSQK!!I>n|N#eS8^4mBu}zcSrP|y7lUcW)_j#-T#m8ZgT)O+fXP)T>j?|vSNe< zPbZ{u-b0jm7VhMRo~g{g{@JDno=6kW)>HiDiC99O1OXnhRU|xDAyvxjX#$_Mx>n?p z@xL$53t8zCtIkiM_ls4Rr69aCj3uXTEqBP%FAm*z%y|C7A{|24{7}=OT5vU;LSC>ykllaP~ibgTZu*>o{Ig-V_n!g0` z`E0IRdvtI7``69|&|e1pnCEqz?Wou1;M6YVJvr_&Lc7A>-8Niv0nv(_{!pC z0D4K8#FRz<@|Zofr+uI%n~u2X1p39zpHlm;b#Y|}2FV&t{_6m)G`|~A2%dSh!YRk^ zJ^F&r;owckAWY*weD90}z}MPMDenLEhw8@}4(ZkFDI^f#db6WFbse3N@H&_0|M4g@ zB`{7e8bZ5Y7A!-7_z8#Grppa&cd1FnfV}8m7W%LK2E6vGXA8OQ86bGec{JmQWpc!u zwy!~rMliz&Ch#Qx*fLd^BkZPlM*dE=3pRh?L0oxHnue}-FNM`2b&&9Oq(G#PVqWJ+ zPrxBAsOf@FFRmOOxzKn}&oXIUzKua-&(-PW9b!~4LwXGIhqr$$*EUta)K=l=wBHL- z&xVzF@c_GW?SlTt<|<%EuPKPU{>=&f?Kk;LWfR%cYBn8X7lI=L>w=S-r(P+5*5GUQ zWLW8Hfr;bMwLVw88Ps&aDVu6t!oO$Kacz~~Wc5vei7VNih>#U11>QsP{v^C(^(a-6#TUD~V{U`Jztt94&wr<%PH9uzrDNT3 z{56Y7M?#ZH$Eas0_EykfaL{?MJh&nIf=PFG_lz-W!!cvy=|c?aLb!^Fe|02WrG_GS zXajFhYULjZX6j88m(=$getqMW)2*A$m3#FJ!izaVF={EIw7czo7#S*u;AFjnn$_sk znJFO&5ZR`){a}Y@MA*W@oTUnxRaYBXWP0I7snDd~*1or(U;UcQO$PRYm_1M9PK}_g;1nK)WGV7|R=4SW;6>1n+I8q{qJ`@Rofz^=w5-5-E`8~$Exl9;0 zBP)X>kwo@NV~>=JZGxD|^8|5w$Ap_QaA-I~fB$it=q_Ca6G?-wvC`grcjQ7w`OnZs zA06RE2Ord@ZI%eJyQpLz3tdK`_`O%YPTZhEBhBWKi^(xC~1#0*fm8L_oOkK6YYBb z!3s_!%-(;6>bpLCPF5*u(XvyxHS}6UMT1F@rIGiyvbf%)XYMKi^9?u*k@y5_-Elz# zW-KpGne6=onZv~GE(VQtX~Hl4+_8Zha348gluMc=piQ~m4PRxiQQ3ztN7+W7sDoT@=SjFZ3F>lL$J+O4Pv zn-oAq@ko1*wGO23yTUDVEBH>FC9@|9$slzc9>3~h> zgrS4{_uf^Rv{oC;EMV;@8bSgl+CH+4*Xvib$9fh*+jdq!)CFRe&Yjq~@1D_*d>||l zhl%`^b-2dIO^DqWk_xB#TK&d}1I?HF`c#)?q!3(F5PB^l%4QAegVuvCE4JGuvhs?+CLZQJny~#K0 zT@VNBu}1l5A%WHsPx{D!B0pM-+Maaj-^0R7H{4ex-pv(duf8XEVA;FItl`xWmwi-h zr)H{DJt=qLWA>UsVgnSgE`}bhU5;90_Bz(LopP~ZqVGi}r<}WKO~!6XG#}T@y)E)Z z7rc`Ya+&RmYt-Vu4jBtZ$Hvs-v4)-&N*M)0oU%R$w;g?NOfL{RnAjT4 zZh{%0wc|#74m@wXazuM+HcQN6<=8RVRDHFcQ+sg;PM%4+qh%Y;W&z%Jw8tx???S9V z*2P`m+Vwsqzj`B+hy`Y4{Z*(Y#|``= zv}#k#0#6c6e^|Rw{wRIDD{4G0CNg$WGsi7W$mNM=Z|8h#S*g4dY*_@C&D>%$8#&l3 z3Z63u z)=(_U=`(dJpNvV++rb)&vh@%%VUxG#4A7bpBLfGy)PFX()@)3v%p7>~WD9mq8fX;l z$QMej#Ec!!JsH;^)i!3>#f1#-#-pUoJ01$W8&IC|52a?nPxp||eX3Y>%X01W z#|^u*u%TCGqU=|X)}VQaRdbm$yJ zQsH?Os%)F~l!v&&b9A@VHAt5H!5G$={?SMci-frG_=VOd*+zTQF3+mx`61d`{YMVf z2ixm8GL=Qo&KD0>y3FRcHTWRBOw^T;dTSa=s6-yaeLRkr(`UNEO}YO*3zOgBhjk*} zG{nIZo;A(JgZG$PW;{kD-Qe3+S$QdY_c~LnZe{vj-Im_4mpxdT63Y(TZ@|w45&(9c z?5AB91;S6eR22SahWiPK8ODzd(BUS=crVo|3tZ{^HWd>h1hG2W65+mlVWq}U2TtF< znj1m&rx(u6(!E?)*YFrk<9INgM;%1fRZw0ch{~@NEaicH)mMC!BR)`F)Ed`5n^d?0 zXW(*&m)NIUI5hBHGLt?4h02rke9z1Z{Y^ub12#FvFxN`0O70LrEtf^x6w9-f9Z7a9 za;c?b$4kowl~^K?)m!Vl?yZIzNMXQ6eV+vhX*%r_;55X#73uU_H{Ifmz?@Xi$IqpoH*R zb2ifL@z;A*8WN>-wpp6_Rlz;~*x{6&jFe9p}Y=XE&HPQKwtJBL7vg#X;Q1 z!G2F^nV$Y5lxLH=kYz{%CUz_DZfAL^%S>)U^BxB2=l76W%vI-6X|&8iP%C$7drcns zQYE`14%>jNyD>TD-+l^zu6$_#K=V=iL()?buhTH@Dmp)NfYqm;@1;r1JA#y(Yt{ft zv7Z~QxUzP5KSe6RqFJjZYY=Y5J)y_JrYanq%=9JhrD%5q4tnvERzpvHp*g0=VO+Tz_Db(eFOt$Oh;e*R+f;p5_`E|TCag1i!XCeCCp-4 zI9x@XZ9wcUU%>?g11sUa%vIjXFD3QvRfyy%wqsnkk{-dQxq_Og1C5}!Wj#(p76#P) zjEK{1A-?ilE#shpxO{}9_YAZT#>kT7;`ft#B36*Gaqblw&tXeCoW)Qx9XnNlmqCKt z80*rIx-~>k7;YM9Wn9sLbr~5AX<3%F2W0Oy}{pQAJmP^?J zk=4%HRk?XjrA$$lN_xH3{jA^z_#sTnwFi`RPnYD&LH;DK>q=h>Y1vpw4iL(-KOZqz znz8&$`fAEo#mhnw?sM05-K--ws^4oNC(C#jiC?QSY1>;i`eSEP2^ySuEqMN|Jm{y& zmQ>2vqc`ZyZN7E}Q!j>G^YqY13KOz#=|dwJ=%w-1rOoN!je1Fu(p9?^>XBm=D{lYe zbbHd^NJd^CU|#EhWBwZLLnmOM9HqhLKS#Y1vZk#UFQgQ81jUthU*)L|K(9tw^6u+7q*s zQ#OU6)vU%o3npENATV9(Vl7f!=??c@18jj;wa?=NMKx2sVn)MCwtR2B>t zABhUqj~wsXEd;`kTDWKWqL+O~!XnP&;h%TV*H7)!c>*tLz+&NutxmOWM*L-cI?MC1 zHn6S}FpkFEZ0Vso7qSu;@y2F9oR*(2jBe^mKV=IfE_5N1(xK8?d(S#(T0f_K&GxwL zfeneTUFiAtTPkdN1R|(s4ejk?L15GM^1-`KgBj}*z3B|nTZdW@?IF`$wVXFbbg#gz zah!KAIq-(E=Z12#nT9MFxtp%C|EJ}5YQe7=gfz#6d#`bTvBrY1aM!= zvObfs>Kh4sB`mcIbC7NVYjMBdCpM=m)?Z18ycRSq`B=hS0Kt7Ag$6BrWObvnvCD9S*4x%GWNZR7g5uvX!+$9Qqvdc>QT(bx9XU8%ln(XBSDA! zvWBFr5Y?WmwJJI416<#BoNilXvbJ4mW~F@zruZT|tBIN4*Xm7-uIj-| zR7mR|9|p^Z$xuIGL^v}`R?I~gxo;h)I@WM{`fQKBE&9D)2NRLb1o#YO zOCLf7TO6N}d3T32Em38t+URGGMML5`Z``1+g+Vs*BXc0g>!ngLdi^*|@WvK4F1RA- zW4mevXtFlp@$CMBDuoVD+93K~Gi5tgrTPk+l2ydOkb~EA(9>#Pmr9$R2*A}FJ5*Uv z7mufI!;HfXd>+g+$*h!|a7-xIvXu>?@lA0~9WZJn~fE zUgY428t%x)jEHRUYs*xL;4!?s4v)&mwOA>oirKOd3x1mq_Xb9INkV7IYPncpk$VUy zi#~-0M~Aq{&XV4^$>0_vUvn={SJaW}3c{s`-PdiYdcE67ic!OQnd)=5o`&L3d@IvX z1AyW3wYEPUYDDo7jzYsZ{m9z{al524AWcXAfk47r0$qKb)xr)<@e5d z%_Fy(<{g=*T8`5db^`rji!A(*O{ThM_Ym2neknXOIqqV=-5Qjswvpc}OfRs)&IZGr zhJ6vFd};nY-+^EqH!-`p*Ko;t+2c$TS!AE*_Mi8(0OMs??D_0rKuta?;fxGjFVb~J zS!Zsh^0r$wF6a=<7w1UnO9UZQ%58L#TS&?^f&3*7fHLrecX--l}B~mLLoE(y7|FX5g zJE=36^rC&1E5{$dE6+k$g9KnE3na8l!_a+W8(4B_#)!|;_qk)J=)s*rt%bn}!*$%Z zJ$=SrUcT=FO&n?PcC`71i?s>f7bt-}PSr?0Zq?C#9kVHB^XOjbh!H>MfbFJ+Wd=23 z9=d0^zev4CiY*w^T`!Ocx60C#6@gT$uzQxq2$DLFkFpS25`Thh_azHy zc)h-N>iTeN;&f|F+0i;q+Q+g>kYA>DuX(kdcRs-m4j$N2b--CRMaD~2f1JasAX!>t zVlrn6>;3#Yxh5&wLm?sDmhHwaimd4tICqanOPLPgrFl9sRlLx*_9Q;DNULGrq^8v2 zc}7O$$M@}IA;{2HS~rgd`t;Kk&#=5~)=&nw+|jf-6Ygr==^dS}F3Hl>UC}%?!do?Z zY^5K0pq1Yi{O}J`$VYa%aw3*JQCeeuWSCF&a%ui|doFSd0}vU2j+;vSJ5IxQB@0;a z&7@GX$+~v5$3RGT;&{rizp|AhDK~qoUGdzMP|bL1tFEn-E(w*Cmi17*>6M{i{d*4S+Q75@vnL(s-+Q7zKa^Sv*3)G@! zotcnNX15g=LNQT6A$KsfzktanV`tXuF8RLwEztsB@CGyFI&-f$08s(Pc`28sxEZYJ z82RZTa&wPFA;Er_9gm<9{9ONf4n~sRui93#QbIbOXMDUK`*4(IL`-#zcg@SyQs2m1 zzH%llT+89(g{!B5vcU*IFQCK9(@w8ctNaDT2p7yr383fZjr>tZC1q0G69!fFC^u&b zi7%{M?SY9--~sDc!B-eH)7<&82NvwC5UG09jBP-n>qLxUxwhhg(^0f2f-ZHX%BoKDIOGg<)RcR+R zpWT3U6diqv*IPQl;*-(yzP(K!XLuW`fA{?^6JO3%uHL+J4R&$uOiYMlxPX-R-At07 z#^~xpEA!F>gHndKKt&hktVwp!E~M$Rb4>4% zqn7Az18WxSR>R{jj>@-sgCMk)NJBqulO_aLsl6t1LrjKYoRdRMZ_&L&0ob?X;ulbet8^ zaxI3ZasNT$vbV)#FJ8ic-g&s|aj$Q6XF~&i%Z&x%&7<%rCrIY_y@G(2n^usJivYvo zhy*$e?1#<95HI1hfAK5W0{pzJ!z#}3>$Yk@S(-uI=3*OorblbcvZUOjD~5o-`%jgX z(Ho$iM5*dggS!sLhZ4eOQ`kfaDDyfwKjC*bYGv@elEKlWOu-i1xa~^^q-or@d)IZAPq}y@h7Onl(JnsC#hiI@Y${L*4t9`iRPd z(WuJo*+YS}yvMt3v~VW;#H2~*T0as#^t{mYY&izwPGb7u@Tt>Nk;~Ip=wOg?S@7O>uiU~O;6}Fx~sClN{`6hh<12=H9wr)4YM4!@wzUVytbmzTx!%Y2kSN{Dcl{>s5li*Zh<;?F z%cW{A)6U$=utXJe$W_WlcZur8oxIqM6eeEruNtbKbq5=2{cYk5BqrEkw{Uqx@Su1D z$+{N1jtwAwOy!?BSRy20nuOZyu~l0b-SlK>4s;2vdB_24O&O*%0%C(=qT;YyFq;|^ zT&5~~n@|q~UwaKmP^9TsJNkZ;szlG&nX4f6TrX5R z1&_TC4)phKC!7(A?_F)vb_Fla&(9a}n>Lo$jf+_5uXr1SO#~be5*9*fdApt#m(6Ym zyMZ_aIwfJpDLjg~Pjs&~*p&LJ7Q*`S4{Ee6YZm`7zw^nJz5QS?*8bzcnews*W$dwE zDMv%FXRg_O64x7D%lWxlYHR4#xpziNO*32qEDP>eQ#h+07r#^xLFYX!;#ncfs|Vv= zo4h&*mLf{-_|`fxQY>?+594KK2^|~<@FxwU;qj`A2<)wd&9jYC1)@6;pOM zdlh3$H(o|b5k&;$qnC44bW7#BLQ2OKw4ZZNx$^h-?M*&bPUO!l!Qi@UDB{7oFP|M) zf`0l7fk0B|#s@RH(<=d81b}byr3|xns33sWSyZjLJ1le8)Stv|B%*TO8luR4@2!aw z45@c{>h>{&Q`nMZ3Q9_F;#je$wn;+tD>C+U;Z}BYKn__Fl9q;7IQ9cK??scFJgZfq ziM4;#{GI<*a~MsQN)y^OT2_q<m+GAw0Z>>rtq@Z0v5vpzfZwuaK!^)AnmSg3`zW$ zku01vj7B3phrDK%45Ap9ze8!aX!MOK&g04RVVx2#R=yOWD;i(<^UWsV?K|w6S74)0 z%032@lbz(HCYVOLZ9zPt_a>ip!YJ+Xo+k}GLuhvGq8^ah>iWB^mCej_CAj-^j5e7( zNtuiv4qvl_6n8ZZe{A2Ru0>m+#`rQE7UMUbz0iLSi zBx`~h{zyK4!_jDqFWMitdMnN3SjRu8o^Hl@0tO(j=>7-F>-0)M39%XZftL5Y-S7eK zbGcD4c`l`uwi&A;>jk13ER)scOBTA+{41i`&7W?%K^xI%r@AptYBi1nM<5up5i@O< zWF=y0y>UKS5C_kW5qu@wEuJP~_h-U22X=K3CuBmvoU@9@I@7d&pj`m`z>*umA-W~J zf>mV!( z1K@%Jvab@cN(+&^-fc@g3DUU4QjhdpZkc11m8AN0e9lag%kAJQeq&G`WTFYN=Dezc z67!VZO&6ySW-J`N@?_J;&Xbp9ih{!gUwyMSbwfh^7aQ#nuzLLAt50uP`@%m!xvj!4Wd&CL_SDuZ*C;q-5|)AH z6x$LOVjfCo_oN%RKcZK)5X~^tIAdz^Jle(*>XsItA}u~qW1&MjX3n1gvaQ+=fL#Y| z6^~Yw!NGbhSZrbPK#8`UPvJc%A64yCKeu@Wu6A`h{!WPC2E9qnQ+dn}!q zS&zlnYUZpi18F(ZaD}b-xZ2;@0u|3xmUo#fX1@W)C`ESj2bXU!NtR@yezDmqv3m0T zOZd*=f!VU@_|jr=2hpEN5$Ro2wcHRRg8+J~T-%1yYKuqZ6Iw=FM+Y0%x@UAK?a4}PzHo<`UB!!?`c>I5 z>gY+plz1BZ!}#*77-THTR75!YAm`&{+;#uoG+4(U7l`AY-0`k)L4I=+(gDZa(Ah4Nc*YA5TgxM2mc z2rIAPEX3-Lq}B=cD65>uD9uOO0A7wi$eoEIPk5EvB0kqPxj*9=iIgbJ4M4ELnaKQi z1ASUf5-eH=PnC$4x=~(>4|fOKO@p-71UxnL>8h{J8Ix%*n83a}@4)1W5piAD=Bz(@ zpBlM~#7{IDyAX=Yxye5yNZlf!TJi9M?!0@2Hk=olVrOs9<6YNVcKnIPY@#~bd8Gk5 zrlqUIes8HDLaabepj0n1nbq8R+`~U*|7j>IOrnOfAX>Eg%Zr|($_MKw#DjRA1a(0K zuM6P&fVc7KBIyi0NxFEhS-JD7pSN3*Js2p|Y$a&4^AQYjg^26J?d>^ciHG`&PBu<8 zQ_^YdQHjI9Gd?xKhYfB|YUbSa+MW)1!q8x08iIvJX;ff?G;$K*OQrfhPetD~fTiKn ze8=k655O`#LeN@JH^!*nP|r`IDJPcb$I4Tb5+;+8O{=bJQK{`ck6yg$mw^N|$<8N` zTyh9*OKj79No!SU<2G+6KB-B9(lKz?3$Q#KNUocGw{$`fQj>W+{Snd$r>ZpH5mq`h zipk_fm-7)?F#%c7d>|5JJMMMIcEU+ik`|*mEey7}6HEOma6`GND`1EGzG>X%WZzKJ z?eK!GE`;mEc3ljsXN~LryyM>U-NX)q9kI?8I@H2F){f%JCog|;Nix_@$Pcd{_docd zNgE3ZAAH^sP~yIDKrM4E2hERV$qvOmt_QZ(kcL}n@vHK05OWyBjK*Qj zrylzw@t#j>y!;Wll4mh}X4#BnAUR&+b&mc=IOLN`K>qvA2W0G0o}lH(b{`5xZtK3? z)-g3rxs9CPImK}7Zp=1^xDYZ!>cHzFJgkl5L&b`h$wr*lg}bj#)~={0Jjsm7wrfX2 zukRk25Tf45Xjj{2eYJW?tW}BIS z%F2zA*PZAj5PMMH2QXZx-0aH0NPc9-|k`>C-r14y6B?~Uljs{)tI)?|4R#c^#e z-B56%0%m+G+iVJ6$1t#uOX>_&tD7=3V330IpjO8eq05+0ur#)xOq&cBAT87bu017G zeR|WF_WOklSU8RFCx7L8`g)r&f*(J{XsmW2d`8wfTq{d0BPqa|S($Qkc!ALL=-4D@ z%tr~%!4KKFDy0(Cy*IU@P4y)m^r7DAZ_e*QZyIqrT3xsJEPc~*&wkuXbbv<7I-4~+ zlHy|!#g`|CBPd%06S9tfpuTs)YIORj*S7;nd(qlTa>TwPjbty^vq>&zr3KEA7OyYK z9fO+9t8kOv+5pow*z|@8g#Qv(5TX6D8l+Q+CrNu(KCAKg zWteHhGlMyt{w_s)Rr3ylGALgFL1L#UMDRJV0^(LP_P&(Rz5gcMXis;4m>lLhombva zcUTYX#6UsIjIn7&dG)})`_jt|b1{ZX+ZfFaU3gArUit(z1(TSeYci)>vHCai;tRh> zPy@giJR}b!`e7CVmM>G$owGxfETn_RtG!6oIJudtv%d%#VHvDRRx0S9BLZJC8TlA` zY3fukirUm!O)w&oV7#z)p6^Hf`zV3JuZYKoOSNCc-+Q%=PcmqMG<$XKRwnT{?wR*! z+|F+5t`^>|?j~y9xYlY^_U+qcA*Uzu3cOzR!~uFYdtJNxx+!hBaL>rMjJdW=N0`A>T zm$a%BlL8LB?M(L;A(X=exFL*t?7T-#lwAbI7E2#K>P;v;@Ub zZChb5)+sSfwtEt*=vUSsVP#X!$O}HhP(`*R7Xe^m@aqZNOVx~6uz}AH86zR&RlPWi9_eQ9kOtdNlvjHrOYaNy zw4K44i@A5y(~Mr_jv5F5e#%(86mX5d&V)Un5zqt?GGsv=q9EjG$aJ&1%<<-F#|0dZrXpC7R`mGcZseU^8@dotY?}Iq)!| z>PgY7$I51+fh6EW!i<2mF&G)A+dQYtu^zY3O(DQW!Ckh*jhHO5HT|*|tvTpq`;&0X zj5%Q!GO?g<{cA0}VU2}~PeXY#v$osQ{YLK3;04-U#gpB4znjc_Y$zAS`$Y>FkH2cb zMKhqK4EkJ*YMoT>A*7iLc*L@(;&e5q{Pw9=?c_S?n$A)PKH{ELF^ytJR&4B=CVv<&Y zU^_c{nb4)p!unNe8n=2VDp9(HLO3o?iW#DQ)CdqAj*k|v&q(nmQycny-FEzncQTpU z{*Buz#K?E1EUG@cp4R9%4=E-R#_$bnBj|q!ZI{zKt{8PbN;H+%TM$_pUxc-XZmg`= z>vT~%aBbFbLKcs|PSSz$oX)JKYzCAtq_v)heJ5QusnuH)%FHOphdX{3&NO!A zXdbrET2`UL0!yq~kkwSWa%U)USUBabg`=pQoRff&__E0XzbMQgxjg?VTtbp&MAg7k zZ{yympA-4$G2KWv4o@B#Qz{HF3yz+)h?f9DW)qt0QR1;*>EsgEg>=twS1WVju{ zNF}{5LmBw7oy5pvW43Z_CfLwNo2EZXYI0RA-i`#4-#fuz7%M$LMPpXyfG%(}By)|) z`z~beylix!{8gjhasbH2+Z!m?q?U)#@$>n9-wC~Is)e)?2gOwV0>D9^Oc~DUvKgy7emD<3+`**LP5B(ACKB9T@;%;j z;AQt>=xJirr+;6C5}PX4Zr{Rr9Im604Hy2aXQg^J-po@Si!6#NRalZg+gkhsXa~ z8(A1JengFlJf54L>E`m*VS_n)>Wn}Bd-S_t*6ydZCoC>6){FTA-VI9?xW;c${i2vK z8$9I2=PA(|>%HlNsNoO>I?L5mYD-GUlXyrR5EvqkeY#LK>JDKVl)+cupP@9mD>AX+ zn37pm!*7c~&_oEP5}W!@E_e+*_$-)qD->2ei5}>J0pTk+P7fxNXz@mfK_c4gwg;%e zhjEOSqLp^fi@Wm~ZpzP~Vi(#Ex8{a9FGDDWhqW((b{c=sVLw!Wix zI$+E)jkxWe5N5;9Prv0(loCcu(hILiQuXIgsciUUvJ^y1uv@%N5CsU8V7<7En9#ubYnq^J z(_hK%lB`h}qs!q}e*l||cv9QQfObJTq3%1~F8f|7R!N+iwqXk|7s_7=ObH#``+S4Tq5$M zu;9Br7aHn15h#o=h7sw#++$YU2b`9K6`MQmI6&AwS{y=K>SoGU#-6n&5 z2cC3&@8+etEmYxL0$6eD(CiA`Gj*&2!mJ+#6w=@=V^70U{92Ynq0bGZ6XGk?WX4Zb+$R0FsC@JgL6mPi+;uk4RIhoHO5VmB?d{qUjK zzzh6Qx;CtYqcEw_{ZH9rqmWAubjcC$t`BSYhm*(4+)g)kZPF!5vx2gO9F-CY6LzMp z>Q)a#Ux|R%?x8pa5?dHtbHD0EDg{m_oF zv%wcT6Q}DkXA4RSjQ4w@Dl;)(j&Ff8av*F`a<~MGS}4J`OS2nl4CWQ%lNbx0JJ=EplVFBGwM>#dqji*EOLPoWjSG4f9wc) z7a~37U&)T&kB<-T8RjT_a-p9Ld%QJtebRYV?!fgBs}txmvdU$q`3tS;6FcDPB;B-6 z{ZSpW$x(Fh6+EMMY@*)yj`1psYl**($=#1EXvmvw_nLP?934)he439B z@KTezb^R6Gc1-@q5#%#9@InV%O#9D`AjSi7HJo$m!+m+%7~)oA%C|KY^)lG&&R=TU zMu{o!x#pUmndM%K8uGWa3BRHb?d)Zn3_f%%J~BR*@Qd~!M8fbr)h3kStr`F?&`uo} zdOYF49DlBAAnjvbZu-1Vz7CeqGnRxAjfXC|{u{}Q@AlEWuAa+o2c%ZV!Nh{61e5i$MliqVW;9LXBzSspKuvr9v5Sk#o_~Y4Bs5lEEB2Pd zgwrS17d|7oXe}{A1)>vNKa+aw>!S>+ZTP5?yNfST@Cf2P_O%;|@W8bonwgFf1}6Sw zcQ^t`xgwiBOA(dU={SI&+lcyos6Kzt?6bSdjcS)ua({snxTKzLU~1Bv#XO4iLmjP; zG38UJwe^N@neTp|(RE)L!qN0*2Z?a&I37}Jw_-97E&Y`dZm4f4)^ML(OuW;BLkxJc zBC20Ae&uG$cAdWUSnY~qrw1B5nE(~t$G7Ul1@|6*|E3g~2#LQJK~k;$K~W zjCr3V{yg_t2SV+;`>&5#qHlnU?xLfH7Y@JpPti&5wzbXx=%V9^P5V$5cD~*8=Eg87 zN696J-UP{f)T_aD&bFG}~MXkC5f;fty+zHM`xt$qYbR_eq2`Y3?n9m>7b&75Ln( zfd}`(D}FokjRWkjE*=>DJ~_yDR^K*Q(%967nz@Gb<(r_oAbG;t<#V$&f0f8iyn{}- zy)T2Rj#F{s?p?cMb)zTzgz^6dTtA=kbtYV=o8+2;)>`l!114SH_mry7v-b=O+D43i$6bSDeFXqsxaBZFc@Ct5a0UoAm)LNS=rHCicWS5Yw z?wh4FyemS+q<@@mrwD$wRUpruW9v|9IlvxJ6muY?B!=zcXCdsaUMb~PO3o{f+>q1~ zhna&{I_$XN+lt~13gjyA^Co7s5FCn4H7FGq(T0dFa`CTb;1)%1R{nNDP0IMZf9jid zB?0YDt$q)5FUrI2;5d{zO{RRV#=UEO7NPYVD37S*^<8U^d>y0v=g%OgYz7~o_L@^{ zPY>MUjQ~N-Iqvj5t*6chi|ezb*+D?~nDX$zBDre7kf=W_)9CY)Xk?We*wc0;U{^Tj zz=b`^w?&GsB^+x{a+1>Hi2zGW>^9Jss!=kC-55bnFsy~KCk;f7w9iJ)dhuHAqLIF+ zfc5v!H-VCdKo^Th_OH)&-utxZ_!aX2*ar*E>8bI_MEiIiP8GRfXZSMTkdoW`8QVmr zFX6ZLs+>IZOR@JiT-eQt@3>X?CJ`P1YEkcH1-w;oioJ2W>m85asK=+A#TOez_jb4i z$On8LS{XHZR$$)sz+HWOZM-Z(BbdhejsnEJc8ycGL2uvPRmmcK4RBdZYek=HRPyx* z8s4>Wr4cOJPJg@G=DK;pYQi6fa%=T|e|(wK?KDq+QV=I=fLCyqIOZyF7oa~)ov5cY zb|egArACmt6RLH_K<;9>1wwlJ_5Ogu>#BhVui0rWzMn8oFC^6H*?SaBG4^vIYvZNe zSW?8?{uITA@pA`EoFMo0=Z}$v4#6r808#O1(J!C1D9!?(CA$X{R9%;p&us~**$*^} zZacy!jBlkZMssO6=BjyTraFrrt)~-Iz{|UCBc|?x@>9Dd?`QlG5SSfKwvfs%T(1@! zSk`8%Bj8La%ow^9g$<15-%FM5Wtu7y$%-jZl)e~dY2G#?fT_RJ^%iN~8Y7gv8|dM? zy!*`kYpoq}PpZWDfiG|S*m9^TvxjX;w8ZazBkpeBp6aQI80Z(f4;&S7)W_?5*4YN21Os&zqgxUm&SF-Kw}HC7mP^Vm{{q?p*%z&aJaJtcOh8DHEQXBqRxlZ{ zJPPHi)ElTy-Jd@mW4-ZkV510(iH`EqSAkDaF@bgHo-&RK$;Z%`?p8)PdKVTj9xg!%2alWnG#eK={s2fv zEKi4i%Iv?$Sd(my_m!~+0Xn=~)#vFv=t&$RR=VR$YiBxi6LWo@fux5&_6~YVoFz1` zFh@PgP=nU3{&{MI&(KA&tX{B?(xa8HMtqjGaR!J-)A?$u;4DCskfro^tq!@qw#@5Ay zo!Z+rza|&{LV!R=7kPzs+a$7&cH zC~a#lYC3FLZ`v^#{+xLdS_;~lwOh=vy@wcled3$S)OctT!8QbcnMa+0?0N+}vgqiS z4FzN&;O33JVQ#gudi zKEgp*Lw`$qjcbLIwKBDsQjb6)Cggyji_}@kAW#)fP69NHna)_b`^$O*U-+aI02K67 zIvjtuJL$nw%^kkEc0mEfHBHnaA?*fw3J|Pd6oG!fN=JljgL~Q*)xW<7NSck1&_9>F zakL54-;FrjygB9Dy$VF=Hnf&(6vyyrP#Uq3K{bycW4F%1eC|0>;$NOO0LJG2$n3%| z%H`xFW{8kv=@v5KX2B*WA;}X~d3+o5+URfO=9uMcAj;Qt7zDjTa&y6J=1(3*bfE*2*EFN+Bk0DRf5}agM`eEf^H2O@? zNT#bn{19>K8^Q(A^*f z&>BvL75}L;fYK>UCbS4!JSjA2YVb?9g$xP81J{KnhSMbZsTTc3sQ+4pG@x?9?SJSf z_D`R;bLao*^Y%~6z<*i>{;#wQ%rK%D|E2llBr~5Yo;DNN$TI0@7W~H#MHQnn{e1&k zke369$u|gy^L)H2x=j8{Y&Ikfr zL>td_x71U>%6v@Faw*8MXRz9Ft*n~Q{~7TiZrJVHqNb6IQ;AqUlA*Z;l-#0XlhiVN zXt26wP9XJgdUjyBIqX}Y(4I|a5-Flj`y$B|&!&lLK5NGjI!4i}&6JGJYi`^?an(;n ztp|J;#5(cQSNIa$_>P^W*32Z`@nH1I!C>mmC~jyX7&jQ0Xew7b6u2@k5*%FddcAar ztz|QuMKF|&O5O@VPm+cG+A2T_B44!WRQ+B=lr*SG$djr?tx7k!W4RRv?gn}}xkZ)# z#JRE}C+zKOwJkq=tmkURn{Tf8;cBH#X0i%i5unk?F&8&};#x^E$#n#Ierp>~4Yaj7WB&iOO^ZB$N4NIQL;vUc zHQY!L<$5jSWth%q@zCUaf^!puc_Z}3U|{R}+D(o-6kQ zArVL4I^OepavFJ;x3xK)j{ammQTH>7{AbK_#-x9lohv&ZXMqB4!zPcZ(hgC?eru(< z-LQ`>jB#RD_h(7%*=ur7I;q+@wg2FY0N2-T0!n8ON|~ylYBb_S)dKbix)&QQX(ZDT zbmLQbX+Li~Rkxpl053);hT+wzEWT(IxOk(j5zS;Y#0IoYI}AV!Lm_3|n3@l2iwj1V zVSoIa6ZF?>w63q0Dr;3S{OA}9_%pJ^fIP4FC}~Cs$^b8LO1zw{{gpJqg3fIYLqUjHL5jg(^hp@wO6FIiP*7c6ej$=H{ew z(;qAZY{WD#v~dn>N=C39@L&J=BGumq+ZK3`_lfXUZEVw%*f1f|#Apb$_9D-~+sCLox_eb-uB&*YogcF7o+k(8|__{&Ym0Io|jkfpSk<#B^6`XKHd*$ zAGfvdivSV&B9Po4#L6{!EV+7+HBEeFT}beOV8OrkstXk=?TYhR5&e6qVWliJ4r+j3 zqZP}CoSf<>tCJg}P{tOt?&d}|>4@b%+xBrHrP;yd-88-Dj%UXG8_<1a@UPiFJhcqc zd=Pj5rs;mHUk*EOAzxNmzfgFmeS9gz{RZDHvjItY+?>WW=%nWGM310EGGD_+MoQnD z^uECSwZ9(B_n{WrT9AXnDhPv^|17|LK|>pbt;W1vtrsfuk2;d-L;Nrw znNQq@<%d4-$KBd1S&7PEF90_`^@jhl{>}gw8myz~Q!9(0zFTO2I-S6-ZTIApg7WjT zC5!HNZgVS?7dBlrbyn%(b6wdT)2HAjCPhh!8soEOLe0Hn_Bt>YYpVbancj$Z;i)_r zrKm}6p|}CE6g04}pZ$iLVo>@gMV?-jm=KFbc~ZdLJ4}t%X;ElulqcOEx!)%bNlB z`d`J3XZU8Qm1I_Hz5?6-5ONzVxaGY<7rqtDFQ>Q@Xy&?INNOLqci&kq9I0qRG?5Qz zWcFY8X~KN*Z0Ek9T#2h!zd6@1c36k;C^AomeoHTT^5F*>7kf^Q%v`s<`*a~mE^FQB zXWH`KU^UyV+#dmZlXW*)9-02o`|884#?G$GUQEf6M`v{BRl^5&cwG{w)(dz`-u`qTvgDI<@l05C~y;VAi@Iidi%}&1#>Zh}f z!JSKfcSml@2npLWMVBsQx(e&e6Zx~vn69oGvnI(H*asNS5MXAjQ!a!21<9o@p-SqF zY34${JxE(JR!F1E^4e>Ou411aRp2@Q_C#Z;a?QRipHrLbo079xh6pvY%$SwA$DA8m ztJ!iW#i{`|zACnE&jk<7X~-^zBNnZZ{r#&k`pG1`p32OcUBrE!=N5X}rz}v;ORD80 zSFkxtu7pBgoKNb^1KjEpH0X;M)*!gyP5GjC^e3X%t>{o8w^s?4Rm zkak!{JV*)e-HPVLU6PA&9))@AAX+8Q7WVC`CD&5Kq*V57fUbcn(k%_1bYE<=B8{@lGVNo}%0o+>{-k_!BEGQ@Yb(=hbQ_d#-j`pdlN7wFy@{Ra?&O9mm7+Gmg z{Lr*hy|H?PRipx&)GVti+W|FG-ppOIx?z?bc;ZWn!X=?c72ih->udG$bw_2D8GlKI zm*nQE+!&0bX5I4?i(V4;nDo^V)nxku>4P3s_Hox-3}{MB0)0N~SvXna@+i|krI z%H#?ik(++;*R~%%`s#jt){Iu2kW*t=;qduVqG}_$G0>*-{5xSUGki|6*0t@YFhOue zpVQ-tVEOn}Y~hR6Sj$&!J+SVDg@l)u4&T5YAB@gZXBbpUT=GDdy;kAkqr#5YE3E=7 zQcZals}QsKu-mr^#$^0NIa@;Jo!?zVay#!xN5l{YW~P8n{uz zfCUDveuqET2jd1}f1c?)Be_gN;UJ0K8?9E-4!}lHQk_!GUnNsym4fOj`FER$|J~- zA#_M~zF!-whPspQr{e7`x-KXs*YD#PYD=?MgD#jgVyTRaI8_vLilEQ|C%yCZgH#r6 zK99LLHzp%z#o|`y_CWJ0?vj8=$5N)1M~u-l(8qscX>iIkDJN_zLel z;wm^XsJuri&)X?>yT7tsMjVRRn$U0H4^alK+VK`CxpLqHlF!7^opH(L?=ZZm1TdDf zs3=#UVo&3Tu$GO32v7mxQPbBuAV8W)_IlR&{Rz;nGOx0)Dh*`*sINRg;urVQAE_6! zKPBusG@pbNahB8i4)yb&vG&LqP;cD9Zs3u|U*|$45GWUW*zI|Mh!sY{*lFX}(0k10 zJ$MIb$K839(1rwD2slCQc^q3ErTq?{|Ap#Hq!{l}`V@A)o1v)XA^ANES@BI(2Go@9 z(Ks}xdVqc?tgpu5b;~CbYJPaBZ87`;=jTCEh`)f2Ht}*;-fOCX|LO?<<3@KxcT6>^ zYA4uZe(g(?>D>%`=BCwc+q zw!m9gfksQtmHkfYRV`wt@r!M#`4}%9OmGjJ1E;5ATp*MiGZ`+=?-zjrK9f?2DueWg zH~F(J^>k(gMX6(n{X}Dx*Ue5ZRBji&EY3d0c0>73ojP+_)fZROA*=EFZ>^&HB#g}S zgx*o%p;V2M`H}1}!0?f{Bhsktsp(45uCDy2`K)huZ=n-eqWnEDM|GT)3=-{#HOla* zk2RSfMvcDy#ntIp&k3IXKKz}9kBpE$$1lR@?+s`RXhdk8pOQF~(z}p>2w{1lB7a$k zbF~Vg2Dm$fI3wrwoy74CQRcKFP@@}C)2h7vJ-J$DQ33Mmm!RQK>>IMpoH@z>#BPye zCwqQ4iy)DOk*zN0zy)oV6joTod!)<_dG@2?C1i=x^YZj-q)A1;AlT?wAgqsOS$#W zOtFCvB1{~EpTia_Qls1Q_!oD~+5H~Y&sGD_sK?i`>vj#0V`N_mnZ+U%{zQo3;-pJo zLtGhr3;DL4?4zep5u{uvDh|Y0>+lbs6Chc_Cbp9-8`*06i=D_DM(B{abi-jz3$91k zF2}uIw~G~QzwJ@IQhybEQ@Z+HLkC1e2b4Cu;d^4=`qI?!BSB=H(;2M$4W0wbW)^=U zd0Rbn8DWsEnX1^5wGLs?BA;8Gu9Nz1UW?iB`e~tWj(5<9BFiP^wU$(2GOX3@Cj5yi zwMjw8abC>7(wq$4yT}Pf?q+Yjq}m>_Inl%DE2Cm;1Ri-Dm*Id6fWsL|I9nF`9iIV` zBR+aB>Kn#p1?}>qop{2BO*5F3TzJ}idYFXcMU`06!9x63u2UpyYvYcq znCNcdOkKV~A!Tl54g-^@+S-|yO{iWzTB~@)j`W2`6?UWIRqEA;WKUiK#w}1|rPC24 zK)6)CTY0GbqR9g=vH3)IZVpSgkkc~3<8;GQ*xNng@DqZJ+%BWN!>y&aoJJo4Ze;1K z8b(uUKWKLJK=1#6o8dAC1J<}Of!&$ON2IGqLgbWGtpsv>WDcMu+M=B3nOv(*?ORv& z7x&Ynjh@jv8pWGAwcq}BI0b(XjFt@mETT7hm|4t+Ekn$7D%jEBsrTGIU2tPCvn|$6 z->xnfJegf-VP~DeE;ArWxH+5bi#K}HC!M5(4Lh^PiZ9m$hm_y z^zhMACaK*zhO*;tvkz$gD~4f5%=piKHAIML-siB<97OPEoX;gxMF$M!J;D6!AFYT8 zevRl2ks>XnTvMFwlO%aoDavbH3kgT`Dc~CoS#Gikup**3c+!!G{==@9TKbFVXE)2a z5~__QPRvbs2Tk8RhYJU^iMRJZXwPnk9G}&}UazVVO;eht4C&oG>+v49GQ=tK+Y;AZ@196e!iClTBMqB(w|FVN^cA zH^5-DlbBDelOI?rSrqP)bwrP3f^upVZ#RHOr6IJ!;# zo_~OXs_AmfB6h+*lOl~`4D;Hcvp|t z7NkWS&SE~tH({0=KvHA?qxrB!@ZbgcV?)o3<`@=K@Z@U{`Cow)a2c=@_p6QmP8m>z z1LiDd_6{6?iN3Ojj-s?EpTIqeHqrp9^n-DlUrHTI0sZz1=CcppGB0XUk8fFsBpvTs z#>2%1llO$@57)X|m1*;ijJp5u#g>%tAEdM36T0UW7K{gwn9y zM^2u&GF7Nh?oBM7OUHQ+B1+l&@&wwiniJpVeEq4CQ?fJPdEP>4PWh(WgNTYg#s*1z zCgw>s!o@89oUO0MUr|^*fPg{WgtfnAs-fo9|b2m zNBnI^e-rcQDOy*xs)h5L{ky8y!_qmRGCTGnaGJfiXU?zp#3}%{z8 zQH0l;nu)BWRR5-J-2o~H>ekR1jBqYhJ+;}ttEKv6%>}3p`SiZ;M8zWoRuF=bZS|R6 z?V6O1zqeaJ-_;Z$%AhSPx;HCRd-@B}16l5rA|qE|r{=~L^%f^>asp(rxo(YNN_@z2 ztpTz7KJwjvdZl9nzgXH)e!bu`!`Ium#=i ze6q3}IoCTm8e(#3@z!V&zp)9dG}>I}so_A?0<{FBg0ZJ=D6QI5a|LWFmN+1$4o4 zJqIV+Cu z07C<{%JV+$I3sfk={E(jL8949y)oXD)u zxe3%!D;(V-#Q))h917EG41DLw^_BW2L=X&KnZtuOb;2CrkU(~4Om{(V})C7J_a_om_lbjTG zXSyD?+Q3I_CKR;Cm20Z0kiajfI)L~F|-B7W#Y>&?>V6)dvV{Kld)t%KofDyg12-b>csgepq< zq$ddTbSYlQ$;Y*2u8EE;EUOxn*7Bwmg13{QE$Q22oyH30LTwD7u})AwT|=_}Hp%p% zkPLV=LCNJtV}S2_*F!2#LkWsULUB0*%=_X-ySgQ=>KicpEbSm5(X7ASwLpCq@iOsv zY^!Y}zVU}q@I}*}y@e0E#K#8+6-V2!u zQH1Ai7at-~etdsS?n%3O;lUfsg6vp}@dq$et>f+DuCXKbzE;Gh-AyfEOqKmF=+f=kxFOr6YGVzFl99z#zB_#nj^Ok02@cE7Gvy9PwXu1F z+VULUePP2}Y^SCqa7G_bI~qtZ#Iflx2k*?~VrST%e` zWZ~WQ_{v^B2n+YzjpMzq>X$44LKF+mI8XR$5hPO1OO%g)RAhNX(B#AmTxJ~bj z@6<`*!pz3vsYt8m!m^A^%A*z%r8Eq(x9MDEiq%Yk_rs#PYn57U=Y~=KMAR67-)LX>mVK}E#$ z^;FBUaGC*hjr*W!ow`_DVm2jP7Fb0p7^Vw-idRZ0#l*2`t)9+E&@u-BqomngiZB;^ zx`9xEYFydgM}fp+M}B;gK?Ob3)LtUCx!O0`5_W$|l!W54%!5a7=?o><*CmXa5MFCN zSnP@6^Bm0c=A~LxQlEX?h7?N3@RM9vn_&Y3%Hr}wse!08AeIOIPo zgBwsCA|^h3Gn>u=k&ZVGgvM|{_%)-}8jTfpuPw!S-H2UjMlm& z-mopC79E^*^GI)67V?8;|Kce>^`bk_lzMpd>||?f(d(B%;&mk`a#;)AJQ-i(VQP7Bs((bD$^tjgUh%k}y0tc+G3c?{-FJ!F3Jv_fZ-t!skQfWajE z(5$codiu5eKCnj`XM%Ybcln@}pk&nTNxd7J zjl3vr-X&u1yl`rqri}N_(9grzH--4CX={m&DvEzfP9;+%UVFahH10rrJv6keUM??LE%r6cJqcuvG@PQy87v4<5_Q}_9M6rU9SqaTo4HyD7DK-T z0KeL@@X?-p5U1oZ@@Ud{5A?L7$N;$tiC;a+JAR- zP`Kz|dgJiX_cXBc;27#4cRmUGl8oD(=K!Xf^j02T+)W;O)i;e>WAuHP&yZ$nHZWN- zM$W(nnih-ftzhnWhfO1m=8$>Y4JDcZep=bxyP%6j<~7>cs=J#807gSsrz-~e^69d= zHR&1dq0-KA)@8HkFD{ujR{DY}=f(+nTKYRL3S%|*V;{dA@XzdmHPN@@l@Fi`Cc!75 zpSzp1s&24G0FGdSk2U!%qSFdQp@A-lw`g2*@aQccQ@7OxJrfSR*-s-Xl73x zjL``LS{fEWZSny;*Og)E{fek=Tmj?)FhstPNV%u-ae=yh_i`}J8lDJ1jZ|I_v0k4P zy?h!Qrz(OL%ex9_pIH6EAAE_@@Lbxr(9 z38>$*N-w{AsSors>t;-z*wB1ih%#p)sc&;(eAm%^uv8f7E~H`nPzYhP^o6BiM-$%Z z;KKR9k~L7{@Awf7id{KscDGyP;1oku-42`QuY}ldt*+|kJ*f6OwqE=~YGT0hlf#3~tk=&0r(V-bL^wso zzEaFnTPfqY4=gypcy>pr#QG(@VgipC+M@93vKOnisN&1Gv@>&O?1npcdEQa8XIT&D z$2zJC#EmNrcP=zLskkerb%Q3y-o2@ITo+#OfR+q#na5J*zolyS2vR;QUJz@0Rx7+9 zpSqh6i9pJf*LF6#%xtXX!a=IBG+T9-)m>x#jL2)j7BP;~H!b0Jre5~q!QA+=m44|E zc1Eem^CQRfg^K}xG%XsGf-_SuydP!sej`{jF=sd*lxN0~5d_8&+ivsk>9M-R@?A@{ zc6fEsZ+WNW16F^>8V+%++q*pnzq^1euXl5heW%TlnhJd9?eDVSU(hp_|Iqcq0Md90 z!rItBpEv61xMOK4^s;zMsy3l|uEe{G@&$k<@8nqKD|pF!R6lO$FN5t2CUVHLmchxM}SeUW)SSdn-B+gX*&8i}WH-#BD60ljWCpl|&-Xnowx%L`KzOPfT zTKgKQX3+i(eL2}SP01T3F?0bL?4x|O2BAxbb~^Ec%%RT$ueK{m3vl7PNQmxNP<8mr zpnMmpuKDXMfJ1cj9Oo>Q|XMPE3~eib^L4k z``^)0RTAl^uXO*xJNwHxc*SE*3!s;D9-U`RzBN|>Ek1k@PyYiV zOkGj+g;+iyls`>BV90VP|`K4?AchVa# zLI^!zWB((a)46F%_w`D_b)o;YNC$5knrL7A;938_-md$GHnPdri2L;aS@pk7R7!!y zRCYc;C@?ai?R+xQ>4iRpvK*hgfl^*at9fRX!Q z{R3C3lFaNQoc}+2{=ZZ7|7-S4SC>5T>L4fS&eOrx_a3D6XnplG*O5n@%L$432k{E|g%a;YsXM z2q!m?|C0jJdO%YvA}Lxi^nXjI;F&OQ*xf?j$<~NFkRj7VKl`;|&|XJsAcz7~8z^+X zt1P^QAQAHb>+KTK%SKEWyM*%iDs1gfU2V6Y)C;;^kUGowdrGwA2hQW2d8Y?dd22=8 zoz?YDG7hz?dN37288mzXgA`yI5ZrO)+G0I}pffaS&*qR=)?J9Rj~R1;~? z8}uRVj|u*3>B-@={q?-n#&_{=1+Qup_87T4&n5Aa=F~*MDz&7G7x|rz47)7oE45kw zM}ivC2%4K}8pHB8LF-M&q*z^4JfCWsRXA_c>YN;)h|P{Q>z-$O=+sN&f7Z_ZL+^D> z(QoMM#;G*%3GPotDC`7GU0J^AcF)8ym$=D=Y+5}#VSR$v*dy={AMUUBG8w0 zzJK}6V22J@cYjCyjk^D3%9?0mVE%~^@pyj@cSXCtgtt(e=~?u^_p8~qJod^fMF*yo z>#faRQrAxbvudZ?s_*%EZ04r=^87#|Ecum6Ab#QV7`Q9qPqg$;O?&0s;r<%2RL^hs z_cbr`6}Rd8I%Ww~j5#vpeF{OF!z$0{DMcGT5n-sBku=R{V*9+L5rwdVnD z{MR$mr;cv0=(eWZVAwa$Z;MpX{TK8Z=viER7AeyKooL&o@j#0B>c$0tr^kb0$F~|3 z&e;*=y&%jVsmZ^r^9u8Iu*XU&@4fwa1N*yB7=-uh9C`UM5S9N8O!~IWunxrr&6l+;>dl2c>Ja)`~=_!yrOKGoGpK)EaT}!?F zW@bZ~R{3-mMm$65bu7Aud*S6gIh*)G{Enz0<9wS=;3x`+uINq-*NCY@bg2v`@dv(d z5y3m52h?17ehm!8k+*7ROkjjv9w1_N>r1|OZ(&*uK{rEVP>O7iZ5%+QdVz&|9u&RQ z2CK)+?wou{R2!}E2AfQ8zZdg|Nll!m^i%p;&ZT-4vX>Au8tV!fdGk}KwNcyODPsOw zqFt24=;~Zq=%H^}5dt21ut!*CT8NelP0;EkljT6P!&3+*3k1^z`{zMfO{ClKy%__X zeR9wqwv(9W6cXd_ydHZHAw~a2P`>2(vww+_YWm5eX0v-oP`}Pp)Jw_L2uocD6op)8 zD>DsUukfQOrM6Gi{Jnk6{DAddJ;HRAD!Ak9d~2xq>>KUCK&mn@t7vt)n;yj{KNol< zOs0Iyca?V1uooLqT#r-1{kX8V_jVNzzyx zW^bi4&S#tCT{{%b9asJe6+-a3D7WIy;t2!*Kz!tYt&JA#ZO0D`arR`2je)BU(&2hi zRm#Tp)>RevxaYO#{>2~L53%+|{4ncT$V6~YExAp{F7>=1*3+I!3$Uxaj`F|pG!)X_KZ>vQ00iLLcLU5VPGyWPbB&7HPZS9V_<6dk#Qttd4+)i^&C*XqofbRqHhtP~MQf zsyDyt{~1uuT#CycD2K1yef!prINzB%2%I`%HkeeaCm8@lL_|pV0Cckq8{GYVNdw8) z%GwUwd>5V*_#2W5IR##WBeu|w#=MHLv!~-UW~f%!46QFi1MOKmJGVo}WD376NMvJ; zkde~lHtDXAI+VSYpGA>O0!!qa9C5nsND_ z5QcPf(%7F>ASS0rsX#J&4xRP1ZgTkJaZU5Vv^hgo0T$|7*7&)GfUg$U0{4PG3){1G zS3kb(!Wo9e#W{7FIpsNwOU zGx?7ZiN;$<8T*;s2gJoX^_Gx572w|To-&M+HNQ4MQ#0llU23e;z(-K^KMT-ZH6dt8 zKV41jh4HqE)b_e(-CO*<>Qjp>)-jm0e*TxZPMDdmM=b;KPBkqcoH+3k?t!;FyK97y zA#g!nO|I(?r&e}WX0=mvSSU(Tmf2rX?lr=kc9Gpkr|AuX)Em?i2hgstbthRX#`Z;_ zdmwhR8^Fdnm!@<#`{m+98=K{eHnKeCzA#$1U=;Vuzb~zi^03QkRug@?nV^-myB5?D zvpLdM&I8Q1`Qj{dGYmlcZHzDa1B8`9-f8Yx-oLdYO+JTd-aZ~SaO&@S>EG#{k-GWy zUf^p!FXrXPkPsK#g50Im?-Sd1zF3f5^gBlszu_Ap0-UH0GeTQ}QejaJz9Z`aVh&%% zaPRc6kcKt3o*-RJ%_}6B`mpv*trd>y(6IIc*oE>?={Y+hA$Xe< zR4p#TSfE!VHXNbjz|Qmg(Q)*Zz^=CGOWvq zQn$QQ8n$vG(~U0k*|~wxfw)VZj^JU7K<6z}Xa~gx_NhHHx_5TK6}!T5p;9ZR^J=<^ zr0Q)@vhZ58lcI_+zQ{;-wV&^CG2IUz=l6SojiW^aySibu!aj2bU)0R2 zjo^mQHw&wHyy_6ORhIHKZehAn{m5omqa8RY;yUtwO_{qnJ^fcMWF_#A`VD0lm^FcR5z7%23 zp{I9q-8aWyqo=}V-8a{u?tX^YS!J*9J9X+XbAJz#<093Y|03?dp8VdbSKDd65#djC z_H~~YP|(4o$}Imz=3&|A%$)N zMWkvMk>BZSbM2!t1{@Sk{_C>(+{u9;;#Z5 z&n3#&w_Ze0u*U&90Yn;TGH*kc8HM9WPcRbI+B)E!%h_=#*F389Cq5195Q*=E_H{#4 z!N{N+VC5Cf=MqBVYb1TbB{1b_Uf?Fg>9jIp2PrZNVhE3)gYs(L45{g)1tQ%6!+}}m z5Q=HI;wAJ|zk zWY{*X=b=#lIs*<4&2QJG8j>>R z8C6ODK&FqRE`OOQFyc$Sn>nXggdQsY@Euvtc^oOF(6OW>ka(HiBBbx?WJvl`_r|Wa zd-H8K9|g^3V9+DEm~e`_;)#H1=`Tu2TL}_$F^%&cN~* z;rK`AHgx#C7IazWEr#OF;%A3q9GG3kt>%7L{Tq$%w9TRvOIpA+MJvR56|_Ek#U_IGL`t8_(kG{Sk3I#W zRy1ji1REpe8-P^6Oj{7m!j}R0aPTh|k%D}brBYMtkdewal5fBk_I?AU=)`DwOc)4S z@z^HI`h}uuc60UtT7;@PGN0M2TYQL#f4r~W&LoBIoUOh<1OFCTnux1)D#Z2V?Pi`w&I6lJnM|!2kRec zoyK{L+IJbdbP4v;4dUeih(~4D_v*Z;27QUMmLV~blY>JxDarj1r{UkO54X|M&wkb=o^FUcGht!>r30@*zE(s z)kAcbg=n6%mGGPK{k-&)(35X8SMMt#Dn%rFx*a^)w3W9&Eg!n6$_+xVywVm1^?8c7 zXHnpg%NY1edHRw^8xWKH^u0WN7DCUtPbaw&m-9al$4Lg=SQ_WgM((Y((yhq$+bGAY=e_Y1}> zzxXj>P`>UC(r% zJ^Is(8RkT(3 ze!zpGOrPl0rnS;ED8_$CCKqji{VW;mGT#ZkeXa~II;Ix{BCKoIKAng)gY0=sG=H4` zQvPstv{Ssp^Hb2ebJ#(zSDbFKBBpx1S!UlLGM&k-50g4Ds+CEKTmX9mg!t|dPEBpE*0|F)# zKYr=@9<+g=rc=X=5z?fouX~9Ym$3Pz^<%>D5ci=vNrJ6{AKUX3nr*ewq#e~*mr@3^6sm!ZE(&OcAbHK<}`Znc@!7VbQ+ z0?<_Yb3ak>puwTBC4QoUF%i3f>9?!!bifWM=wNWS)<5HO^dpJ}3e^^6pbR5i`OSY6 zTdHC>XfXsNA>raaw9OIzXS>DiBVsIW;hG_)u0HBJpto~_TZ~p=1}HJS*AEgE4hI%- zk8^(L3jvGQWt(dtsv~{H!P${lIoV~Vz4I6bxzZ+D%i5R&(rFDCgQJCqz8TBH>MJ0$ z-;+B5BdWaLqU0SioXZW!3F_^agJ1QcTKwpD<{7zd0CM#}z+>^KZ@vpjU zsyu&{5nQ3b@RQ@HjFOK)%E|QD5fhKyR=ByDEh&(7oj{CcdR_5?Pc9*sF?D|-v^>8a zEiESozuD}&AI|pREIxGUKGCe(s}X0B@6s;uMnn;CEj|2G(RBno^pfFnUtzrY($fft zO8}?_=E6;AwP1}wEL|Js&bz~WJD;vV!z@FV?JM%!kb)&!-(hmRRu5g5_*T*f7@XGi zQARpFvqsahr)NrN#fR2#2^dL2Tp;;_lAF)M&8ifw5@%o{E7xE^OCp+c>m^iV)m@w1 z?sE_fIt0@4t<5U5eziE$yUr7p69SW9c3&EJ6cRm*y4kK+XMpQ=Hs_A6$ zk^@8IOPJjvZd2X~<+9%FQ?mS=vHKuFQ1=2Y3UJxuGq=#+YDIOIPNsG$BD<1u^FTic zeP4q)*3FIb&8^`^N*Mex6m%Js!I&!t4xcfzb(A=M?TPdC7Z(Db+||($?}b~Z0WCki z7D>zXA+FNPX>f=%Y#YS(Ez~V`$`}Tiv_oLCR)Mb4WP+9nK`=2m)ndMFm=$G_nYI?k z=%=j@B$NtNpKrcBfuU}!!t5vI+r7nWmk+i-vR`2cGYElRiFB7_IItnNxlC#v5vTZyLRigDNLyA(N`$&ht?&6{ z!>QwxSEi00^nnom|uV6eHLdmok*~D_u97%~3_|S?1Jw>-hzFYuS_` zyTGB%R&DSn+4vo4)(U(P$Q~V3LCFY;eBO7JAI(4I&kAy0pA;(gtZTVbL|rXWX_|h$ zrzphM8PnF#F4JCuBTXT=-rb}Ith%0Vs3u!b+Pn`~N>!)J-P;{dlEnx-Q-8 z@-2jpFh#`)_>az%XqVkq!EqSH7ckbp52>Rl^+Hi-!x)g z^2u9)jO_$m32*6|wCl2+`pMiCG;E|OTo~$38`O)n>CX7jWGOsRHtRTEUBCyg= zUR%A{Bv-j4N{sfxZ!DOEtod8qBfpWO??(Wk%6=pFfCZ)3^ZKhA8VVwxI(6;P`e5+c z=fQb)@sr}?Fc921FQ_-@XBLpwPAvBGk?ag|XT{U~7+*J4f4)w#4KYYs5Ox8b?NYaK z21>Y}RZF% z*m<{T%`&TdsV}0~c~h9&iKdc{k7tx)jE!YeR2qudioIRa9=?lL&@R8W?uw(lG#N;R znL=`c7UbW8+txeTu(`;Ep_YEty022i;vC>d(4>bSQHl0nLRGC(JU4{$;n3=NP@T zDVmLFStpVdtek%Scws*jVzjhAUE~=w+pm49Zy_<3Ry3IxQH9|vG;AX5#XP{svHh`d z8@j>03+#i1vj=KW!kSMOJ6;Vh|0e9`fk|2ni+waoa8U34tNkWf*SGpU)^DBk(ZcMl zVp% zuR#iXL&y0HMeqXB&RyBHUw9OWwjqvGD#E|-un}e%o*wRrj^T)IRpxYkSFWb#0oB`b z(XTGV(<&ds?LbE$&_p8}+|?lhf%u`Gc6tjAEyvTpRfoW2qaGWS-0j1hiZMTM+-SOl*x ze*Dfi5@##FjF?8=^NgmJlwLltd?qOrEgw6XcI4i^zehVVp)yiW$LS;OK!gg_OL8@A zpT;j=P1@bglHhcn?h&o5r8ayl2XWATN(k2C9#wU`RJEvNuIVtKRKX;^Gy;et&nOoY z(f+iWhZ{PI<nLIlUKx~z zV->7j_b-OS5(&O2A=dRz*lNIMS-$}Sub9PQ>aaKX2U50tJ6(uHv)FhWoDM?UF` zGIl()5r55!r#S6gmLsjTsKitdj8U@((HfbzanFbM zX03?omflX@o-88z^cAfuwaBYflQ`VSXJQ?q?^Uay7Ms2cbl&@uhF5wSwmMe0<<^`l zl`0!ZLrXzGBRS`=fHUn`=Xb}>8bP>g{vYn%JRZvS?H?~f$ z+4RGk@4d$7d`%)OrRGR%d^#+^u26nIB7d`^*E)n?)!F+x5B_w+QRqF#-wLE+65{&_Xw4nh(}#M_I|ZX_IE(S6wa0eZLPTwHVg7FKM4xdN#_mp1 z++b0hMv9wok(*rG?X>6MBG5h~bH`c26-j(2uVmeUZlxT(S^ZO7swH`ie+iX*YX1}} zmF*q_EmEI@)(bo5O^<$82~B7{WGX{pj{THzCHp-bswn?_2!Vse?#Dnr^a&)RI#3>H z=F=BT6E0~!arA?AbxOtdX(Q&4-AZ;P#O)p)Sm6tYO&3gc$-j6$6M~1vQYyf(KZ)m^E=#udRm5=6ZE9;W;{#0WjL6C&;E4eooVcgxhALGO$=AT=^V>%-Q4%q zj@+9j#y7@MP!~z%)(g1S=HaSOQwdHFKNWfRo^hP-uOyfF1r;co3^dw2$RQW)Uo6=4y1Sm( zfg`_0y2u-sNI~Yacn}2#yG6JQWd1<8F|Wo#+m*sjQf!QWN-PhZd!_P-QpYTDa_t8l zYBc+bEum#%=*GTsPmCb_SbE8KZ7h7ksuZ$%G1CW^ zl5j#iUD&7;FgfILjjLPy_&nv>!M+~-CGzaKZcLOwEa4s=<6dhs<+N&DQ~))aS*w*8 zUpDs+4q`AAfokw-cl0?Ba=q!VutKdwwf#MzLS40br%7K>Djr11A}Fc?RiaNlNc`H& zAmG#8SrjiT+nCAe*^Ym^CA_jOfZALOdROD-c8__$yST-Ht3E+=J6li87atC05*x2<^j6}Ittm(jzm2chATm;`Zal>a5b9i{u;{S zM%%;_R1MpQ*R@Jak+@XTaN7fI+e$^mM%d~`wf}aTS+<0*)$tlrJ`#tnJUrlOyOw9m zHbw1U6=OH8Vyt#;eE!27Tsh`B)?4<}RxIg-IE5E~bp$@R@Q*SoFso5h{xzM#(%2v7 zSDk1YrxmB(H0i~BdC3H`!M-ycnTqMXQ3H9fD5}wQ$IGFbA&&2=ZxK`L=TLI|%cyh+ zro%S>S=jgGu|F`W1J;h0(M9ivKD1m=ui(azsr3@zbeibr$Lv(Qgtm&8Ok|7E>%NZ2 zXXzC&&WFsIchRC;O^)vA%BO?+(Y_G{wLV?%of{of*p$7C9u|ZQ@|_#rLA@W%jI7;0 zJm{kw^Db>fu(*@+-i}^w0I6>zN}D+MhcQ`=72!NuS}EOwS=VUl^X3a0d1E0VlbDHp z!c^XDxpZv36V<~vuJg0)+6nE(;PG4#w>kEI%G6zfbbC+q5VUKl`FdJx>bvfw@9zFn zbF~kDB7S4%d*)Cduh!-S=a5mux0KO8^pnu{qxSe~x+JbC;)(_#I~TLjAA0;@uhwLe z;ytGQJqq_95OfN4-kjKE+~_wlM;9z*4U>J^wCWuN$>&cGK5@`vk~|Y|`Re#Ci22#3wN3^j{^|E34lb07H9LN0CdsBoms0LpD*4`> z!0o>JuEM83OQKp zu?i$etz&sYL2UCJWn zs0|;S(c~G$&TdV|sMZ$dglCFTQVQ!a$Wz>^C6wF#IatHOF9`?t4{X0nVMCw2bl`_E z?cQ_K05ApJI=T6NNGYCQ2Xe4$3;)Q$f{q(!ul$gMr3DK{zziH3ma4;>8vaY=Mf$p= ziUP+1E3&qGuG`frdoSHH!%9vEygA_GIfJ^7H8Ch%o*2FQR%zgaJ54$E{ZHlC{K51Y zypnv)hDO;qqOkTdBAnE<{<$iYobq+uWY0p%u~^}X5l7bczR%n7Bd?b~EYH}w<;WWO z(0@pw)1v2Xc!(AFIpLfT_;R{7cE#bRtrVa2h}uP^smDsSbSZ&LVjR8lJND+pTZT`P zHSrsrC;A5RM-6m0hxK}VU=oQNWIYQA*`18fV51L@y<0TQ-s|Ty|YvrZpYParc z(-?`;!YjOm!;*`$4(9$6t?YLZzub)`#uTf_$;1%VLUM}ELNyi)jV4LKYtUhdfYsH-iTV5_FY_91Sq>qJUP zr*~x>XvrmO93Q*!^4_KC3-(?bu6QQf@5_VnmdgubVjP3QL)*w@WwA+&{*+8CygLg0 zGZQm47nID>y%MbteZ%^9rcI~{-}-%06uzf=4b zg}YJPQQzobS#lH49!m)!ID1TrWxC#NGQ^BFk+Xu&2m2CC3I2)4%0zlvY?<4Z?J1CJ z3n@0c*ToK8w-m2>cD-#$vAoEraNhxyd6NOJmuUlob>Vs8NFCoIFpzPzkUHedZRw)n zP8Gpj><7&)s6y?KJQyEH+}96ZqzTsc;x!qMN`2z)(#;V1bkgIiO&3I2gb!t28S+4b zH|b_oD8(mxl{7i1kDGeJQt zOZ6RD=FfKDxEg=ZJE`9Q?{eeu(_<-xkeL${>L6w`3h$VEC~R1Ur))KtB>(52dIA}1M< z9-n5cQj0GTu5SL&VJZTBJ+jwIM!DZsRe*W|(L+4F;i{TM#2BsXAR&PU#O=9hwoSwO z0Cmn6ya?HIcmoaS_dVegLe8C`mJ?N7-c7rIxTVxo+7O;M2}wUJR+3>i#ge*W#BFa5 zHwGd$98JXb`e6#=#bcaLi^6P%{UoPTAd;Vo;Y!o?D8uvD65W%kN@NlBqVjiOSWbo=D+*%*|z7A1mp2`X!mn!Rf;X?2(()Py4?+k2WFPU*yX@@FO`> zMD=_cf6xUUskqXHoh=ybYWl&`JQV*(&jN~_RpR{F1N}pmFW)S?$!SX4WHfa{~=im zFIU9yUIl9&V0E#k2g50 zm3S^m4(^bq-ud7!V^wjbsgz*0iR1rPtG~ED_En~s38N}$^7PMA{9{LweCt&f#W&04cRZc@@(q`J3;m@gZ$JBk zHq&0o8-a>GYoED2hECOPlM5R@^qb(H9^jfl<6%)G|Mkjcq?6yRg6g$Flge-Ky%YQw z;j?!)=A8l(iorjh@JzV#lnHF$z;U|v z4OdC`#K3$v-smU+?SoQfNO^AnG&jEDfb)K{{rhTCb7;cF&DD#S{{EkT zU1@w)T_8QvL>{6bQV*f8>i!5_GumrlE{rfJ40Ggl`-jy3_9ZgvI;S5 z8)8n0_a)piKhn7}t0c6>G;<^BY%>%yd={If8KoVtiCu2wa!P!!Ic$DR)J%v)Ve!!r z9A^sWo%(B~;zLyo4dedcmMa}N;dGw&9QJJN@_^;sFU{voYMtWxP}IYoqI(+tOE1ko z2W(FVNbiLaYn@x(oL_FaQ26=rzN53s@DHT*VULtg<4Z?*#@YW~f6)EOsS-v@eDM6? zV}E*O_kpfciIU%zAR{_|7;V>e5e3EzfHG+DzB1$$u_i^6RyDYC|g~2qRWd{Tryfh&SAy#!<7bRzDufx z7?`QW328sKLbK{4Y@g|w4z^Vg2TE)OvU1Vn0OV#%RyrDYfJ6LRg&QG~OfGl4kF@{? zvlELSj;jsS2Gp+3bn%HzEsZzbs`aP35X4t;Z7Is;$Apc}7^59;B;qq5qcwh$`6V5Z z%gf7O);ZLkJvrf_sTrz~B&P>F3txgXG}5vu>_&=`pVg=5heOs)s4d?whKasXa?CHR zUW<1eu8>-?9xo;OFOJoRh$6cPjS{3Dn(Lw5ZO6Xg4r zFh~6Rp`caW&z&U}-<`A$B~;)S21*2$Mg=KV@7K-V;QWTEj2i<=ALn+S^#@>~uhur4 zc0y-Isw-FL1|*i?%gq7;{w&>2Z}Csp35TX~W_~E1{=H)|>O5*G!>^j)^C%qSeppme zl3VwUW!{}7-sOmJ-!mJxgGmxLZO#Xs{6@7kP+MccEH4;R;_}>uUzg7nw@$R0Z@ZlS z1NwfPmF3=EN4<}^gJhh!s5y1k;Hb9Yq8H5zf-TbYi8odk8qmmLduELq=24Yij|DcI zGJzl6MuSb0UGKW~h`~r^$T_&5wfC5VwZk!<>=o+Pw6Z8>$mQ0zNpOXvBNUVO_b<0B zIHsVVE;T8wi4|E2yWvrgSYIm9JhqlP?^NtQKUi8x>dB3&npPx5;GJqs8^gF0+~s!} zesunO7J$zOlQw~YJOm<2S^7H+Z#ZL)F??;e+dk`9Lz%?96CMo>A+{7K;yv85=DB#;3fKS7##t1PsBq;_XL zU`N2_Z618)%Vk=`5N_0JT^jI@DPC2UB|S0)U0rqZ_Xk{{VV#F zI@7kD!ni%Nb{6_qa{E5Q;l;SYd`@@1-gj4@?O?Lp^R60cW-;F!@uOw6?WKzqmTVs}+ zwf*nZ>`+PB!&5~~JW+^(3c<-0TG&3adx}YEaktk*g!&y=)NH-B->hzk-_*D@wPHJt zlAE1;^Kq!Y?98B}W6DOG?ZgH1jTfb`VY$iHwgz7psQ!Gky@pPx{IDl75*yYmSPkX^ zcO6?cLgnh|XFCjpiS`<&h7__J4Qa(*IM@_N>e7~9&S-vnn;4iXDK7>MY7nTp?Gv~(p)%0)N=*_=Lu3G7c zdcw?}|*a8+-JVM9Qz3#(@J#p(kylLnh-y2_S1C#pJ`;8 zck24G+F6H^{Wl4+Oc+L+=*UjRb(6H^ro4dJfMF2zI833A%HQi*ta@^E0)vf-{dBs_ zx1kloZkMk#?(XsG4l+huao-1H-`soS(bxP(vUUys@SkMRd3VGe`Av4YWH2s_vfXBSuraPGJ9Ez zr5elT2TPnAq{^gN5EUh}D`X;$JyM`ZR9=~uR2`L(+j9Vh3?8oZXqBm13cYsGQh9pN z@urKVcDV;cr$-&2DJ*6U^J(8oOwF9^on0H8#cf- z=zGs_pD!|Kc$+dk7-Y6yK+PJ20UsaINi_jCjfEX5`}X3(#Kv@rHtOYd9=@v3$I)89 zu0c|39wB8Dat+I3oJt6u!hdLx;T^Km8S^9C_naQ&>K$g-;B(76Xxkni{niH8=hL29 zy)l&#B3P<%*`vZ=ICXoBN%<{&|A<>*EwP|dT#G+(0Zt3ETjat3&2_iNCgh7RL0yj6 zdp2`tK1QLF-whvk-l4AB_6t&UU8g(P65>dV?d&ni@IoU$uPU}cv-J#UBY3*D0Q-lsaD!YGv(D6)QZDu)gPPORwguM?|@3yZUeLYukM!)^!;b5;P!sAcM%uz^7F=;&=K z#TGB5)9ird#q)*-OEU+|vow@T8e_loQQAtAvE&}Y*+!6tE(KXudETdsIo$gPO1fklDz)d{*Xkdd+{QAiRhw0Xe3C{aL1UZjN6pgJ zGc~0WwHhaH=m|^nbtP`*H(isnxEJ2rQ)H=L;-1M#xLfQX4N||7tc4KHz8wQ~KNud} zUUBh*Nzy?atS36G%?IQ+{ECXEl6KXO1L8+jjH|s-L(U%$plyA@=2&3}RSV^`PTp5- zGrR?1F_ubO|76~|lb4R9*R9&*_SEvzPs%y~>&c0Dbwc)ze*V2vn}#Drmd|GF^th$+ z>bpDz=ASVp%tscCE7*NG=~`G?SW%F}NUiz)joH2{?aA;1aqm^TuCB3&v)qJi-58?c zaH)Nip{=g^Dv`Cu{f0817B;bo;&en6eC;Txr`HNf?> z)XPR!-g%NvcIO-avv6*_F|52n2$xzSI1|7GM&N@Jx4}VFF(M)N*XL9nY~d#R5E{!r zCDuP99f5YtcDouxqD9R|T+XDJQthUc*N0rueD)xJ=SvCLE|2pTTTGO*xI^zN*Pa(P zcS7FX@&3bw?XmjcOoKwRG``y(HRXo0871kX7|W#II)mCX4z5Pu8njNX;tkj;nmH#K z(4891C{;S@zFW|gcsHA|&*uoNyY79M`~DO4c~4C%+{BE>8$%ml^A8T>6XHLLRxMsB z%SPoF?k+Mg_vPv@(-O5NMCTz65N!;6dYnBK2r#7Xhp2ijyc8B9QZ@|ja) zOlLE^QVW8by?&Bg$0}CxUMiLqve+U>kdhBQSSqKs(s6105B&Xjt!B!%m!gB*4$JV( z8%YPnFFF&IRkG*d`(Haa-%+1^njl%1t*gz*2n)u0e*gMR)A_=s(H*m3n}~d_S!Ja~ zJ@v=4Mj84y)U6nqV8KMirB6p)HVN5Gwh8N-;p+gu!3bTjvw5&e0Vz9%8;)3a1=aSE z<7EeG8eKbNo3=imx}>a6fOA?qt+s>Z0#EiIA8PSw4y64&s1YG%W) zt*T3Q?FXbJN*f};l}`BMKzOe!uNLpz?4Vy(Y0WT8!4{M-%49m;=WxLoonLFzX0~sn z@m0*zUoUW-=oo8RubdX-@h1Lx0}@uWZ|p7+S?=`V*Zzh)@; zZ9c~;AHqOz9YwO-hKNAx%8bhyV{m$j>?Q)Oqxs#cq=mE?dYC%`IhIP~S)8!w2KEq-9m zgcJL=F=BqbgpY9Q4MfH$zt(ztfCl(%hBtY38{=h0ZaOQc=zS^Iyw?8|LO!$##VpwX zJVzST-5pH>A#Cm<*^2#1Gj3pdo`1RC!)snen@TIQqlwR7`!Y4mmWP)r2Q~p1PG}LD zO#S-t*~^Ys?*d%&BRdMj{I@(?+hJ>a5$>nk`8>IDPc;3&jeo)&9zBBWHa&{9Z&x?P zqAo`2ez}>g(L*p8K4ThOn=O-87>tWRP>aeqr&ouH0re z6B=^z?t8`WmkuPG54jQ8_ur{)ONJ7c{Sw5|eU% zm#zz3%O*Z#4dU%x0gYle_bq}NItMb+!)L9Mr27R4UT#)tcveAA#Qdeud=!_?aZmqj zLMT14d;o{i9!|uL#%QS`v6{x2c6*$X`qE8H?IH}Emf&vg`qS zRj<96jAQ@FP%fFUS=p;nHteO{<8E${OU=18Sg-t!udEYGIFJ2q_)xkvUPK{GO@Oh_ zW6-0*9WkERryZ)c*mu4^9sBjEJbyv2)Qoq9oW&5qC_l`#<-#j}`A}(uX(d|MV?BBtQ#G0R$n>RxH2YXT$_X?g6d=PDHrUJOgh-POR?aO z!Sg+`^K^|GgW(Db?jdr zgcQc{#!x|_alnVV`CMyO{`nX*!{x-kXWahdDjw{69RKutFpJV&gd(D4@M*z-9ZJ}w z?6%vyX+UxOejFI2p~YuRPuy5fnzgmI-axQ58)TnPM2hs=w;NV=4d~DJPZC&%mafZL zD8%Pp+?}+o>TUIrZ4iIO-DYm@TW31IjpdeeyKd$+$yUD7A>UxnX%j#Vsu72} zTxO8v`GGF~{K{mXiCdRy7}>489PH+J<^I?)jxG$uyCLpL+v{;X(@lx7)Lbs@lm4hFO{kXDb?A$6lRy zr~&5-U% zsJ6%bCOO&nu1Qc?1^^AK1TwhnAEG<(=|(C5Uea{q2a&m+t1~(r(vEKx0Q`NJZ1?R` z$T!L2TmbBN4fgJOU2r)rA|gUybm#2Q7;+eH?*q$@T}_eiGW&SI{Sre(QcwuYG_gt^ zK6@s`cR2)~_06d8;wF-)foHf1*EHugopaw)fb|PKma#pSb~9QKO|)gt9D~vxKi=V_ z56E7pm}G2I{Lu3rR8x;(e$GV1?RQr`g`4Epbbe1bx+EMj8@M@oxP;(fE5p0QtWi)g+R>9bHSFc%R~!I)K`KK%w(_VuZ6gKgr& zr_jYU3aK`hS)}absF*$n=h#hcZH%C}SADua^uk%Y`eN(WHo1)4+}whKvFsH8VkCR0 zP88AzRSY%Xc{hRp`v8gkgBUKDPp2;jB_+zbPJCb%byJ;^Tpre=7r9{nXcXoR<-ReE zuGLoB$N=?9_j116(MNc9A+G0FY#u1En3W<8OR{K4^eV+DigDP?o4|kT0k?2L0Q0KT zjSN~q+-eV5E%-EbiTa==&NkUzI?2qdGg)yN?4X3PInXc>(#{)=CgKWZ((8mTTssvd z<&^&D`q5uD3r>z{r8=KGQW_=inNDN+sK%Z)^tueJjZqXNI>nR)Ss9C^2JDH%0p& zz4(6><0tm4@~TNaBEFH_o5weDg*u*#e)xUyfy91dyUeJY!yQ`Bqt*Iqg$D`dGxV+D zrS;m}0D3Zz0W>T1$M6aiIZdaH<+WRXOynlkIrW_EQO+Sw`fW_c#-uAyWBoxjw&B6& z->WlXVDj+EwKh@jKy8NV7|Y(*JLyr^m%Mk;&g1#Y+A== zSVV?=ZLXrmvxXK%W{YmcibNZh*gSDmQ)eaay@kktWW|V>4(6RJ@cK_V#7!6FpHj2b z$Q5YlwMO>2o~_--!Ess>#F23)tqhMdq=#Cg=|;M?9cAD#2xB^xcNCl~kC9fN9j(75 zyhr~fd7Zu#f)zzp$Hm))(jJSkIPr`E9NqJ`gs@AHZF? z+$vJ3F^Gmj3{L28zdm)ARBRu_A10wWLat zQpt9SUW(hS74;9hY5lWQ(m+WzaLMG3R$jqMb?SX<`^aJ|x8d4YwjIGArJfq%MM z%YB8vngr>~FGA{rK#woB14q+`K0OWnC$Ro&qdkHIf~>n4z{=+5b+cZLi*erX>e8D| zbPby^%F9l2kGH%34%8uOJI;Carz;~|x1S>P5NAoXJ2jIz#sS<1&f+tX#+aM2BBmP7 z?&}M~<-KlJvD@QCVW-{qcg%9ucI{W2N*?+sF9$6s5YFH2Van8>;`Zjv$V!1}L5x!hEu=03sQ0Sd;0W+m*MQ|F4b493a-#YYX51=E9eM><_?{KkZX40z0(&}h z3RLq_B6<-9cy693Uv8-RyDJQ>XbEv~jXFbhFQ)LCp2L{UxE?|YGO}QI{`$y;+>6Q2 zt`+rmzTQ?Wx>d0-ze>Q)zqmTpG)s9^D|?Ed`6S3Ud{%s)9QK)BH%ZKtGp>?Pza!4B2K;v-A0Lh`wd%KN$I^TlbeLGzc<|W>s2EUVnltC-&q?6VCCs zW(@ZA*FsTqqdcPlZ6))6(BkASx`I87!`(&|iF&Piz!z7OgS~k;w+t*1 zaX3)UVs+SK6t0k@HS7fdx3_ggTFrLrES1}8v%LsgH4lgTLYD~fJ}3z8U`K}^X(|Ou z_N+uvB)`u{PNpYcS4V2Ty?AOpF;HI4dIq#-kG{2@l?!!i)g4$@+xcn|JP%rq;s@Uq z+)y9ZikCcJ#<8MwN0eJ?umTQg(n0gzeDfM0U?E{Wa^9;s9MDP0Z+Tz|vcj;R5dc!Z z1+o_5!Un+-eWX!ZdNY+4gwg}yP`hlM`axyWkA}ElMgOg`EszSQ`N5-t3cK3H-p$T< zSJ4{HmQwRhl(l(PQi)b>zVc`1o14{cL?mVkk(R<%Ic=^C*^Fb3c9VR+y|+P-u>P2F z4S5Gdsio3hNDTnKp}2Pc$!LRXxN6i*PAi|SC4g^hB>STSsAQaP`eM>Tww`6zmJ)Vt zoGPQ<$3uBR>XJ;&YCrRy{%3uGb7`3j!wG9F6E2Lp60=+z~E!&@farda&PnF z)V{GdVRsisYilierC#Tx8z3XsjDzuk>8{9G)eG9{H9OPFJ4K0ug8>KU3lpTAQ8i&o z=>0>Rz~AFQggerpPw&R*jEg&z-#v(eXgV9H6Dl;9zSA7`Or_2jQWr>{`nGaDEUXkt zT(di*n3%D9)UM;u0}oi1COYaA@6S2$4|LH&BvqGJJ3+erm3K&A^9y!S5jkP&vYmT{ z=}K^DXHZsm=B-vxY(dotY^8ppJcWFb&kk~P>fqe=AhyRVv@B1&%z$u)^C%igs+hda z;FD3_{%+aD%T;#6V;CNr;DCnjPal=5&{2<8jo&8iY>Qgmm+wX+!?2wdVXrGe3Xgp8 zu?4$pr&YA}M4#AF{t43bDIG^@AT)`iUWCA6Z`1bnfkC3EGS8?j=gBE2Eu~Fg4u%b$ zUR1LHARfs*agmJFidL|0Va)|bFB$LZ7(9mLV|nsxOE?4xJjgBgd3myR5+p~>GAQe^ zX&68!Ip%ZuEG54m(&~oKxd~T+G)e8m45T+N+Ltnqv5}i`?l5JY%}XfY%SqQAKAYt4 zQ6I!`D%;Z?pKMx9EUY!N=}0^;=O7sA)^a}k?Jj44{F(4>Z>@B;RK2sx>Ndv-vEFxVi+kmtW9r9zry{c|N~ z2{o;M?&y>hR)O19ThKKp-7rpbg6~LbmuuDGYs}O@^~<=TzVm5tKc{TC=T-Ai3g=fU z?j^-N_g_cH{j?@!$IclPB*qo%Wf%C(=4MaOfCj&OMtgbpxpS6{uf%k1L2A`YE~;mf zTFl|Q4uE5*+R0hDq<4`#>3lwJZqpy0Ft=~UzY9B`Hz}t%=`rdj0Jt21mqH(7Q=e$* zAkx1M9asN|0KJ~bFb8erS z8z(>VT*vc1??v%S-9z&2Ns46rl&zftqpQ&(TbO1+&Js(fNCL=8^g}(3w0!H7mNcnWS{TZdav6>z1c4t)_}p(zXFBMXof!E%Zudlhw~| zTAq-(L0`)W>KBiKIRAn9hp@qedf9JD>`b!n1a-pxb{(FYe+7$VT!m@?*fW6{tT}@j z>2HLh9DL>}7-S3;Z-9@{{ZrOArWkk`QyL>;3pAmP71Nv-v0cu9z7|Cb(Q0Z>t!!-K z&ofJ1(k-#kJX$&r*mHLaOk{0_H-HDDC5c%q6;E>uIuYqkZkkwKZACqin(r1R?oMrT zV-ZiuytO`SbM^yV+|jI6W{Xa>RGX~)9!k!-2msqPa(w{Qn{fa31E$th!|wBy21q08 zzN}%;WZJh$kb8N6OPYod-FkUUSg24YvMoPfXrSCRd&bKR8xU_mf8dh+F5`f@v9k@8 zqadl4;=@UPF85lok?__DJ`O<5=!M}*E&&;8m~GPBoa2f@M}kbJg3snwu2YFkyNgV9 zrx~w3u~7)5N;*a4Wm-}+m$GFCqO;tz-1tZE!|fjN{)NC4+PA>Ls_YaDz`NK7E697C z=2Ldx{L2y5eWcstr0DB914yK2l6KYOGb$%dzJfsYJ=FgH|^ZpR*nz2|!;64r70 zk;epcF-&O>w?GO77RcV$Pt3CU5y8NTUNIkdzP?hitk7&sM zIbQzp(UYDD&X`vR|0(}}bHOS)tmkxCMPK~>E1wtC0o+yug!&T4QqPBt`wTz42CgHS zb&S#$g?XZt#Cp=RHT{Qp^}k*nx^t4BU-gcz?$y3@Ismec!FShmR{1}RtOWB=1j|Xv z20z%s|MjPT|C8tNNl#(~*JYVsKa?K}bg~>Q`q_2r^z8q&ly!VGlXBskVf+p5`}c3} zJ%a%TH&L@=qWFJ(((~G*F{CK#k14yf!s#R~Zj^BLpO3lDhYp?>9aZX*S%nkx{%6KJ>%+h!s#}|73V&_&b7^$e zw_;yoi3Wmc22IwqEf~Uodox=oB6!WOz^v#MNxm}CkKu0z^TUGb_FjaFv(W7Hr;Gk) z>3;m**lyY)mhE%c1cvn3#Ct zhPHMr+Xem400gn_&C_#P9J_oT1m?SWcW;P*wknNhyB`^lNE-;S*8n`JGK-$y+S9pZ zx@RoBG5e05W!)fZOfZ|2IlrS3w5K|fAU+)O9t&qYA&9Gh1m>v&P9v@MG?@9{PVea7 zPlExwvoyJy>X7sGthB}AId*>n`ON8{=?{$Uf7oIZz6S7@kz0$^5!$ov*>l{> zxM$%U?w=y;f2_R9VYV-Xd{X%|4;@Bzb=$>o@5z%NJ{u>~{V#s~`rbeQ%{BwD)RAJ8 zdxzg1SCEvEmDMs*?UM=KK*P_|F0SevFx!_NPn7-_AFZMksQU8QWb2BpBy8y3RA-7P zcor+ROD}*`oJj;nvIzX|;owa6Bi+TdF!_a(&IyU(3CHLpa4z}$~h{? zQ33V&PDzI&0G%f*hy_sqk}j9q0u9KBhL`NEfUe%Yl1l9AD=@{M7zM2V^Llx25bksr zCMPEL)U=nF0|to#iO=ikOQmP{@ayf&m87MuA?QfefJOu|#c3EkoU?LJ_o%a)ZCRtT zT^diIT`v? z1}4m#rRfz1UjPL^dW}9Lknl!VD?%9%8fmDR$NV5aVALi0t_|2019Fanb)b`=L4gp! zvLZpEYXGQWiFIiT6wXx^4K8WNy}k*yI}z$UVx9W;rk_Xel6R%}PeRY*KCXQ|_HSso_e_dFQ2jAXI&!TX0Gd=kF9U&+mfu-q8#3Rzz zv;S??hl=XVn|M0QHxwz*Hf_VhnD)TG{M;BcF#3YQl-t6P;j=4GQPIN2($j#k-z*EN ze+2U?gC!J?bC;|Jh@k9X0lp?NV0%NhMkRHA!;g=00>58jG3}*DF%qA7E4D5#j1sQ6 z_7*Dd9Ts#}Cak8bTNd;&OXr)^dHt;Lp@lV@f=x2TeQWm+K2ed8kt;Ll;ZrB%+&&kZ zA}l{NuP|r}vQ!N1zDt!o0^I{CK>5*@3&{NUI%AXsYU-dJpb6sYTSB)- zn%zaY4}D5rcFfQ?)kNNuqM_ zg*4TdAC#27Kf;W7x4tUIx`T3Vq#R_q%YZ^$uJ{Q(lCr%jfEVW$u7u*H@ym5VG~^RB z-2Jj$CvxvSl98K-0z|X@^Q3cp*WrkO?Uj_tuCH__4jqSdRBUdMpD|0r9k z01S(Y~yOT^~Th{;miY~ zAqC}XVJBoQ@G%CCQ2FbZyFpH)8E@SFyV7jfUecWAt=k>nH2Q#-5}h! z0P_4GAOk_+y4s|HP??c)HG~rG@C&9Dr89%5O^r)5zZXY-PEY)v)e>EY1V}+(gV)*x z;LEZLc(SJtYAnsTj%_5aeUw_iGdjLhzM|Re*oT3O_lQ@MG~*@rl|Ln8M*S~T`>dl) zF5qsV-xb6zr-`a8qGv`e3l9Az@;`r32dZT2&#zCRq4xLZ0aPCisLxu!g9+1AP1u=j z&;mR<8jnOb*;mq(1JH7O5U90UX}&?3x*Fv;Sb8H%BhEr?FZ%`kHuS#vY#!Y&ZUB+M zOTW{-1v)dsXcOlBJ}p+GM(8x%o~Kc6az;tV1_jwf%}m_x+5`H+93G<$xH&dK6F?Oc z!v!m=pFI?{zD*%bb5If3i%c-NUbJcVnCpF0K*He-SHhD;FDaY2ijoxK&5`1OAGeY}*I#rMK#A5ibl(AADFeh!uUW<#4n^-$UjXX`8rqB^&>M0$ z)$Z(lbNvNk;`DE`^!-AM;9(nv8DXEczBzA6_nPzbJ`K4|onW|}b#7SqfAQA3-#_>` zRORgc^dPq&J>ws*>r|k?eNKYLlbEi=s7tFbx}ET*nn5#3-b30C_XzIXotndF3x8as zXH@Bq+;_=1nlNLXU5Duv>-tRWFG@G@CS*2hWvlib&gbCZxKr&7$5t^`oBNUVT6Hrs zGvf|N#hSVsCc9M20Jg)i15+|iBPMRF9?sqNTZ?NC>&+ZYN5BLY1VZecmTZC z!+hmGzVMpNtlzjVsFU^>Xft$IOZw4JWBlQ-oD(>59JLvN-rin{_oaJPrCHFHn*QZz89A0f z>puzECVSc3*H5Mj8>1(@c$>KnXw1IJcee*jX?NAIbQrA(g(nt6-#=o|R# zs4PkoV6yE02(D!?nzmmg!TN>cKxm>Vj`7N=+thqnpZ0mTYu_2Xz7!I{0py2EYy;M8d~ z1B8D25)TF6ecvB2z^P6v)w6FtbZME*mvFg?t|{uLRE8WCXaU?RVW$9)5pkz*?OX=p z#uje=h0c{)9uUJc2nzuuRpWqBPfaaQA2fT7mjA4>|4}c2bWSKRS2ugG4@WkG3+e)F z4^kKn5`AO+VIV^SDyh!mw(})`^{Ca5l@Gi&8lJ{h1?|oVG4cHRBxafxmnYwJf%n@> z8X;r#ayJKZM+4#MMWdzmy5)dBenmRdZ~ITW70PLJMB^RQ@c5en;4xDZM@MJ98@e?9 zysJXmUCEpB4T{jf?>lHm!^&?%zEq`d+v@?64^VnYquw5J9(3W!`=~c`{E^Ff<#xZz zxxw*K-B>J?XzshzfVQq@sE{sT$1>9$#<2^sS-ZkinbOt5hkrZZT@PQSMqXf+(yRB)dBI8QMO{C%TCgDz%;xeX)NZ z@W;|$oldd;sNnwn>XX@h0C1oMK!ya9ek4_ThGfyFT=qL?MkM zi*26etf?me5o&~KrTdL&K|Pn3b6FaFCo$NH&7KHTTXskpe(O_2D*8_{ycR>;Yy}UbG<1sGrEjsnETQSNSYfEy?;OD1 zYnSf>`*fN3d>9{-7*wRsB-_g(;|ZA0aD?tetE)WHwTAv<+x1(wjwW#q3KG0)aZ2(!$PT@gxCi!>H ztPOOFE7DYwBr!5ZJB{Yl$%aF0j=+>McbF+IF44`|*Yrxum%GgZ)j( zT~nwG$SyoTNUc-OYP>zk`2}@zDOuc*9s?7e-*kj?Gc=bZXvsxNSNGdurp)Fl@wWU! zKAFm_4{yTyD&cZUWo<9f^?nn6%lp6~O3TBp1wW0!``4B1y_I?=^QHwxI#;^Xy7ibn z*fKN!E|?#b0vRUgXB9e&d+C%#eKD$?rVC?t{Cx96Zz zAy%>5JxvOQ)Z*qwPBbW0N&yQ?hnolCyE>m}1t&*E9W}8UA--w(>WjN4edKkV26}gF zdqESxpS-siyA*%3O;X=8NWM*jG}>5M-E-#$Tp4V93@qCcHTPUy9R6Yo53AHn2D!Db zVt0hU9K5(7R36-b?_~VLR$)vT4+CUCWl!8!jSKT`;c@`_4@`f_H*-_t;F3kka6B+5 zvGN-=YGn1(6=zogxk6a`@cbB5e84Cul5k^pmgd?7^quZ+aE(WpJ%3V-5BdStJZ<~R zvBA8(y`9|`C^Qn(g21vCX`~ujsch4B{Aih?=VGdLlBaf@g;Z!~3}opcuPzTVHFIbR(Y z`%s(@>c8${>szhjg;oxAt+19+P}{sPyU4eKSMgX>UmUr<0BiGVM0sz*kz07C+HGb@;xHNV?5F}>!HCy11#C_Hr zX>|c8ATH4Af+QD=@5Zv%@{x_u;k|>v|F(eQ+1|_Ez3HC5NQOp|8s)`>g7*qO`uZ_d zAZ3cN){g^J8c9f^LqZ@P;QZElir=cP&6Vs}94#kpL-tPhE(~-_1G0wTwoZu_v?ESJ zotDBGPIliCjRwuQgkq}_)SgpVR~owMackz#Z-6Y&jwtC=!auy5U+}d8=scRjSdL#* z0U}9=p*n8I_v;gOq1Gc(D?nTPM%=okY@sGhVSAi+XQsmcf&K|`tLM;E(_qZIWQO@l zB^E%rZ};HftRvwH&1<%TY|dDuP;rT${npKc@i@@K(=MplbimJUnwD5J?8m!sZ}>HF zd-TW9NGe_}kc5Du;xhoMM4bQ{c zKohml(s_->(*i=`DINn>cR{_piQ|v=*EI4ddS6F5^ahnEZq$H6nU<%v)^N+ZT50X!A2()cFhUy6fjz$#Kn9lIUIRNB|I1E@z}25;6TJIg;S3+&RX0u#?&_L= zooxlU+b5)d_{{3CelEaUrUsaWZb{?oAhx3kbPc`%mt3m`r2gBmU+<+04{MFo1R_nx z8j+o^`4%pw5P$hZmZyLG$6*|~o2s3nv`yC6NK}h-K>f=q($YEK4dt zVB#X;0gEh#O!K3(%3263Wg&I^{}J}pVNq`D+k&77A|MC?f+!`eq?8O64bmObFmyMJ ziY*<|N;e1$ofh3N0}Q3oEh+u2*Ug~kJHLIM^UprlILy1&de?g5e(vX97ujc}#r2<| z00nt#5fhyujc&>Xn>ew$UdZmyY9ACLqd*l^Y~=xFA0(}M`4v+%nAJy=cR$@IBxwwy z3kP71XiyOsh)wye`=mi@6^n@eI`~f;<$&V2UzSRme8nwM4Hn2~%=3LzdY5VOU zy&(6b6syGNz9d`DR30XW+nxR`AP^&c8)-sI*M)o8A~&R%6lVcnGz1X;1q>rS zFjMm4|LN^1J$lS&2{q<8IId7X97-+st+%tf0H*#rt zQlzpr{S*R3-wOi@bdEVG%H(WTszZms;XWYuC^Qki7Z6#^=i+Vu&!fc``v^2$cR8Ad8{BTttv;ccHHExAh#Zof8=V8Al|<{kJ3eke4BG z-2*VeY+I2GF*bBpu5r4605i~!Xy#u`Cqsx9u6ni_S;9Gw@p=T==|vr+HS?I?Zr-qC zBi#$vpJeNs;B=q$>{~1wqb~~5AW?!9H#%+&eGT0letFq>D!<`Ddjda#%0)y=`+5XcDKxmf_RwV0Tv z`gL%A23mH|9A$8dn8p;PgffZ0<=f}RkO}h=Basc&Me`1v?Oz%4M0U; z;f?TrUPbvJ6{x7Y zamr7{UHs5r!jcIKr)-l)g?IiESNhK(f|vtj6Rm*!Z3+NKC&10_6(UBVvQ(w!Pa0Bwt4Q)Lkf zru*kC+y)1lnE9^ozxZb`+lPdysj1KHnEv_xM>3;!j6+H0tN$GLeSNj3JX#!MjHyI;uvzn}hu5q$F!*WABBk$<-R4N$eA zWz>rO-%tMt%GI<%l&=4}OR+;nTm-B7hK#fQe?R>jz)XpyURC}X&%gXX*lTb|z7j;4 z|L>f{&~l+o8X(Dx|9E{o9fT|CuIct@y4Sx z?f?DsONjH|)YAXwPsWorH8+O>tI-V1p(*IW0Wu&uG71WjfbA^@99bJ2fP&1-%=o-` zdSKmMzuaa1w`U_B;1W@Ch5*3qYmp96+L;8Lq+(7+)w?HQv$L~BAdc0pa7xS7LCONa z?b>8ZWH@MAXa01HU|i+a;hr;XVAr6Pl@$aw0_k3qJVfxx0;$!^Njhta5~Xtr)0Fz&i^v{8 zdK%tR6a_(dBJV`0)<_-Y-`-I=_LPJF%ZZ)0Rp7l&+qcD=Lw=F0s$*v4`+P! z@9n_w6Y$hGgUrlYfBs-K2&Ug!oZrp8)ZEgsXDmH3@jyZ~-Hvs0>r&<{iTGA}a2PTHoE5ryei#fJ}A`&))S(Qi5wB;e}O!lm_Oioo)8;ha4fCL2^-?FXq-% z+O{qq0PhF!Q`m(`?%d_}FveQnks$h3!R?CO z+M<+3Bx(bg0WU}?7(mAKyqyUis1wGlOLW1?QAO5T03ln z6pTvxE_;hBPk+vd3O#=ZCPf5a(Jk{}A-)*Rl@LbTwsG3Eb^B438Gl^_Pt&}2{}#=H z0EplweV0{Rqas$#WaEcrW4%O-5l3^*#fLIpfEbrfTIE--%04QK;te(HczfdxkT0>~ z`ype>GUUcw6r?pkM2c*V~Qy?Bdq` zK8~3IEi%^fU|mQIhfcwF!Kl(L5!O`F9dZaI^Od3 z$bdMZ6M!&Ew9G(-B%u>w0%hRh%zL1tLI09uq$%mRe+Ba={VAHij`_;FfK$p_JCE|c zH!nwyoc1ZGUA)V%X$nK(S@?JMc7O2pY0ya!eES5(OE@$0@GzLXwE0`%85gc^CE++D zLZLM{qaq-Tp!_ZE(K-Xxp@v?seh`T*jdHF4O0bCKd5a(yG;jEJ;d!bty z3G#Y@d2I`Ga}oL~i%$CDgGB(&(akHr6$3jleA>V{?JwW+S&gAp=;QTXs^|R-xo5s6 zo^3g`6jv1P%Kt#__6fwVGp4(K*WV{ejeL|58A9VZ`QqpkPid^zox=AN7|)5zr;^8x zzY7=89*gq&${kH&B>ZdDKmnjs5omX%T688xO#*WqwQWkN_YTxPiU5rBNGM{Q3-B_v zK|fmfGCPRhc>reI8oeCx0gmS0{&e|(0Z8_Ju<4y0sOPP^g^xOewBH!)7Vt{wfe3V< zlVc33HEw6m66`_+oLu|)_2HRdVZZ$iO)Egas1=5mHnW3jNDXLXxpcD3Ivv42BK`A%w2VwBz?uYVSvST9NBs|dy9EmK zdb(4j+B18>q0@jw907E7PFTD=s2WPz&N%5HXIVP4l(-vGE4RPLavJ} z=}~DfD~Jo)VgARn00yR@-LR#%cCN3L+#N|=f(Ul?mfW$SDtG&C1*sQfUh{cHzPoOX zqlW<53IVD!a!^rc{!8%#bh4`~$*o`S0L*YQ7<0A>2M>S={l zz{OMmO=yvSaC(Y|I0=a!m?bceh0yQ@ZNmVK$0g>Xp&2Q&(qQ|-9-jGS?%1*a{<-R9 zm-Z3s+IlnLce)rv}P#1+kK>de8ozy$e&L!hyc%)y_?;2&P zICE<763y0vM}@hWSi(i5OEl?bpoPxTaRzb7oL@mj1_pA*-ZTjnTGnX2B1%NOOC}7>$2K_S% z4860rp#}=5rI&iv6N=FQQk0M_rIwC31YM)`fDz-Wu;HAN#Fw)U{@|5jc%Do!-4R>K zB~LAf;!S6udew)H-pjHxmH7Y39)C?V_&y zi&wf&aQ~(zF?iv|>;}KIYZYAhU{fL+*uxT{i=k@j53pOzo=jClu1=}9LKfo$Za0}0 zpeRs#PLXI;Sh;q@3-^G=E{OWoE&>`Xx?zKchdY|xWIo?m=r+Rg><+P?jodqm?!n$=Q~^ z4dH_F?}RZTuGtYFuFd4%9{t*@E>&txD*refUqQr6Gb7{xkS`_y6@}Z<`G&AT#-&4` zI(F3>vp9V<`KyUbNRXxfO2cWojQgl@TToIp>3h5q+rJF;UCOL9ZPjZ)o_Q^PcBd6k z>i~ML$6Yydwb;+yNPLsbfdf!S^C&j{3Xls)fw)and#!RYZRuh(F^1%=!pTxv-(5UQ zuKFfu4bDG+p~(!3U6TeMRe{@7$eZ#EM)6$dXLIa((S1eCk8{?b#A78D28zn5Aor{A z?Pd6qD30R0Fhi59BXQlT{-Xk+fPzz|AZVzh^8|ez%mM=aMcJqJ?W%u%4MFJEJOy&U zFY|@xDMDg|>=U(}HPMrA?i2E(Hu+cF)-yMPMIc&}Jq=1LcWiE|^82F5!;g54^@~6< z*1`|3>z(9DKIA57bcR%Y&3nnJ8f4HxzBBKiO@OKunZ9!$+-kVo5r0l4v-2K;Cb+u+ z$~aA0MQ)FAWZe!VbVBR9H`CfzlNHGU_reR*E{bQ1)A>%#1fSv$$&q01E|n%Ihp7Db zUx@tKH!?i$dM_1h;amD+++Zk+nfVt+d5XSm6pb>69w)#Fkl}oX80pvDjNy7Ki5|?; zWK^GnUoX4wY31?Ker(GCJ^b6<8=Su=V6B$8G&5A-23G;C?`gb#d%wgs7(G)MIqWMY zDXoBX6i_m}6UC{0wK8gM?C`+7x%q>^ZBx)1Yi=u3Mdk_0BGrrhRe;$V4SKv{-x=m; zr>$$KqkU?ox|N9q#LV^5(hW7 zd><}-#`Ie$JkSsR1<@!!y=P<;0Nwdt@mf#?hDjNEW1KoA(zW}Wy5x6THD}(x*7D&N zbMdUu2b2)SrP}_WgTgCesbjis5=#XAiMIfk6j_{`-Wh-DaD&<@ZrytU7=K<2pgOQa z4W7WA#j;hq@_T@2bVTr84HIa{h-43Ia=>3)SG-}ZYbe?24ATx%|QQ&7wWQda^X+|FbE)u#J+g> z>36PzFgQuua1Cm)Q_Z*MKaqm-36zMr4YheqDW1-Qo>*I}g338|3~x>Jou_9Z@z#-| zprDR^vJeA>55M$0HgtgfpC3RRS zP~HSuYwZMDRj<=rb!@amu<7|xEdU*89NZ%Sww+7vUjsCGDkd`%nenAcuPkQS7{XYO9 zO2e`CoYMDiok{Nzo;+_VH?~?2 zvpe+xaTACFYD19WHgIfrC3r$X6+m$El!E|F{&v7PS0{F!AmO-8X!&llY;o6Y@@q|z zaWB30H}8IkFCPxPEq7yeM zB#)AbTWOaX^n^Ilx-avgt{D&w*=~vdN4^S@nI600Mon|QXIAHNFHp63T>Nrx_oGw4 z0L;+i8-DRY4tDO^6JO;yJ@SW<}=MMU<8d8&fYwbcsMh&>F%VZfH_8{TFbJ!5&01hyV-R!^L zG6g=OU6f=(k-1`6C@nQ8Q{3k@%vQO)Oz(F=47QHP%S-aYu9(VfLQp5qnA*yqEID3d zd14Bq#HGty77vv@z}csX-~#AIQ?C^Rcl-qZcUuswChJOu=QmUU!SF2b-X-H?-~J$K z-FQUAeI0(WDrRXY7zhI^ZiBd^9a4~brP%DcnvGz{m`Ax5xoBvd9 z0@(^pIv>X5VZM)@tb{tLdVM5^L7NnBoMMe$(cPaP7g(kNtiVK?396SlIejqcn^wTB zd{YZ9&(H6Sk#E3!bg2PpQ+pGJHPPcXzXQEp&n(i=dr@E;Fuu)jAJX*olI0u7f7sFU z_^Gq++oPPVt-UGn_10P&-@f`MFI&(z4Y~}3b4`quwKIHCRs*iy;JyAvhsYT3HvxtK z^WflMP)6U*&R+O`H=tc}L2&Xd{r>)3OB2nQimKtx1DSPDjCP#;WwZojw4d?#o3>}8YU z!>YL^3`D{@fS3$`ReTqXPb>r=kKdY7pjYNPBhAR`y>nZa@bfO`bl#KPz0mpti1^J7 zROhSftA*Wi2AroGx&T3%sNG%So1Yxg9|O8SccoaW^m8!zbgSf0N1gGQgx4|L$P{1S z@Tt*Pa!B5=(5;Mf72hV>Zh-4q(7@q$!Ef9lNBx^20l1AtMeV_vE~r;Z7D?p+G?6WD z8J4Jt+2vZFMyAT-@v|9-t-EBL@z{0&M*F;8y%n8)LRhImjhAx8M^mD7j|QaimzTT_ zD^dZQ7y!s2<5x!;wP7b`MJoH)5@rxrz z)nt_J#IwtOgw3M1bUB450izp<&DIMV+ta+Ot?L@ZZl_0Ho7WM%TH%Q0WYXQd|CyxO6yGpqkT~ zBH#}M-;*Us&#*%L06RcV_5`i(HYAu2CZ`*8brUA8ftc#jR!q)u^VN5jcMU^x`~6eX&a0C)Jpp(^u?u+0z9_MCv6hKPQUnKtNXGOLx~>+{^-@ZhS1ko={Z$rfy) zf+#0gkZ@_kgZt}M-0=hZ4Q;Uy-0CR|fi6&ajv~MQ!;itKI-GSW-IVlP-UdCo7;m@U z?4ERO0b!)wJ(-8GlOJnj;^`0<2__~y>}lwTd94fYNHySR04hQY5N=P?t8i+#S50!| z!Nq*2j0SX>nu{y|RVe~+P_(0F*IPuAi$ODaP>SGmXOhwiO6B@y3{A~qEPF(;@Chyfr9|Kv)2+k|G;}3ik#oQJhi`) z)kDn;APNeGXf!_kCekL8z?AZ7PLOH!u_{t(_NjPGiIs_N0k}h?)QsY;Awxwt5NB(?Qs0D` zDDTuezCjq=lA|Dc?k(GSf35r!xO>aD6m5L#;KxUGfOj)Qmu~v#)#?_t+jt=%2+dY0 zk7asCkS-xkr_{7tYMhYl+9O;Vk1jTA*VL5guza&@I7UYyCZ_=dGC5_P63cG`lR*5e z36jC;?-pR+-I$N&k)Y=Nz6i-0>1ya>HX*GP+X_m{LO&2{mbz9`Tr62MeMC_WqG0oy zhdE1g(+=R`6R4GtZ*?SF(ZK*zEx=wiIMc!1w*}qeq36`h%nQWzFn-m5z5{`>o{$7a zY+LCCUwqK3<=@_9oUsv8P_-nI!E#ORJQT@0rBujc4GhC2L`q}v1l1VD^R;N(XivGa z)~STkpM>L&%jKYn^L5*u)yX|#bB?7r#R{`eElvh&b3LPF0=e2(41IaQZM!RY*&2D) z-mBo^mgaj@`b;9X6*D&5IywX!hCe3GshyMn2QUm&F~khVf-ebQolf-5w{*~uYK{av z38jR~e!nE4YE-wfi-rt(JQoDL>dK*=ugc~DZffs$@tp)Ifuf=RK2mIZA{>%X!l#<7 zMZk|N9@yXeIe3ZqQc0Jp>Ss&8_l>|2rKqEVE%{8zGZkCZs%CfKBz8{a9 z51#aXCLLk0pAYI=?`JHe>bxNdwjDY?kjw-6e2U340WBZU6(xdte_a@&%ryE=v11EE zh!6;RYN`%TG4a!zrPcR--Hp7`oe7&T@RtK|$BMp8wfjR!26s9n zQl7^5muG8B2qixiCcwcRO2BRt*$k8#n)7soEK}Nt zr_s8-vKOW~DwKdI!5v{+Mbb*;?9Zn=*u&PdR5I7|;T|!0mzmdO`Pezd=K=hdT`&mi zPRCVX3JPPEgCd5uCL?`#FFR=D*3-z>vl7f%t*?%VpdeS1-OO_PvADBP+!-7C5{P3E zR?}7N?tH=_UU?ecackEG6fPZZK6@Z^4!82u&zbI&NpN}5%TI$>CBBd=iCqCXFP{YI z^Xd!JjKCd00?HNV(_|PQh*{-F5dEQ~T*u83T#j^S%?r6z&=H0)jA35(LEetqyFVV) zh;8?;zTz+=9o6>BEN5@AuCD3wQ~B`5F>RxC9hCNM06LON;PAXoeKTPVn!;@!NdnjU zW@2U7#D#wx?0rt55f@cE1K#D&0BIYWqd-9#J1VVFTxOsqrUF=^mi$t*%IC}1d*s>) z{0dInZ8&W_yTo|^bqMJGjd5Y7r zEj*-3Xwc~mUE$P5ffkwj3989}K=^tufv}~!ei<}ITB068S6@6}v;V;y5*5X=bBAEW z8l7*8D(P1)1wQP6sm^*~+Ix!^dn!KdPh%RF>vHXu6_oy?`f%+zEbIK2v>((0K8WEVr$qa_;RuQYvJ!^GORoYF0#4m>Um9iu zbU)s>gJLJJ2~zA9nBV@

ZeQwga?5DB?!8M2E?BWO0oQp(4K`*>9?+ojuBJpm;?I zR2?_|ZsdSj;v^7ye%vGGW%dnUFdr%FGU@|p)LV#IM$JXc7y!e;i|v;Q=jhB~r7FvB zFKNMdn0SEooS<3`e(+6!ki~IRF(K2t6dQ|!N_Cbt395SE^mV38P0|L>Z{h&|X@U2Ty>frFx` ze-)^+VJ~iri$crv&khLFGaBu&<^4W?p~$?omMQu#aL}H>r^+D*auY7L`npRqD0yq% zgd)&&_8-ojx; ztB?Bb4UT%->uij2X-Daq>J>h27+?`(f}Q^uck>Aosq|`?4+A3P36oC~Z5Zr3#ciT5 zFq`WaO=vYasha2Xb!0H!%lp2GRidl1qqTg+?o$VLP8IHU;?v%ZL)&fs9m4(gHl~om zD}f_UrNX(Nr=@?A!Hn)BBA#5#xUhsN(7mcGa*O+KeH6rbhHBynPV})OW+6Bm{JMpa zN8jlya&h)@Y~eF?XuGe2OAGHDQmX)h`P+p0)khTDGrqK@cdWiHD2`wIYZ<1l!8}Bz zuAe)8PgCphS1BPPT{p5is3hx&_D;)x1 z7dxMu`Wa91Vwdp*?hjm_j}v*=FpmIFx?Tt4Hy^{hz0*5wb{FUd9>3MwhV@@R`MMYe zE{F~?BzbI^4;EeJSm(G${O2orjIdT+v=|}b?~6LYkA7o%faziM-_K)o%1-8r&%UP* zzKoE;xAYWcBcdgP)t|&g5pEsCMOkqS_n)V`FpHp3y>~oe1dM1Ui?{2e9#(}=jJ0t2 zZjIQlU>>qzuccH{YW3OSPBwq(hF9KB-=7W=h7-6?%5Y$lVMo&* zS3oH5Z!W==62p~z7vH&T5qpv&bdHY$J;lwenmn;zUwobRRB}$kl_LLGn>{v|$Gnn4 zKDIM0@xsMRG>(-rr9W-|TB>T+u2FFY|Mf2T>e?NF(Ht>>KP8uyI9SP8{^!8sRSPCu zCqCPn1czhLodTX}0oKO@A$VIqzVdvpy)0PTV)@;0aRvp&@!H_vO>u(FV8N?+{DMYF zXD(lglGty-5%zjN`Q}aT5t9;FMh~6wq|4%LjEpHxW5jaei?GQ*Q<~z2l{GsH{Wxkh z0(?HQi>RmzN!wqqJklK{`Lk8*&;RNPR%{M0(5$l(r9B=$*t3cOD{dlmo5#=VCD@w} zR%*@3pv@3|r2pp^3@Cqn+zM{~@M^v5`{dRM*T42Ln2Y9(pyLnJ#$)aXtTxF_Y8{zP z$h==qtM6LL>{XT1)jW)7EW`$`(3W}u-Y~g<*74t)U+EecWi|9!|I8UQLHcLa3W!SF zh?^w##>>&vnSX9=t>?r~w(z|4?05zQuZi|{l@#SPmr3b=kN*em9bYBDv8-{b@me=1N4$P_2K0KI4ev60n8Wp1Yi( zPI<;I1BF*bRvrV;X1a+>BJOuTUq6nKjEwC0e@yzs2#b=;r*qgzjJM;C`G4z)6Dq<# zGOy4AP~)}CSD79Trc(zQ3mv&ahJXMxuq-tP=iVJ04eUA`F^ah2d7w3t8$1oe{v@!l zEQDg7@4UXtq5fGfx#7-qFY+t~J99)1-glGa{GpF>*k^;MO|g(RTC`Lk9DlcBK)Z4L2F^#_ zU~z|RY&XFgRg>bYxWnf7zXUV`2v=gl%6|8|Kz|cZ?*cYz?`$Cn_SL;*C@i|}`jJBe z`*iRRZ{uL5rcXFqm0~b5m;Us6jI)6UDv8^%V;@LFHt<2j`$~rM9}av#BYr7oCq<>g z=C8Pbx#L&C2NtvA#Am2-`FrK+SwkXecs2uR|6Z6iGLP1%N89mVh5sxLJ@8-~U7ymk zk3K4NK)8>TyL_YiM)>DTiR97>uR}4VR9^X`YJVEAjqJc&o!7o&wEk~IBib|Q< ze-Nr+O*$#eC?9vZ#uZk5>86TR0IO@3R0jfGij% z$d^YsyOn*2r;!#)zSuD!wh30U9B*02WA(TaseaYd<(T#so$VAoaeOEkuGiE*8~vcD zLB+T5bzOGh*x8e=oFN_EVbVHgxjB;-Mv*7^9SSygO27T2ed#!6Wf*5hlr6J|dKY9= znevD`&mI2@ry4M=?;piur;&I;S}7>J^KcUx7IS`D`y~6Z>?p(aiu%#^?bT3kq5to6 z%3lHd8~<$F@w^5JD9o8#sOYOTih93WINxmiV{SZP^^Y(TyOCVyvwn>5{xOFP*QpTC zl@q3%a2t=OLr|^#Eu`uZG0KY*!>b9 z0c_HKRx=X27khE(#80lq*k20!WX6@~_HOm~o0|9sojiG1!k+xk(M4RRl2&{krBvAM#XII&nYK`A(~DY z#K|3x?+WwEX9Rm)r2Fy|?sl+5wIn!`>llHGKW*0m-gq|Bn5~h@+Xvg9(Axs#8`Bml zTb9b_vClh!7z8JvL*wO-qh(@Pz^Ct73aZ;)_pIyZ<6)GnT^UENeyc*u_}D~@<+uI` zDDWsXjjRlc_IjVWx1L_o;!;*V*!5b(bVfBt`*~P9ax{)$cLEL`r8C*X@zr1c>E0W*&QV6$umUV_;UivHH|zaUy6C4bp}6(V3q@OEoZ{& ze(gwRSOD$s<$_!Oq zzPR~UTQ1Xw*w0+7LiM;?z6~}#xIRg9{C9PDPQ5=Sg+?o7?Zyn4Lj5i({+M$TcP_vG zdy5KP_xZv-FSy;9;{in^|D0sd)b~+$SqyHCwIh54@|2@HiWRoH@An#N=9$?> zB}T68=EGZePzKvmot>@Zr?bC&6*c*jV`_;Za{F#>_v>mj>7%2bzj2&j&MNblKG=T7 zI%%;pXkYYV#f|qu>GArEk^x82QJkKBG=%Gq>s`L(C0OV2984hEi)_tMXEkHTmSU&q z7iBBt6Mbr8qsrG9CpfYD-Dnh7*y`tM{ixaMratreoSFhWV-c_S_YMxerKn(fL?iQYp8U7y6G{#cL^X#eALU{y(!wyMV7NVq4lnZC>2-GvG4k*6tE(= zm&hh?v?CCngl(+VNZor<_nO4LWheIm&wk>`ThCEnw~T~5Yj>URw{jjX)ps>5D1JZA zSnGe_dQzY1&?p^hsy}`;$8!%bAF>4LpBgoafA9A}VsV^PE~Uh~?9&?3*;t9zQyO z6<^jle6l0qo)c#CYP0+cqVk5Bnapa3T*cbe&)QkmbJ{EW1ol+&P54;bgb0CL&}^gB zw<8l=pM-lrXzy#r-tFk%zrU$orj9fE*!kXdkz6Kl2+ zg_MQ#59~%&7~B`Gg&~<|C3EKGb_VBjhI%${+h&X{$#~2Q$#NgZM^92ue6y9lB}j;k zE)k?F?qH!)0?R=x7C2laaFfe%vDQ|9e{~^L#B;|88J};nbs-Q7I5Ch4;yT_4Ewb=U z9L}yuSkMdS>Q+Y3m3};J!DVYGX23J%vr`?heWPSUKWPQU+Hid8R?>mp7fKRE9%TRs z89dwzqdRtHBWx0d?lEpZG&vn_DA%^*7^1-Dbe6K7+)iy}KC_;v>eLT;`E*HWaSMIwqprV<6GYqj?F$(A zA*f}oLK&;SMGydc_es|NXJ_nL2k?-`=j9( zg1OjJFN&WVxY*Gz?DQjDEF;a<*k=V-mwa}PE+VN~hYj>PR`4UY@Gres5gh+KeB^@g zNF9)1>N%H50*@e7gb^VgNA*TO+x@D|Dp&9K!|uLWx!2@3Y|jl9)gm?h-s>L>-mGZk zDJ8fwaq3MX?UAXiKa1OZrn?Akn?GEBL#gU+N9eDC@eQ5t96DwuWn$TgMPSNP?TrS? zgvaGRVgksWk5vom`$0UO?ktJwzLG%Opaq?1 zM!7W=%Dh#sg#FcCboNAJfrj)Q-SMjX1|=_@Ot7#21P+?0YZbK-{2ce_;KK^>D2pc- zOh2q+= z{z9P#Zb`MinD^4LCWbtR!-`MwKzy>H^=J_iHd>h?jfzTD@>XPUqcme}t`Rh>Q_EFf z`YD&!L6uOD1dm!1phJ{1qmO(H;yT&$;1^--t$Y(S#Blfgj?kX!iF*aw_wGdREx?ds zu7x_WqB~SIVid<)^Zx))^saRcie8iOsrVQp2TU5uHqfcH6qk*_xUA(?)Y}>9!E~-i zwxwA$-mr7g0z<_!IT$i|oSSd-#qw{L?0e=`}yDCpVAC(FZ(Pi)!qLMb3 z9hQG~+RWbTE&=)wB9&(wi%v5I~nr{xtbkhB6?G-C}4=YBkM`os~;WuSgVdx zMo>Q8i`2b)4IB1&XfWKq)Ab-e_reQTT^r|#!^9A-oqKZ%ajO$g-)z4cy((l_6I8{;B4zxKx z1}d%Qdake--0ElGq0b55w}>@0y0(ssd67tVKKZgNtHvYO=T>ebuY1+F<@xlc`w%;=#gq2icIMg}X*DLm!^xvvxa}iVFWB~U^UahRtI+@R zb7G7S;uA7|^qJp8EZ!%!ky{I6{uo&20H2?Wj#lJv;t4`GOD>-GwAAu%r({uevyS)K z3NL#iW1IO$rN3Se81`xQuRo8R1ZgR5-P)Jv2|ZV-0hGPAUy)uJx29V`XQV6htg%U7 zg=4n$LX;}aOd`8|csR|JLFQt_L!If$pUTR=(FFu4lr^HFN@hEPYfy6}wJ0u`@V>iS zXX_jUmcLXgdTyIO?rlN&tq9QE@?5;2uAi86_3-Bh+Wdj9M8`{nvrN!sTjZW7AaE2{ z0hBJ+Oy=gZGOGE?&r|ji-VM}L?&zUQ#Lmv9({q7TGhcgL+Vry>BB%Q!pi^|bkh105@ZW=9C$+WF^if+8aCWhs_5g6uTDt5jNdu=0a?UcsZ2 zcRo3#=H4^t<__Jmvcz#a=Y>^So;a}5#s2KdTkA=f=eHNAAoY%@z7}Q=DNL= zK6Twshh{EnPqfOOi)vViuC;tr)ASF8E2UQ1Kj?sS%odC2j{9kx9LZ+E9a-@&Q@I2E z1Y(fjv*m-TWQKbhaOo`sCiJaaetceZ>E2U&#QnV>-;8?w7DrDqUF zUk%R_1CAe^t#FLH;NX1)@6wawIO_g^E6^^!dQj&*x<*gk{K2W6o!ANeI^XruN)D2&QNE0?M;~2(Q%}B|GqE-%D%egoI(mVc`U)I5|0R!uFX6iTW4E$PROA<& zuI1_PD?Xnma^^v~$_75I-hW)uLtODp1_4$Gah;+NRf1~&fXt+eQ~h$gCiU0bketQB zgOItBbM{9143|;$o1fD(s4sqBUQ{DlW!$n0sdDFuu@*W;OF`wH0n|hg&*U#0*)5|@ z+{Zg+-wXT?@Ut>nYqj5hnHlh$m+LiaU6r?ujJq=LB7Q~COJllnkpaG;iXQsmg7i_Y z(ydh+%W{rZ6Lfo3!}Io^)n6%t0dOP2nyQhqkGoyT5m!fdD)r0mu7+e)A~IK8xBp5b zcWDzDF+A8x3nMgM;|QNVnHWjKI`*s1`_Ae!$F(e;>Em5QAkPinV+E1!(J;a7F-ml~ zQFZ$r$|OW>b#o|QcVrOKLNed=L$$K*R66sQj>AXXH+!k(qR{T5`qwor-@)~7=w@O7 zGW6)Y3f}KSQ%01^@>N&{zi-xd% zcZbIsU$%l@ed8!nd1*+UcU!UaIQt;*xDPTszTnuGM<3PO;NO>$nhZWPsq3v7v+4X+ zo`t_4H}{5SN{48KSgt2ctFAie`SZ6sPe_M)dj@$Te@NN|KCPgf(GYZjw=U8@{8dZp z6+&?V#9u3qVG#!b&0mwanq{KGhM8D%^^^Tia+MHoT;^4n$g*0B3J2r%(bMfW=z4#8 zJ%zj%7k|OIhf2aFVSwb@(W`(AL^U`F^NZ6;i`W-Uh!~2Z-(Ec23R_?AEpj4E_Ow@& zc^>L2*TdgNN92NzN)4R2=+}#LcJ`MavRz!IzmE}w)RiTkHZ>n=f_W-3KABtw3H`V+ z&hBC;GK#B1F(P?^Nq;DQX74(i1fyiK%T0tstk;Hf*KCU$Yo@sRdqiHx;>gKwYWFnz zn7LV04i*{FuLAe6t{=i=Nk>=dkKK7XsmD)`1fQMxXXyj2=Kf~}sJ@{@&^qzCTr$xD zG*jHY`AnY{i5!xm-z4c~NSw<=Ih5U>wkCFkuOss?TCJ}O=EIGlB%Ct{ChwA}MXLKY za;EJbO$VjL~td|QsI%yhJ8QBtkh|LW$eFjw~qpGEaBdp z^5bAH045Cv`1G?$a>&sUF4V#`+#b~4PD+aALM@7Wk7?~2`=kXW!1J8Y6dBFd33;xn z!RQ6Y~>2Hh#cSzx*6~D@vBNZzmA*z4yfPO1#HO1B~bEs(tOsrsQd1U zE#hf0YSOWmyIA$d$8_(*tcV5s7aN3*rP=x&Gd76D?YVQ*U6(K3z}FnFrInrhFA?!$?@gOx7Z~_A1ppi_NYAj^Mi6x(RKC5dI_(3Oe3FG zF%KD|zg}E#;mN7N0b}ii^S^ZL_IZDz8H~}nFg_loET3?IH z7@?gudmUIp*%wr!=*xEO^IqQ^RCG1#o|eA>}g-QtFJ6Y9&dQtWNA3Qli`w?m%tNiqz0LDTfl z!ZY9y>rr4kceG5B*?;kf_JB0N9mp+`pp}kWk zT~)7V5Hg2XML9KOqi=G5K#|1?fQ9#M8{l~!1$x$9qV%h7Tf|qO`XQb=!xiq9GseZL zV+3Z!ycV4}k5d1hrk?CIt!eW@HQ%irMXbDJIhhF40(pNjn1OPwN^X^-(7u>mmD z_Ny&hBFxY5>A;WwG~50jX*AdYJgSAq#Wi^zBg2SW>*0K>HB{V+{tVvwY)0?vi}oUZ z3TsDAr8V{JnJSfa#V$#1keri~vpA$J4l0}X9vKo|`C3`JPDf*YX>^uz@L51KzD@3G z+k{qg7pe$lj1mg2398u>QEiOba4I;{TO+kFHY!KcW{^7{CbwZC2Un0QYwXRca3AAY zUGM(m-w7Z zSv|nrnU||v)2fJMX8%MIl=`5W(|s8S|Hwcg9#GtbFwWJuXZ)!&1Yf?ZPH7XQGJS4> ztaKOhx;tiV=AJ_ToaR73jnCeCoGI+b*gzp-=R6C|`pBcD*7CD4;1aLtIWsjR& z#nX~W8dQ(|{+|4uZ@H?ABE& z{wA~4AlPd+XRWsh*O-BSD%Xt{zcmjh5_r4EV}vLOte~85d(D%u@iU}M^`s#@#vH@B zcQNjCnjCsoqwlcuF-N5m1F+>`-%VVC+@?&4jkuoFTxwC&Q+gda@x+qY+9bh*!1&~v zSl;PJ?8KHiaiav*T!lz9<@o<2>pGyRe*b?Ykwhtq%t&Qb%FIYagd*#5Q8KS%cJ)2K|2emgb5BRl=ks}<_jrx>=dP2izS{Fl z*M&;$R5=O0+s9xukP0w>pfGU5`Yhqhfd$*g?+?%NHF_K^Ue&8tdO4N)i#AU48%(c* z*0cup-QuNsGC$2CZ=2|MsZb`)4!3LlOI;`WiFGL)%W)(977d_hGgTOUIp2$}sytcO zC0JF|H^uW|{cPQ44%$;fTmWMJVELV9Jd*2`u>(3~gv`L+CDa!|Q#X(w;=rA)i1u*2 zENW{#!U3o6P=|40=;L|95oQHeDD6jsOwMOD<76kw%11o9@9vJpXPF|F%Qzo-9q$}R zJb{7ReJ1j!QYSm_W!t;7w(A%CL*l zkzeTE-Cz}Kc>uE^VPir|Dnd$iFJJ@obc`JNWgq+c@=?QEqPY6)K>=eXQOj41w1Pe%Y)$8BsvP`%(+MnJ4EtTkUVnhrgMD>l)Ic@MO2nCAQ z4d{lgh6R+#pn!YpbgG5H84n^@wB;j~p5iStxvWB7?7goDMk*MCeQf1QH1%hM=K+Jm z%BI4!AQFi<=y?0z^S6s9@MQyS2|L2el*BBid>Sw60RUah%nb+D+b%>D`8HkkbwjJq z1U0m=d$mvscLWE^KNNor12KASn0SDc^D`rrDJKb%cIZM|)%Z_LEeXJu{^6st0=j4& zSlj-hF~2JSU^zQVeBw_D4PaMy>k3__y(ZK2*q@TaZ?SA+G>vjW_O(Oilum`nREr?e zvVjb1r8%s5_mW-rLN~s4O5*KEhX!l!V7}R^`n93#X539piq(tUx<QbM9H-5%sEEl*mS&52L{ zzENsl=YFKvG9@O+7bVTum++svPUL|n@D)O_gc~47aaClLJ0frxPec%9$QrkQaoygf zSZm~KPE{_qel1mlAV$j+BH5aISOgy#YeT+?f_*fM8i=EAi0ZX)3ks(oDkk4B>Bn>p zth6Ga1`002*{txaLg-0tC~983?5@LLcgh2rQ;RE8@uC4!eHPjol{)NCho+>k^{@5}?ljoD)hK~#F0?wDDkwyFl(3YU1EgMe4EknPlJmq++KWiEBBTpZS+&J)$f zCi@{dmJRqcx-JOnH|zAniJxE)Uk|pYK`}J9N2~``28xr5u}Ii#3wBRLK%^Po`{e=I zyZ9Q3G9*Uc{!JpM%~5ts5{exTQp;+`C~0uQQ6b^m*WxgX%Y`omX;>9EnjLisq9H3LrTDtiy>qcSU zm3YlvXm_N7MQ;GLT3boU)=yz@;&g6O)6rxk<@lDl+N=uni}wO)ued>=gf;qbD`jLVJ zjC<=DXzDWC_R2|r;|2E?vg0ExvWk`Q*5uS%O|;dlu$4fA|AXVa(Jv|H!lqGo{!=c5`yK3+T7oo(cCB~v*u8Mrn?;gvQ zp1vw$L6h8u#1zY6;m5^Va-w#<`9-honV65>N@`N{R~HIev^qjC77*Hg=D90h?LX~L zGsj2T9DHo}3rl*|ya0?yyu9zypLPihHI9aiDE-J|`3W)FZ`O{)evZut6w~Wrh`pHO zS9K34u<;7i;VR(`6DQ6@CuD&23hs9w894>4=>Jk(G?FlBnl zJY4AbWhG>K01EY8DNHpYv}i0uu+q)n>vYo%xi6Xz1bHI#s;r~g7d}UO#bmeiiaOGY zuAhPv?E{&QvFkksuE4)(L*Ix9n=s=1EWiDQw#IAS<@@?G{Ied?Y6iuwn@zTB*@tAo zfB!LF<^y1Kl*(1VKh5Avp4(B8#Tt(Qt|Wx88qIg{5AwfKLhU%=hRAAo*NqTk(hS*~fv$Wkn0I0O`yl3E^v_?pxldAJCbPv9xo1r{U%b3@N{604=zr zUaF)$sovthhH9C9^*7QaOIkZ551}gMEQZx@_-Oe*y!Op>8Qrnv{b17>$uI8MVbOa0 zzYvks%6mo@zuXkcAk^or8v3j}mbq2NHc~}E==@^2)7qNf`*slK2d=MpO9Z+;aJh6g zQ}p`sD@MX2Ce!vB+ITZiUMe|KkSd7||9T3UAdvmjF|nMb{c6sqPn|xaF_Fa-f>Xyz zMadhWjP_mYPFW=EEI}d<-{LAM08pBt6%JZ(?b@ce68#vr+k-UD^MSs0nkTheyI z(phfcRVsMew^ZwqHkL`4jg9Tb_U7r3Lo#O>?j)WpH@OkjumFqp~N4Z7(#;s zrH*@I@{eZQPJD^xRx@1{uQ`8I0?wKJ2kA=6XSdGXnvvA2C`EYr@S%AFDG2A_iijE+K>A0AuE9GFS=`vh)6?h<20b7iVHc5^-_`KBA zmo?3a=|ZKI8c*>4S4vG*0`AC3rB;Gjg5@HK_wbs-T^d-R%X*}%(;$+gz~af?Yd;cu zZBSLB1WZE@jP20qQZ)460v;b7#;AyJ-!Z5NtpZzn6kc=TqxIylWU(Um4oEP$bHRP? zW!3VFvgY5-3`MoPz}<*GGe|hucrI!UE!rLvLjaiX5rbsWy-3-{3k$^Uz|`6**ae@I ztMP_v7hjUseJo{l#;O;pa>3v=5HT8~pwpXBrO&v%QX^p%G@H8au?AOP;2*UON=Oj} zb8PI(ZjRhcoxiZ;-L7Yxg_+KH+q>491nK)|i1lqxEp$q|N)XcmN9kn{bV=(YvItcC zd-6DL%ZSmVF0LHz&B*C!o7kXW*zhe*&L1VoAL}rQYgd-k2kN~^syY%Hidjm}ddrtt7iMoE8ar zK6vBJ9|D{Yl=;r{rm7l$zWS{4r8GsEX(!7#eQ3p(5hPO#LO*B-L>ULm#!C#jXrAsb zq{ZpCFjI{_w3N27)32QvUgl^sotysmH?qhPx2%N$x`TTl(*SLO&J*CdDIn*Vuqz<2 zU$-ucof^1?Y%`9GZr7~1*1AGLwG62swlY3mkC)6PU>$hjZb1m{78gpkyOkCEg4XHJ zsOmiZ3aHFJ)4*~VJcxbnO{ql;cdMB8Ft6KI?OQ={pp{LhA;+ba!#AO~}Qz$V2Zt3vxEwAc>uAfNXd z=GWTKJ*9M)X__cZ-0k8Vq~iqZv5MxysnCfO!-G(}fxH)QA@O~@Pi)orSj0uJ7Y6A1 zA<8UCLI=~imQg@*>hr4eCO@G%^7Lc|VAO$UBgdZy2)dL6aZR2?Tw7;I&eL#|cmAE} ztvNdPK3HFK={$K3LK$F@nxC`uvLR6|U{lVU4Iq_2c>jYIy;0PEViB6n^(1OD)xZmG zYEhsG(-u_Py7FqYiPULcx>D!clAXbMV`!S@!Sj(L_dRtD(#gUclGA^g8*VuO>OFLg zBeY-R>>Uz9+$(5nOHr3cDd1uE=XNchMciM`u@^2#6H;Vj8`@3H>NR}xM!I=}LW9Bv zE)E^w1x;mP#&v=cyM~y=+#GYC!;~JKCj4>i z35;<0mC5}iAlTYXtQ<#*l(F!O>!iJJe6LjuI&hs5KE8emfn`2%>Yq-HOw{W~iAg@Q zqn2hlI=Y*KudY(eeuqi2@?0@+e~>y&zeu^-HRhmi@}rn5JCe$6!&yt~GrhWf@_*cB zwt&ElkEN3(MS|%2MgzUsxB4EKfCZA3M{ld|} z;|vqBqYJG!PQKBaHRMk6!C{@H16l$o{-8>+1~>-WG8qfE5?Ke`f=Nd+O?qRx<$lA)V~* z+W;zygnfn?V72;M-j&g${%a!pUF^yCmFJgYqze2wtPq)vG}L(-UTsO&gH;o7Xt}d= zxbrOt3Y+W`C_0bUJvd=gRF3t)H8MR>4k~x1%O#Z9fE)*kVQlAVWh9}6bs!$?UbQ^> zWiUC~rZm%>vZ!E0VIv638lqI-XXm>%-h5JOBf_U_OQz7Lk=8RUcKf?D(+GMrN+gNLgDc%5 zi{qzki-h}}NIDKXAN-o{Zy1dNVYG6iD1+pWxwe(gDkv1&z_S=$ zhWm|3C!e4ux=WX?s9%Kwwjl(D2rx%U7MEqnW)`(95iwdu{gWtaVu{hAU1b8i=r1TC z&gYRPA21wyB7$P9*=0YG)o(euFjG6J>cUCu^8pU?46n(&2dc0J_gjw!sg^@{MA%RL z*Nj91ie^r;JVFX2111(v(NnG5f(<{Vn-?bTX;rgB(Wc+ZbJ(HWW0Va|5G>}hJME0R z>1Ws^XWu=n$}A+F+@Wq2%+&)7c#I?v4P7)7(hiTfAu}Kl?1MxxEkGHSx431C)DVGG z9`bb%3UlR%V&O`arw-hMMzneDG9CwBjBT35{LH`3{|yeYG zst$X0of6yqHO^?ZeBGp<%ISROsQ4r%8*ObzgT&QXC2;q{yY&0Kbtvp$+T0yW_K{m< zDaN8a8&ajJ_JBC2|~{}j)1GBt;FK6 z3e}fOcb*l4cwIm<735QzQILH5{$e#t;?5Ar0jW8u8fN!;5PgP-woOE(N*^Z0Hg{vI zn{is24a+V z>m;#2f=P~VffIe?fM%gnnZ@^L9-eQ=5$0Mr_twAz8!Y-r$G+Sb$yl}+Qdl-J)F~0G zDs1FM92;ram{@HY3;151-|}(Cn~#Nh8B{;xhwn1*as|A~fJC~cYJa~~@SLzg^{FXR z9XkjSgQ`_3su?)|na$yTY7O2n;}Uw>;oRAC3XilC`Dbz}Wf6l4*tCIxw82=By}6iy zlPf1P-%FNIXhzi!m|82fXi*sPy#EErB3IMg?yyS65K8Q2WJDlTMZNnb5IETh)1te? zjwbf|tnme!s3T%?ii(t0Hm>}*6iLEAcuae6*PrxS!*&Mmiqc4Yz&Yx)>^=7^?db>Z zkc5j|El7}6OYiY%0cx7I^!0LC<*5qV2?6A`==bGm`(Y!fF@%C!`z|V}Jb@wgVGRW_ z+O6t9+k%It@Sf){rCv|EtI|1H<=22 ztEqhvPrL2{vZra2VU2eBaOfDHk^Fx`Y^zV6_9^?@RLr*Z*X^I@-v~G(ZC2gFG4Q0| zo`hE9boxgg`?yl?Y-#@j@#yHZi*rlNWo)gRr^*quzZZt?GOcx~t%tu`1RK=atK_p> zQ%7VTZ8IirymgWlL0Y7Bb2T)h1yIhA>q_f6^Ng26*Y+X#i4W2@&eG48rbCiVhhBp2GqPtA5W3u$L$%R>2?DNsAASE?>>Jt6mJ|6!|v%M(OMuo}HyRrtmhPV^STs z4uDk$3;s;EPuiAKc2Ab6>~p+oHi(rZNcJSoZ+W2X(3@AG`ECY-u2_SH{d#SUCSVtf zKdfDhSFeGzs^52jJsae>R8|y%M#JN5(mF2omkaB;Z!=;|;0!Ur!76{Y>RYm4KM-7HH+-GAGv&&V3fYItA;Zf3erd?f2MW@jAe34s{NP52w4=L>pfS=^b8 zENWXO-(6tmz%eI^F{`jHRlbLyCjl1K^QY#@P0p{D-v4v_-=#~2qek|p8KRmHgJz2a#mG23Sc_&|5X*4RMo5r8}`m7$w!yc;#u);1`8N-uiU`$mVdxbpi@40 zTc?smgzzcqrR_1hjeUny6zlrxA^5pggN1|j@rdK#nNnlr3~B5v(Q&&C%A9@RRKZfo z$@Dz3? zE^9u7?`W=Qe<0T!zyw#qnD|Enfo^{)S0e9`Qe<&6z-QIZMIl?aRpRJg5z{X2l{Lg= z&qd#~26V5`OA9Yl&#!j5kO#&=ed^`p7ibi@5}Jnk@xq!N?Jo)q!#RD-Hoe7|Ax?Yx z9yD@8KJXMuWP6JWhl|wVBRQ2V7%1v^u_<>LrD9{(PvW60FoMII!|+h&gSGj=eK)5Z z0tySz2{yb)XYI=k0(wNETT>Nx&={rnY1@n|V0*MoDjR3up+tMeWBfbRX3n+Hb$xc0 z-1@+u0lYAFevvL#IlNdA@)o%0ntJ*C*6&Wl=i4`U53hP^6m!ECdm61#8ODQ?93HD-EjX6E@_pW>0paeXX?v0i1c0U01SYlR!DYvTj%U77}#hI`F z=;Zg4cv^Zafu_zRb{(Yok2y_J65m8xK5*0BivJdcCl0k#Lua;e=GxI50*NI(3-3w* zcf5uZE75;Lb`PczCyHYeJxWo<~6$<+p{3lN7fH>94e7lz*x!)xZ?{#QiIcO0t-S$mg^8&AzmC8Pn5P)mO zuqNWhluEyBJMRG2wRIeF1-0>++)L|>-R8L`8kXtnOfd`L?C|*5coEtKqx7GFr^>55 z`)YM5oq)ZEgN^M~ra~SbKi0hH#Y(BObefyy3dNr;St_r+uXS{M?nZjg=+ewn?(NI+m+=Z(7d9 z6s(~cU;DZX$46hlx;}yHy%@xD8^^+#UEnP{nq@X|7eX00NChE~Rs8DR?NJ6r~W4MtSXwC**zu5<0*|MZ3=xrQ6LqHi!o{d>I#sfYLWFplB(LdBTx>@6^T*+{u1opyk&G=i!F43yYrH(yJfc z>s{tn`P)W^pA{vy{!PnXbniRz@9+5Qjm4J|G?pbg`lC&t{nMao)0A&XTE-xyfqjqb ze%u)yLFX#(xEq!cc+7e>7~NDuJ-NB7qv`AGx%ne5Mu=%n_tK&WQLb-A#*;hFPe0OZ zFyYHlkCitjFM?Fq#;oytgMYwBJ8ww+{Kif!t8|v7gg}|CalU&FS7dGWSI@qQr-tRO z^Rk@N(oTaPw+oo%4|WPI<5z|L8ce9kltCZ;+cl@>%%47b=Em*8-)KiItjHErkT>GA z1>|}YrtEfyYPv(pe$RKa-FcXrO;QlL#A|D2kB}4-hzB){ng(-tJ~0T)Jb7yOx`xYi z5_7oS-qs8dJ!3b%+CeQZhNobxGXgjMhP-{A#hPuW5eKjHm_U8S09c4C+D-MQ;h#(G=u$_!?f zaU6a-fqI(&sgewFfh09z{HTcl(!D2Mr?hmFY3<;BxzeXz;<%#AMsD2+oPJQu<+r&B zX5r~n=tiz|hRcs`f5GSwKnAHNEH}j=c0p2vR`S0>}o+(bMnU)X$(_ z^!D?NI-{OyW6+I_4;~HbM?~O`&kxMk7aMSRw!1|_! zhB3u0@46bAEYF3}1@$%+(oZGauv~{55rhUn(SX;VF6zWz6SQoK62q>uKi3{2wG~n8 zyiO(I@w}l3u|60)YvRPv%Gp?X98_pNmbA>{Mjma7ICKdR z`?$uwMVXtggz3#i`2)94lmUh<%9rI~%IsspCS&o^gug7eRYY~;8kg-$iB^`T7R)qT zFdE8&&2@Y_;VE>GoRP}0zIVkG*QW?u4M?0RNVaF+vnt9EEUhQJN-6p^jnQrizXV1w zm3>jolnm=XrH1vN0y53HJ+*BJ-OI-pZ_-h!doIB;@_W)J{fk~M$G4J#Ht&u(LSYK* z^d@B4)Ay8z8jxYPw`)HR&77uEZSwQ~J9>$l?49IqiFwjwS{=glF={d*F9SMlQbHO-KeYtaS@K87UnJSyihaz^ZI>!}PjUw^x74iZX zw;Y{nR%HQf@K|y(8(r!fI;_ZaeiYYK)7xI)L8-W5IF|M|GL4-7GWhzx4*YI1lFY#= zQ;YecDkR^vm61wmR8C{~`}bo-^_68$(VL@yXmXM%KXOr=we4lpV6KG`_yCqM;!ha< zrnJ;6N)&@otj%jvUHO_7;a*B0&6)N|v0uAJ3CEWj_L`EYZn%x4W+@Oq{Co32n^IIk z87(XG@K4vxGDMQCFb%evuH%+bnyym=uL;ul({v9N+1+{>F>8-f?QZEMdr&H;SlJ7N(`^w*{`7J#Tr9 zuxWpR6oqnY3$_57<%g8<@zc;3CtlqfFbZ?E1woZ@wk|P2c2nfIgg^4;BIP9N8Zq8Q~HlGatGWCU{)(O zzZnv1@9;?DElnllek(Gvw9PeNjn}M22d3Cc5Os{-q)s(mikY~V9u)$7squy?t{-8y zV2MjXQ1eg5!97hB4GTZYW+H zj})`}a)JeUe|3BjP(??jDZ~XCVAGl){Y+@)uu7{UC0&u{dkNRSLboL}RhR*) z1o9eK*7}g|qu=X{Vz@Y42B%)g#K!gPRv_DYY575ohDWzpsCB${`fq4yhtBMina2aP zL71O|x5Vdj-LQFyuZEI9qk>4aJL(B(?H9*j!`-H}MCNkqb7%fD(1;p8`)PbjI)my@ zEaH|MOF7yL=>ECl%{x&LW{`5taIhKB`p}$3NbEJVcm1z0WE)S`QV4&QW%| zV?1`o@A1GtM?y-eralv@8#3`x1Ln096Kbaw6Sdf^juap2?0mlTJ>`*ZXCyK=!QvwB zBeg}h?D8-RC@?Nq^d^h{y+O8kO6(-9-=Bt5a))A3qWebTPVUK<(C4tud|;^ur23Mb z$7L{xaBpvGi>hfL)4{k^XB6l`VkP@Fm4p6ujh8&EDHGO>iRI1;P&KsACq;_n-gx?{ z-^4LcO*C39-ND24w`*91L)sJF-VT8{KkY{Oz%a}G(aN0VJB9_uKLiMo{RGhwP(1JT zg=PIgY9%j|x$Y-vOj~IwT<&>2b4zpkJ^Gg8?1DLGE7(slyX;X{UwY*rtAw)*<{%m^ zhWF!DMmjj^uBGtaWc~Svd)^_wgPHU_cm^|knVEg7K;Z# z_vMoN(wPBt`#Y)PmpE{>j0f3?#SipV&uuxIW3o z>E5o6%ztC0o%IK6{2z_n)hK7bE%coOK>Rz{t&_=)MN=sj`E; z-jg{IUYwKKFh4!@Osnv!_$YU?G3g;;768CPShc7;E{Oo!0@GgOrqb`_T-e_s6L`fj z80s->EZaoLyPm`;DJg5SdZIUQN&+Omd6Vo-VLuF@d=}88m&*a%jC_J(uj_QeiX(dg z5sXnO20kK35grFVv#jAVG*f*SC8GFxD*jH^KCY=x14g+Xi(pld`UbRkyA5ADs5Bze z;&+~LXCrBmvdxem#Dl)5D*tk^>O;_VMuO7WX@rO{OB+C>_{nu6Zs-zbuK_Y;p&NMQTD8Vw6^~J6+c5CO?IrVAuqK zh@k||U{@;LFia5zve^%Et9X5l>Tr~Lc?6{hwm|i*`=6R2hZbnmFX0AFJgB1P3Vs+t z+xza!AR^#GPq$Li`PG*2Lq0)Q7)DdJI5)mBVV{iMFHsojH<`M+l2Au*S)G05}ZYgCK^HG`p|bqxCtnf6(g9PxB8 zDk0PE>@e^yH$;lH6+ntNse($%7XO{bq5qQDf6oyCShIkl-fUzVG7k*_zLf2TCh{a< z^?f_GtvXcLCAf|J>|_)MnA*z$u&?BJ26z2@Kj=!t*I|P{Ausv=Yk+wUR0+wX*#A6a z+FG}#Cv_lWN<{ZsexVypGJ+8A$*dzc!yoI6x6Fl#)At9L z`YanGfLV{>`;z&0AwmZP-qHe+W&@K@FCl5}QfMlK;U|`sYWCef`d~E^OLAc%`L+G? znCzQRKthP)b;87drAq^NM+SU30=P|$0q=-P)mYfDf27T9u;@IlD9H%s$X+78BhqHB z`?%sFu)lbb78taU+$`!7;jH)5%RR4AM>YTZ1f7{^Ja0(*3segzwT)Z1&(rtnDk8u1 z{X3^_h1CDdYJMQyyP77#E&Kn5lred)<*J9fEtX7?CyirshQ4!EpZUu1)9g#^EszT0 z^&OE8#fuS=eW!(!M(=;UOr3F^=3#`T2csZQx%b=v&mN{fAToK4JlLCXP!4)L)-m^V?QLVtfP`H^8b{JSBchAF7_y zB?*j>JK0!)G<_k*(g(wDnHd@F>tCX1%V_5WOj>^Lq4S^hq3$mXr5UV(F@Qj0+^-oa zz+`q*lea;6j!*Yt?S30=fT8J546vM*(znfg9lY!Q;G3cqx~>#98O@XnB0Y~;J5180 zs=gL;j& zF&|vA2g-wfstbT*(RU|CbJfr z9o+WFh_WFq+U%Kjnk~|yDYtt2C4`*_SNy%c7-=2)pP|6ip98@-)&HuFKTFj}%(Y@=O7 z&A~>?e7+|=(CYUX+)9?o@J5ECYH|>;8q@Q`ISLHx{eh*suS9ln;}6{OAbY0|a8*e3 zA--o@8l;$xQ5$$QT!Ay7zf2$5eQ9ZJzf;6%Tr+-sG*Kc4i-!Gx<;7<_bc}368?H_R zs1o@V={AE+UH)i{t8m8oO{us=(e!|c86gz*s1a7+!YQmIX*i`>19?~NvSPCfW@NI* ztin%b|2m#B?DZQz2|IW83H(p!)IU$c9S%kV`s~k68qd+%p5HfiX}NqbQ3N!Ikuw~k z2-~O`PONhP{|xK*n#GId&ZqhODwOhIthqjw^oNie13AO@M9078lv@u;Q}|VFXJ{K~ z46s}^{O*(vL_~dlEiu!Zw!dr^Uws&@t1ngm30Jb^Ir=fVa03s_gQHgwSHR;)1<@3Vj{0{T`6n5O!bn_uW&M#rbyH@ zE&9|RvV1AP7v#Y4Y#$_AWzY!Al-N{BECc4r$yQf90riKSCF@Q6%ewz5r#(GkK+XWg zncTSi=POWk0QhXbjw(o$=;^nJ;g;#Z-a9t|je?D1Po-asTqxuDP~=i~1Jc}zL%bZW zL)i!WlvSbAs#1|6@J(}Rn zdDcwuZ*?OhnwVH2f zN3#@Z<0~(mEefvC;%46$l+Gidjeu*VODunBnSACHz=rNhjq*V60`{%+XHOVyq>Sz#EykaIquuAU zt#^S$&dSrpQYD!b=?B+}+Eu~a|3P*K z1I{2uOKql5ZYsEcvRuH|lI`mcPmpX@t`M<$T|E8XiSM^WyKRDtwe^bF>^?7<&d8C| z(TalcTz+5deN$!z)7H0lnEb75CG;nq)|e}4Q$~k7%eTrt zUF-bh`;b5C6eAH4^@%?pC{0Zm2UJhoj)_F_#6KU&EE4vp(L}2kT`s&_X^G;hvMxgO zbR0HhAD;?+^S#EJ+Qzd+t-t*ei4 z^bxqOtdsfO1?1U$%IO{zj$Em~*(rG$K_;xamnFr_<;!NmVL`|g80vedVqvw1&r$p1V3cILpv5DWY6+D7GM8Xc6&Yh3?R}>?nH{P}{e(^Z`bqC{8Ao`9+R#r8% zVY6jrR5DEnK5GG04ShU%v~g~4t$ie|jR!90a-P&MD;NStCfaWD9whyUIcmNT8UF`vlGn#};4gsuawA7M4>XW+9AgdR^<`G<7*pQG{iL_w?1RQ^%Xmi|pEo zRe@UG8@?wAmpsPo1+Hp-@~A3G+(Y(0v{bykI_#J2hHt>xz2!t`nzYV&-E*stL@rWO z6UV*MAivF4+e(Mz!hPk4<|QlVUYNOI-N&(IRQNl#c_l^qG|~CX63%t9LQ+^GQyeGNNOGlDjq>ADT-^<0VbPM&H&^qF zgT4gUw=L)xCgNs~oIL9}KN)vNDKNCue(WE_-seQ_Ck(k63XAQTh4Iz(4d0mv?-Vpw z6*lHrF&}OGdcm|;MR(oht%1OP+QM?PS#BNq^^=vjP`1r&j<8g3wUG?0!FCi)o+K5m zejMH3YqOI(F@ZS8M_+c9@aU-NiE%LJxyERsy&vV7FL-sFYMAHPoG7YSJ9&EAb{&qk zkw(tT@d5?cf_1;#k^QxM^(uU(xuwMF+|zUYFHmE%r=YZNIE?ik2l&Gtq}r;ryrz-X zjd!(aURRmpV(ypIbDR63Y77Z==h&in$R$exE?gb;xJ5L^pMsdG#32m$PaMXF8wtS!>q~Vqq@RW8xms7cu z!cWAzm!yq?a4GnMrwR_9z4Len{P^H?P=C%-?DW{L@=b8h3KVz4(ub0z>xb87vm5rh zjxI-JA~?4aWzN;s0kJS{x`AkYDyQ5!OY9pQcMHWx)b>eb!p3%DDU}f`7f{;0Vv9$@( zxA)sh?-BW*36)onUVrk!`IanaYCiRK?_6#UiCh;b&-7;N*2Tj(g>w(XYcA~E_)v^A z|9L2&ZI%2+b}S7pbFsl;^6=t$)g}f6Mx8|5nSoK$yoR^yZ~r}>|NKqj2Ttvn)lV&u;s4DjY~eLo4P|qbC{|k(<1dGxAwk zuf&!ydY6AvxzrlQdN4KLeHd4ZHTd~S>ed;uxlce=aADZS^v-Yz;@QEuQ!Zx%<5M=q z3hg)QGn|ZUNXo6_nExJbNl8bNeO&u9$2u9mt|+n0zt4#^yutECfp~VOQ1tHjvj`I%L4!93Z|K}+ zSHoJCb4jjzkb^(D8%jnxsA1T>mC4=QBB!%e8Yv=)*}X+kl6WxgmFhr=!ByFuf%wr1 ztwvsoW*^P1iZlw6JDNCYxhpm=S=FK^-u2`c)L;VG5{E5kbB6Bg=@6HNma9+n-q2?;_ zWSHs(wM2-E>!{XV&HYOoprDXrI!l`M9yyz{c{Q*_&0Hi=qF;&63V!aBj8R^_&$_qd zR~l+-rR0EA)W*l|s<52KEH+vj$DCi|aaoscNeMo?ijGX{7NGST^>SNFmdhOoub#tw zsJz+v^HnNe7&WmPz1G>oncSd`;+5)f_5e&&=&!97Hu*Y>Q95U#>nh~t*^ST6qh1A@ z3;4|3`CM{ue_MfagS$NKz38bkge>#V6>$4Wnk+-2y00z%C#;EFW1(90*m%Kn;H-RY zHj^i1&>>vqm6Vlb;a!!92za5UVB)xI1>e<$FJiJH-zBor+TkD z2dt;MzgzE>Z>OG9c(aWe1n6 z=sa?AnEl`(sBk;PiTo#`$XPt}AO6PY5tw|}{&JC~x5b!AA*hwy6S-8BL&74*);qR( z?QpJ(DyPM(ildibiMs2hSQc_u1;a9`#@(np?h-|ZSpQ0UP-bL6q&AiQt|&|Ks_+q6 zde0mGrBCNg8B{h*&i-91TPTPk?<=w|UnD#gNg?u&5XOz)A1D`CM-&GGguF9bb1MQy zf@GLBEHhhQY!()4bdUSGJ8x>H7nt~jV0mN~PZN({V>z#_x7Red(Vp1jf7jD!>ilKws5pkj%_S1gslLVCRF&*c5-Y4c4ddCdJW zQljXo4n~dePNO!N%K@+SSv9ixQ{4}PZ0+K%+2~1($haum)j@czvL6))ozwDYMn`BBl?g?dY5N@rD@rrP#=K?=>)gINJ==Cl&~KVG!L`s$4W+!u zf8Gg8BY5>IGjXbL6LBc%-S7QGy#div{K^@-zhGsrB~tl) z6#M;FDAhn;XmmiMw%)+F5d6Y>f7a>8d4(c=4%xp_Cl`B6l)p-qUZ?=wlsT#^fuli@CnhOmP9yAFJxOm?j<`WHmq zyQs`$l3Mv-o^sc`&iYcvb;4~uK_<;i^uD;$@zM!-sYT1j&YsCLR#sQX|^feImckAdg z5jVt=3Ue;8xE=Y^gVQa(G@>gFqQuMqe@Sd>nowDVh=!W!y!rO!N@$Y)t%2qGClgOWdOP+RVJJ^VX7p^IOIz)P`8T z=U1uzQcMG`o;H&&b6QwrA3yYz++hVQUzZ9=$ zgz|SUTXp%*JQQjCRO&~+_pQq9o%FsWh>c{tMEL>|Hfeg~SE~Twb2p61rE;LJ?%Gy~ zIErISt>V&+J~;1}X77u8L==m!vI2;SS@ML4O8X$%F^^x(R{3y7WKY~Xclh+FR>l35 zQG_LP8r?bjjut=Bdtoi=6x*pIG(QrK7Ycsxx(*t?4ry!g4tpg!%X5|ST7=1^K$|ML^WoBYLN-R8U#o9`U-+Fh zZ&of{Z(_+e!0v*hz61H=RhDPA4sz2+Tm40gYE4>)N8(Q8@n%<&(?`uxqjn*zeTxNm zCOpYFSVl#;C{#fi?R&wzFTyZn$^b1j&s4NF3iG=F|7j`>lYwogLMGoH{w)(gYVSa+ zHp`_4t{ztjCXtdB&TkNVw8Yms5&uWkRYpY>t!64Jz?^uN zO5-OwQ^hN%vQ@tt!p%84m=Sb4Hp1F86k19M$FCa)r_DDklG211xJI4x|QMkVNORGJc%l$(ik?-gV0ZA${3}~X5y7Dly zLYiW#m`RT6@@`l%jJvvoXh%7M-r(3LIzaV6s;cjW2khU%TO0(uhwx&`4&a6Umq_`= z>Lo1&15pnpSn^dY!g;o#H>)$6VFn}iL%W(Sq1fQ*S@`%g`K8%U$VkM>mf650ol4K) z_axx7&DCn^%=-3bte0=Mxt;nmqnq%_oc?R;o(`yNYwvgk#g!gXdACkcZ(PI6^92k!H z4Z=Ok_*H_+5`sO&@itUBCP=&Bz7Kbug-kVNx|U*ubfr0=*t%^LOY5b(Tonq<(Xzow zX!$FEg)&8@cZGw&qxcYi?|Fl6UE%!sc~3iBG64NjoG0@G77<&bV`E<#_BOPe zOCoi8FfkAZ2P)4&sikN7I#EveTLaw!oL^1N)g8i$a5047_C^w%-ChxLB9D1v^Mj)1~z%zXOS;TrO~hd=2(j-89-9( zpduuo;AnF^fweC!PT8LM=J1;Nm86de-3PdGMNE@-8y(uv@O-#g+G-&bCrO`nQR9u~ zc`}PHH}eA1)P;%j_D9~mNwXDA zHhZnw>x#o#$d+?$NiRTRhFEzf6S^uDMG(=1C8fC8Halm}D)G_*y=*kbw!t)%XmL^+ zFpFHQ9eyH=h}7j~TK9CXp>z7p<=}D35-?&teh#vBZV_J*#<^=O(o1v1My?F6P%Ug+ zBNEujGVYSJK3@Eh08n%CLd4SqqZ$Y0a<^^=1BxlZKLs9NBJiIup(o3ht6EZlaDT8c z$rHGRk23H)XGU~l{9Ei-j-)SOGuT|$_t;e0wIRpjmO4>cZ6m;`-!Hu8dd|m07M7`9 zMXF;7TmVhr=c}^F)~8@fMy_YY!$fxE>SEi%iRe52ebwf4ym~Agadx$xSA@^y>g`&2 ze*?!GlPq!1;T<;I+!;O=iSU$*kellbaceH) zRJkZvtjvD0UHg)H_NCo#bo5SJhHo&F)@{pu6Nj zC+5|QhBCWICBD;wQ`FAiMC}8?u)~VvlY3MPD%cD^+c_Se2Y@m-q)(0<(K(zS2}}}B zq&XLn*NwK7N^4p{I51!I9bJ#)zvZ*60|L;@kBJ^lHKP%Ln3Y6%-_*Cpf-CTlMp7Bf ztzELI$FN@$D9uk~t*mvEh72;+yxc{F(IEL1;*8&uSfio4CPnNli+tNl)@foeCjc;V z`4a4(IfUrv+hO(r@Uj)dG&YK9T}i@J%!I4C(Fs)X%3(2r*t_ux11fW@2GGKqGgVG7 z&kvwn;%xi8N2=Xp?Nk|kZ#OF06eHn3v7|ru@0=uhDD7KXX#XNuL@)Jfthr6S1 zj~~Zoj(}o4Vhn(zXhUEL6FUp%w#szsZ-vE%HE!1vnL{MF>W7kkecD zUd$R>m#79K-JkAcbiorv7ogpUQnY)iXT@w#D7jcYkLpLPzI<&YBg=T`<2%(%3?7ve z8+w*^b zzHU#0%w`;Y70wv13d+6OERd0$E7s!@_3MJ_BoF9WA!Kc~g{u&G$E;HpxSols5kYIS z!`0+$xBO!1hHJ|gepE$>Kwmd61UTd-WLscjrG!g}R~?Pu6@+W$JN>L2Ofh0ak#z4t z+yA~S8eX#&H_XJGCkD{1Fs@es`hZqN2sfaHx&lc&-@nAeB%-c4{uV|15pn^5O!xr> z54)~0Tm1o&UKB{Xg#%gF*4iSdH%F)8*P^2+qlHEj)hGqDdc{~ST@7hkZF4U))S^YF zO^u+W=jZq3FTQ^k`nJvYmLSZ4cF6g(K-)fn)&Js zFze1=dZx9*lcd`dlL1~bPF}fd;vOL>pz^au zlQABb*rxCaT0j+Jt0Pc3mKCHH?UR>9bo0&EM2&>6RE;yABK+A~0fv&K5SZOQ*nD`; z{=mB-PA@k=EAvv|x;-#w3{%{>eW`XI-{u)xu&NUo9udtyp0%aikJAfUS6$T<2L+s} zO4}d|P?#u;R~&3`cqsiMK6l=PRJ^iXRTxl;UyNbQq9iH%I>?bd`QS(G7EL_CmdZQI z3I6O%d+acCNl7x7pmYY9ZljDrLc9vWhO^fTeO@3*9+y-V74#7%Xi~qR6(jV=;?wot zhxg_T!OuS4hxMPv@Oes!@LzFh^AI1z5|GVKkxX|YP&1N#rFX%8N*LTH5`J(G6CBIX zC7mvaV;V4Zu5J-bL0lB=eart+ON*U6>yvMlqWJ(y-ejr%MbL)^=IEx&vLbGD^3fX1 zU(|mV3oui>d@0fH?%$B%F(Ewr11QmGfMoo;gcqm1xcDGLaZSWW4L9doQkRyO)|VzcbNHd*PJ?HE$-3tPJ{sB+AJ|D-8h0Sb4r{s zCgLq+mGq73;y&`S8C(JOQH-Wd$!?LmxY>!J3r{u7t|hZOt&IVh2oF7Bzbep-tlYFp z%1g5m$vnB+7zweJC)bq63s_QwSzG-2Kk);W}KNu%#MG9lDMJ+bI&N1RRiVW?o z&ve6-2=5^*+ZlHkWZtc3^zgb^5f3yQk+yZ9n=4M6M@svFR*iDCYReFAkZqEceARCJ z8VE&JErA=fr0Y07BrsO>$rL)>Ed)ss2T&02VP`@5%x1JSH_?&EHf55U|o$v6+ zwqx56imZE2R4$wRW~dXU*ta&{RymSh1kQ<}$iS`-ooO8$vg(QT)aiSpW3to6bQ`WH z97bNoSeIb2h{$ROm2ob2_z8iRi~#7D+^QwDWyFs9jY~#0l{gj-Z;`RV{SP+4lxhWu92XpXO2g>g=8rq$uqGT zMpc&Qv?hk8C~|cy1@XOtmW(C!3>}xcW3ziZ4*Ss)>~XvFQ@K}xmb#UU zBrH1!O8?r{0lDtDurbsQxtLm9A>9{|EJ};)_I<9`SBsj>QZ$=5c@p$G9{Sy$pU&ux zZdn6`7e_*Fei(tqbOvSe)hG^3_WAJD6`@?rh54%k@8U(e)_k?9)!5qNa5xYWUoZ_tKkZt^h8ext(@X6*J&9Xw=3$#s532Dy0EIIl2Vp>T%(vLSUu|8c!PKprU}v zB?tUfU5-KVduDA8(&_*ZE4b=Wh*>Lt!NN1$rchhlrB$XGjR-A!AMGAFtOykIk602Gk0IbM1872~5~BVN zfx}ZD_|EJdhP@qJ6w+!TI9JUn&1CY_4mL!W+SUdM7I`u(OQuF!LHJ)75pq{#vtxr! z+v_KX0TGdt2)^#IqzAYdG&4YW7Lp;@d&E1!FT6MgVlJo-v_)n){I`L5nw?HSX7quR zz;|_nOMNjXzy4aUUMn$47~3V$n2jl}$1pCT&*dei#6my^D9lq)WbUq2jgO!Bs(^Sj z($2JgElV9t*(tuQegN?gkmDo{!#8GOa-d$g_}n5|J$sfj_5no-%M$%pi#SGsaUP#* ze)NOBNL*_LA9d`cOwt1&^mk+nloV|=PH<(kvT}Yo(#6Zsp{*r`^RKvMa_}^^TEbjs z+;Y7NBbwGUqYlnKUjBea#uLOXa)It@P6KJYs^dmqyd6ZW3TGk|af2hjIEz`R5y%o6 z(mog>`**!CV19BgSS$F_^f(xspFd4e^Zjmzp04D`jS?@I0NhzdR)ik@)bv&vvU=d;eXp$iCWL>w)1Hy1%+PkS`0P7B!0x8(aF;Rog) zdv7naWHbwUZ+K*;LYx9BjUl<%W!1G(aI?(Yc!{4z#imLIMhHWjg8(2a!+x^J^1rC& zw;-@cn8rT5_*=^Kk&@I>-`_Nj+rJc(^*C^%k^t$+S4y=ExaiEgQ1iIa;Pkd9%Bi)@ z0Y#f(J$+cch}OymIx%<1_|BDwla@dUaiQ9>#G8&cY@Cvcet3D6BoW1nP4U&%>H^b{ z*R&VQXkyYK8ia3uz^s^7pYzxnk-VB8Is;3&=y@U9G(~F&&tLg7u6FwQUaxwoM#iSk(lnJ;Rmd^4hKJNKlInMn2yhns%fv@4(z&nvZSX0Q5 zb$w9VlK_2JF*p<(qyc`(FEz-))TtE_|dt>$uBCemll>+D}x7_ofk=2It}u2stKcmtjmY;7#~V=XQ|mDwKk#? zE4@$jiZzEA^wQLbhPJ0agW1@ko6c~ZMhH``NrN=Mm^ZpqXQX&uw(+VU+m38r5t?ry z6yr0|DpUVTVKmXo&S~-Z5UY z(j>e2N52UGVJf57A0JkS!b7W8Eg|%~{?vJAAi-_5`%|2&9w#DXh#Hrxk9ymDiJ7D5 zbLhuu4O#}emyl~$Q*mCQNo$(Jf_lKubj!&bjuI_CMrdy5ju+mFhRJfI%1Moag4#UA zf3a75oY!nCk0NG~!i_)U-YG*vfH~gv~I_a=%&|{T~!2X&G=8P(Wu5=cp2Cw_NaKwu4Q@3R1Woh8bbefCy@b$ zqrk8Clk6j%@15{Tn6ev|qk`r`jac}`0kTtZ2A5-WzWaCUiE|&fSQ9=(IC6~(ckOxb zhT0D+(R2(akzWkDHfqP)I0ms%FZbPDNH!8dn@ra}95H&DOc1KQo(c7IeIcmpGf~qK ztDL^KQn?8e?ru&x5K$XXVNA*WzW8W?{t#AWLIQ#_o;vP`=20Oic}N}KvSn_erY7gL zihYqfh?sXABlMQ8e?*wC99iBxQNW8stD859&;j*;3e2{ql7(CH@WnsMBIA15-yfORpr57Tp z{35UP{h0xS#~Lg?PylK0i>XA%2;SQ-vT*{X7N1*na4;Jv$C&@A`ELWssl6p`AFT~K zNVId*C`(HFoI8%w35O&jRe#@XUr+Ef@$f(e&oi;FRgN!9#bAtc>F@uGlCmU($b6VZHIKvrsb8!LJfDa2}3WRh0B)H4!D*m$l|# zp{dFJpWXPr)qXA~X$uNE<@!lE7awSIYW>jxwLKwF%0Z=(=|`h)c`;{RG!OlR;}IJK zbYH0UD#MWq-O~D({VU>Cw>IFK$F}q1jl&dTuxR-8Jb|^Z#>6X*zo*KlC&RlrJS6`R zN(!>Nu2%%N;dU?fOSaOO7OH~t{b5pstRIZNt_`N2?yj>~-?D!*i`0R6@zLkWvqkFn zYHjoJU@%=03O;yG)t}3l3RW%0<8w!6$JEu-MeYyz&$z>DO#6*qy!O3@bzMjVl5G3c zP4nY3^#ScL{FA-pG$35yz zJF_LPww;J`yq1naJ*{?wX|7fm??01YRcDd}*s_dv>6OVhfc<=mWVcs*4ZBqshbwvL zhi0aiiQwU)rTBNlte3@SQMHuP5?P|%8rZL;D)*GEt4HLLd1ZVL&wXC~L$-N!Q2{E< zaS5~Nqe=G{td~zP`qk;h#ppv7q0g?W@R{Zi9q@0lZtC!c2^fDb64S9v=Mm!qbB1Gc zVWH{59KV^|iY3qKmsFx3;M|KSrfBAMl5GaHdzsxId*(Bi4`aH8uF zxly&IlUI#4tJ0+j2@am!F$@k?^<^E-{;U*=S`Eq-8tjKGl$4GWN6t@SkkQ52I)~k+ zs_%m7dZ@{KFN-z7ML(JW3j)Ypyav1cGA< zNlnswuL?5D`R=dYi(%Eq@B5y3)6Kd$k@q5Pr>iThh8h<$6tUNOuKsJ0%P5nlU7Tje zZ!QDWje3zd8UA#p#lpQT=#D#Pr&5IUX`JRHt@-G*G0H%}o(crZ`c%dwI&l`36wTUj z6=K?b+A_%}iF#|rjS*1_#x~14?&s1brBSC!bG_ z#@8Uo&gE@@cVwi#8n?{B=Fqdi0)^_<=VV6GN>eQMUfbR!ZOHV`e)<&6_XA98J!a5{ z53h*+EGGU_w9p&pj~4+#m0yd3`PxBNF4`ZC?TqYXN97h8;`bYEC>ot}zjzJ`TIo-k zoPG*E43n)8->EF)FYt!om>5rp82FxsCm5z#d)&#+G&OOLeA^!VO+Mqujtoy++Y}C{ z0=!B3`vT!R6YGd2pV(^TG+b^mJ~m&`F7&*zn$pNEJk#M?z&<6Nuc`0ZU_%+d_$rNs zQRe78C#$wMB_2KY;;QxMp!|L6IeopMZQvMWv5p{UCwRa?WV;UJsK4S~Q*eHnzSL5R zKGlIqv9CLP4k*C_xF;6`Tbaas&;pCbpKQ8sA8o)7-dR(ign8%TG6a4c7zT8Xu_3vX zn*nXZ3@VrcVO{tP@yPN17@PvvbeGU%?jb=ypauD*3l8&onQla47!8UI)rri|)Sq(^ zdlO}68Y>B?yt{SPLFKWt7r&j)JX(~c6^oP1%%d0<2K04FX8fomOSHPwX{Gf{iXj#J z<4r&$fu)@3(QWrpvUSSnll%2VY#~^VlbWWL;~quZ;&%HPyW+03uQnKu;OQh5u9VLA z(dLGo5(J3W-R)h)X=jawsY#!wLh?Xt;W!rGxOnw2FO!Ln3vP8NI?<=$3=TW;ykaQ5Bnt>oHGCl_w{ovLckrEIsdZxp=M}@!vTF zF*v+hKz-_Enxo&bK4s`scsx?RS7tG39wyF;PATksUeo2GXxmHE8ZNVU%cgS6WBfe? zx7Eck&2V~Nx6=eCM<%kPGvEU>IYv5xBb<-#=Y>`_vV-8U(ftWo=bLGLQvL=g<&Or> z^aMw1It_!8_C@~{ldTd@T}))&v(qlRAPYh*m9Z;U8{JbMSOVix0nK3OGeS@7S#p3Bw`mU))eeX)Y2H$@PFwc{XA3MSI zEW>I4>V5-=J+|jlQ?Kd{vT1qcNO=y&OlQmxb~joKm^Ow|X9 z2ZX$=seW@?>th5g@k%C{8Oc=jflT9;(1wq^DU2?wqiX@U#)yurar20xPbO;_L0-Zvk6V zhU&8iA28s?1UkkS&~EZ@`FC49-)ES}UsevkGCqLFSdX3N-h8)C+J9-XI>{{*^joH4 zH#lni2yP>H+?w37uvOg0nHBkYGE7mNvVBpm@Cnv?UE&L|-vMcvg?f^G)QGMFEiQG$ zX9@9Ov2&*6Z_sjfVsA2ft0vpZ2!Jy<=-&_6R|<_2Ui|Ka{4#Jx+!nHLj9vryx30t} zW3cS{j}T_Gqhz=y)`zkZ%~|CgZ|-ly9eJM}6Gm#+1$Ve=c5zPCQcKM@b>}zwj@rSS zT@pEN48Q9`tP9s+_n{h2g-hXA6eIvrA|b_quN0q-Mew|S?A;rI=7rH*${BbTTE5?H zLok143ZTP(Qbp9W@A>to7XaKvce<*mpXcKu{KOxM^ul(a`PrR%FY3-f?&cKWGld(M zh4|?Evwmpmb5{|g+y#o_UNKJbK4o{rk@0Lcd(@MJ1?rQ*9UXXTEE&8kpGktXy+w z?LY>0b|T}q9RFQ1BQ#28(rO@w(A>7m-~}{z8BpS`umh@(*{N~PA6!;{Kfma_*7$~x zswfB^Rf8um3jDeTKLh|ABG$Y_;9H}mDh(*QqSJXu0>rgbw2O5a!sAU#5>aY0={;r| zKA#t=Rq;94d5YjPa^@+9WNomDRt=dzkP8%cQim{CcAB@>x2p z&{)F63n+NONi02VBjU!e8{ua>J&VlX;=*a3&aLY2YM8bI9}nb)z{#Z=Ew%}z9Za65qql==nw8gKcl|go#_jCx=Zx&UfL?%Cw->!{=1yZ=#aZ@rYoV^)# z0NX3P+iLBHL%e?=QVSKZ9zA~eeEDZR+Q48|(*WuqrR(mQQJt0DQ?Zq31w${xqc1S0 zRyYCe`LJC4t4V;mQoX-@5}v>BA6yVf)+9day;$i@SFgJwsU6O(i|l$l7v2lhGEl#t z?CCe7lf0cf#X_DT%dhJsmr{#lM6M(gpos%R0xN;J0rOasdV%5E*bZE(2P5jdPKxHsDk3ukZ6(r$1r=;Z`e?7z zK%XdxSMDO_D?@gce4g5F4r!0yqAgfD6xzP9Jq=%QPgmj92+p+?8Lmq=U6J;qAMrU~ zs@O0dqWrPP()I!qxjK5Tr}TWf;(qu$?HJ-XzP!N)X_3|X?)|2)@N;jk(~2US zAt1gI(^$4YoNHyfns2HY8>lUM@R0ufki3Eu{Fs9usPHTC?~+S}h3Uk5CT=FH0n@lq zjxp2N=u0l9m+N2YfLud2-V@epJ$XjMtOgQm<5iY2j6Wfjl+~c(lU55;JpPbm8A8D# zAx9A{0&8eZk2(b@bl3o&QV?5~cJ7%j)|zOt*3z}pzNT=8s{?=A)+zrmJB)hNvRI&7 z1>WiUsW*`Jf7VZ`_$hyLwBDA-bc9 zlUEmYzKO?4^zV4rKARj=&#DO)U(MT0@0&AF)vdn3ncad+jG9Tay!NHr7;@Mc9bY~O zy5@AQh;+;Dom{8oS`FexlHKk4sSsZ;`rtU{eqHjV4GRCi`rKy;z+m)dYxy6C#~W0y z7k=-wE^t23-sovo@&Wi^qfoKNFm=x67G&Og?;^-zj5oTO$w}k{Fng z$2m)_q)*DPU8t$|zw)8lDjID(MG;#kDm*}r<$pVSJVmI`8{Nsf%JqVRc(7WTyok$R zl>r*v$;^pv%G0@AYnAd!Z)2x&pp>5XI)pmI1Dvb4(eq>x;3D+Y0i;d-P zy12W&;gvByi;N2$sI1Q$k>4&{Yy}?vM6T7zM6ABz$K++4ElypvrDg0z?2r!?#+Z7$FsQ??HZ zdDnwkxx9o)^zo?sMd9`8xpgu}8E=%y9N=wGLVe?8O_vFfn_XDAS6wpbA&VXoW%-~t zDOQAvIK9sxeLUlwg&+sqpWEe0_7Ism`VgpK(G>S~)^`z&n*OqK?g<6*JzMasC=>Ts zid&(u3FKIfR>5%J&6yAn1~~MuhLx(!(#1?P0{2ykRu7h9LhOft)B?V(8k}`^?`r%3 zx8vu!DPFE{fA`7knBttW1)OpeM@I15GFg6 zRTucw9+6#!dAQFtC3-V6<2JP}T;C9oF9(%XHgO9fRe`3WzcSZN@Z2rgaP5I-BnF%2 zMbGwIl6g>(u0CegDtf`KS=nao*u&*S{`tPBD%oDHZu9BLmCUvt1 zlm>X~tfmyZP{E@=F^)`huYXi!ZRIbx{gjN4O-#>7thx9g)a>8GGdjU{@@a-pf~%FS z>k1u{JYI4)Xk5q)TcUuee&v!^sCS+cy@oBsmws?X1WRIoVKV5p!g%qliP16X+xE5T zfu9ux`3ymTnCOm2gT~HByNlQ!8v4c7i^Ya`+tI4`*o-ivu=J~ZPNh)uE|>{ozdPw} z8Zsq4+93OPLwcAW+xDSNKYw5|b1tS*(?rrxNfgyoH5$Q@#Z>6O!0z~dD17O9uG(abnz)^X`};sb zQLM}RT@hB%T#VV*XID@E!m43WV0~6XZ2}(ywz}Fd(ZZt*vp;KX62Fe*Ojpb0BR-q7 z=*ux@UsE#lqF9}`<;dQY22Le5hhfva{^GT+SYGd$*eMP}BpOhW*U3m-n+EHKq}BrL zX6P4GfiLHNVFAZ%&zYpO;oEb(*>LR;eG0ZHI|@;voLqqbh9Q*K)XA)_d3Xq1Rvn}D zEBx8T6$88(NQ$GwBao{92FOylo;l-6crKVwA9Zgf;V)-~upc~x5~fI{U9;KE<^=KVY0_%i{y2R}~@ zXXOz|sN;C0?+OPq20}(KoFP;%e+|rA1YIVm;i&*!<`o6^$LZIjSNno@k%S{)f37fk zRp*^F$j^>Vh$+eLAtPXTWPwblT}KIJ=t|bSz&$r`+{9hIL>PO~;mt68>z!*Ss@);r zp5@(Nxk(6}7P!$3yl;)D%4GQ#-sa|YF{M1ifB#H!>;&*D=EkRliBDP%FmB9-excUhxiDGxk|0xpFqxfI&2@eG`Nmpva2RG+%xS}AU?xn9-M`@JUlY}gtzu}WdUSnOo&w;U-XfQGvk~% zCP~CDzr$->u&^=m(w~pSx0d$>cQwO!7%$j}xY*ov0I;1UXU%Jg1*~_mNlvoZ{19A* z7kgDqU-=>P{qx!RYaA%`d66-vXkN5kV9JA|`Mk+*6Ry4`mvBM>Z=HJ1^qb`b5e9s` zO0C00wYa;D0`KK(n~S+hHZrj_yx5hX%6#sW3DxycJRFC~!}4;Dgyk2law??g)fi=r z!zVx-?I`p09&&YZ%It$P@Ip&vsTS!Uoe-#9gn^}p1+{pA{RfBX+Q2-NGWRDVaqUhc zhxH;5Kl}DN?iomu5D*G7Li!NF_G~<5FIy>yF|0Z|#-V@yt*MiVvv6@SwA47UjLT?N zU#J111V(v>YU716L*-+dTP5oEEBYqNthS|i3LwgU!FvJs#Pkflp+je%CUEkMvtS#+ znk5V=6DgR|4^SzfUpw3MPSS7#_l)?tj7yOwJF=AQcR+BOY^7bdt^#cP7#)hmT^76b z&Ru2=DK>*E`j0HUG^O<4yIlY*l0BfiOFe5+ExBPrA|Q51$*;FQosi;{lK8 zL&jj(ZI9XZ4-4t#g6MwNjFQ_Bang|0EhBqP4z{?Fp8zfySxq0c4f|F}0snft)dE{= zQ8{XDn6?V!e2QGV86u*E!s$K`%LE7_e4lB@h#1TN2u$ z+IP11^K?B=n_|X)+&Oce_~z1Wa-@h$vdS4UMD^$IegGqA=)k528Zvpbe)1E;EpX0D zeLSgfdOHaj{8o&DZf4vzV@CkZSPKt}&)h|C$50T~fD}`AL#la};WPs#j(*N?;@IxY zh>m@kWn29+cAO=)_)vV1Y{HSF?gDO*auf#zqjnsYS1Vr@vbFJjgGdd3)|bDD%J-+& zQZl93Fef5-O(l^ss47lme|Tm$vD7|EdpUxsKoW2DLjo#eaFMQmFjbjj^iLUR-UHst-)cv zdiwTQ6yYVO?0*PpJaS!p{M8h3Dhl=Tw^Q{)PTeNHS>+7|8m-H_J%VaQt7py`PM?2u z#i}6HR>?p&MCP-rd2P$pi`(_yP?U!zw0*V$=-m|HpF*7aI)jwESFe^9m^^ zXg)lbKBT=#-gZRArii+Sk_`F#5~Jg{i1@k=aql@+MQ}*jDob*y=_n4%T*DCW!w3pL ze`=E(+SO-57cEQ%#E_}eRNFMrO{MI@yINqZNCNuzQ?DxK>DL{TLWYX0_p>DyirYcx zwT#T{5Rt8X#QE437NpbflhK*O7rYLy+a+V){_8N|SAKYq@fhMSk8Wah-xe|;@*k4I z+c-4KYnft{Heu;RFZB$$I{c$7U@eIi7P*8}dKy06n^6|U&woxsHJZN0I5=BV?1f~u zt4CV%rGOqXWO=Yc4c#+aJqd=z|1X`J9>JeT>LXbm5hYKjS5$ALf$;;OZ7VT*{hT)+ zeRhqNP;1(?HZP)s(&GKj=;xOerA#ozVHtMw=ye!n1=y{N0A-Kr?nxE3O3uUq*YFRR zB0=MM$kYy2b%;L_$I*b<&AI73Kec_VShKStYz zc)gelzVKfEWY7Hy-_+hyY3FY@7F>|4;g=nitC6E`TPF15O4}lF>$7|>csY=MssnE& z=C)Lk=uH663Xt2-&`FjBQg+nuy0Iq8;M0#JxB}`L_3fiY<|{1H_B4W(#O~paw-8|7 z4E%R=`7tCk+d!5y0hRf-ZCUh&31Jy;&a}zl+sithaxT0T?~F&tt#F!hNXguf(YlZv zmED6d7Ia(KTHEsqs=>|wPSp;+*r(4LoU3NbwfwdEZ-IIR?UB+>-lDm7al*J*WuMNj!Z3eR zR~QXunY65>pM_B<)=Cl6xgMfit$hh6oe28}OaOv~FL16R*2TVhTxa~cNK?Bg1M?Fi zZm8_TCCPlUD;Z=!Leyx7PmPhij3>E_*NBeaj|9mW=oBU)JhwgamSVG*zC3g~Q>S3< zt2#>1bZ*dBD7wQeM@+FBjb4d|nQPk5|CeZiTO(puUefBRrW*yQCo&iW=tFSR*ro|y zg~>_90!;$)sttXGGl55vN2)DW04(Wp`E$1J9&_wmPfG=9Ti^$zW z!Lgs6c7(KPmFYN6BH@D^J(m3S|8rE0!6~YH1LmYv_ewwIu3kYL<8gECL2=-sSgYL5 z?djzHR)wlPGUcCwdo(x}=HJiDfW%Qc92B+Q5Z$SYdb5X|vnJI%@}`zDS-7U9aXPpP ze0o8=K!A=jopuAXNBwgk0ZJ|oz*lChN$UTl;x>?^D@bPN7UVj2W;caB>hp8JKPFnM zKWfl17Bt0~%)Bm}(3Kq0V87j#({m?Ku@T+}?m7gjLP-Gj(y-6(PJ+*DOQ$3qQKaRK zo5!nG9nc-Qo)waZ$&8L}oU&k@Vm{0odl`P)p&n~FZBFFMIHMk8%@ubXqZ3xGov>E| zTo<3gVAtw0^GMkml+mr~_xWpv!1eT#%hq8zsRiSR^RH|~mgZ_ft92q!zbTK&bI?@) zSAMtd&_}D4+7!~i1v&s(5MKa?u-B4r#p5w%63Z_(7>Z#suV>jGhZ0|E_9jOo(R6)j z0lsXaWj4Jm()n`!)|gOXupT| z|G8f&%r_nFiU4!DmR-i*`?*jmH|k8dD%sFa=Z&Q8R>h7rYWE7`4-1}vN(TanW+%{H zK7%_<#x0m^VT8xJwlA!9!)}w4aDj*@{5Bk;-+t>#QA15d82`I!33cjfTUUG}E=H)y zW6}^5wxt-0k{!Hy=s~klTMqTapnZ zUsg+3TW-S~N_jTqO3=u9sW=@>ksD8VQBX`>Q-mYcJI7#op-$?gtycXR0tdiV5#{_AQN01>CM&e)b^)e5OlnvgB(&^_#CVl2|B?iF zB)}dCi~AJ%s3IQ*eFBbqC1cL0+oQ5NHcrpFpc!zFs(Cv=`M_8b1XOjrE@PWO%c-*3 zF6ToMg$3Y(d=`ei(pcT<*t-gB*5_eKcG!~M#rgE>=Rx1S=bPToUY* zl_9-y{wzx=)@}sBpq6Tb6d%tpjP)p1ceZ|3g$l+skWF-gMv~u`ReQx)CORc4m1gvz zp$SZ4$GMRT0yUbWqH!#ybfuLcspV=H5FcSLq|Gj@G|=MvyU3n+Q9mTn{TbPOp8x-o zx>9}!PT^#$I;-{8z%iu&Zp*6;XgimaYG)X$69JOOq+>pJ&n1d~Na8s3*7|>^`)u`e zgAoh;i6Pt5GHoJ*r!^nFcQ(mOW?ekO5uCv-C5H8g4dpy|>6J#Kt#km8)V?UO2TWWe@7HfKk21N-~_)_e*t{q{OilqpUQ< z(9qyuJicVuHxwt4r=JTqN)-|&l)|S`EXqM56N+u4ii+q{y(C~8?cIHMN1a=CxrToc z(OjYcx5Y+=YDULM8`grhDd{w25MdX}h~v-%V%q0k3#d(q%n=e)jE3g38IE(eIrb~b zdb_xYVY_+pucEy5FBix>y>nQNw0)NmsZ8~#Vv2@?S7X9F#< zK$!nO^6X_=_G8nPf|Bfa?db^Da%VECipIv-3-g%Q46|s{s^cwp^VP8A-yv7$(fCzd zO@ZGCEZ*%QagGh~nR98IBz#h5!rdTOFqLJATBlU%?!ySHyMxv+7YHHMg*&OWX~le} z+hGfFsK;=lQ0!yQDx!A17Y)e}08B|*wY)J_a~hd&03H@8{A$>EFgb6wB0iJMY_|1U z4ZW(IF|5kg`f0EJVlBe4c$uo>hdZsBR640L-3iHVNjX))eM%hLb~DJn+KNmpd0N!Y zmw$BO7F$HX6@kL=x0@2c`&-U`q2=2fu2aRan2r3v^)4$(OMm(`4WpN1d!T%8Z{Qet zN#`A4ZG50{X+gs~UrIC95!+tt<-TF{^9O4HA-jh|z$v2N`4!$KGMlN>h}}WuLx(xa zp`4CO;4t{i|0@Ep{U6lVfe?*@=aeMh8l!Fy$B55NEHIlHrd5j+g+#38iwqNcnNrE| z>H&wG?TV?QUwy{PjIVz*_fSF4iA0+gc3Qxu)~!d&kp7>Q=au`=O~8za2jNfjAdU3R z`>Nq?AC%x+2c#+t=qw*MQG7?ENqcx?iy+X>mO!CCc*nIE9EZactmPi#*G=R4I}71z zexafZvyOun+C+zLBtpN6V?;r#Q1hm@ic!z{b9ZVlvuWDsgM*&^qk}LqRDc+Grf6Uv zb9~P)q|assK(mP^8|`2v%>%fq{=!*;53ki@6L}ZWr4B<5iqnBki}^QwfQvryDnlmg zJ~ZoluWxT4j=!<}z9^2}8qup-yQ&X{L;tD3|GO#1$~;-#vmkPY|FdlZMsR?o;Ix(8 z4bj{Zw?uwdc{+4j$!4q5k+5vyAi|4WDI)pcz$fPu;xRd~n?fc6>aT}9d82Egd8@~v zr~!JHq|7hhd0~oO|Fd#qk>Jkul6P&h?U%oxXhRrYCOFmFLQ+>~=yvUK zMjwRnFNVH?O)DYZS`_QiLy}D?!E4uM|I^%J=z-nfoG?N9vx&&1CfSYW`_9C5&5X)G zR}&|EZgrmWnPBv;oT-RZzE?pn%A-#8c0M=>2i&OHCAT+dd+HHFIt^}wSyY|R)?5jJ z==4Kq2*B@OY&{c|o8j~>-+y4^gBy@xSF)PvQ=Aw#XI(pFi?F%tes)E8wwnyJ?dgXt z;pgv#J7ru5mgV5MFN~@~C^DPw@7-^eC@!xY8Tt=Ntr=V$ zvR=*2s(H?47_QvNR2P~ISK-@sh598%KO;=bU)SaUc4t($nC|v#@|##9LY{f7te-q> zL{p!d6k#Ic=XeoK+|9N2)lJfPr8#t=;re%ljgldy$AMBDIcWbU6WgI}OUmfd6e5V^ zeA#MPH|>2ERf3itBXS*>Ag{oC-aclC?nep5aOF_i)PME@1)-Mi#T=kR14YldFG(+` z==xfP;j(HR$5VmcjL~=C8EsfKP@ZO@mh+Uaen%B811Fw#I@N6*eIdh&QI)=|H9Y;j z2rW2XlT_YU2p+=jT?+qsRGq6|d$pLx-%eRvy(L%E$wYaTiatjO9vAQcCUB%FHEEWJ#Fp z#HXZHc-7z^l;K+o_}Y8cedp?*UJDh(7=y+d~r))Gh+y#5Ewnwa3J~Zs5dH9QK@(nP7A~zt5@bYCGSc8L@$JE zGSIS<*6gz8YB|TEjSU%mYLORqD_5PpaP4^-mSzlVbcpz7*Bd+X7U0rly5dHkf!C^S zlVjc|r28tmL3S}!a;@N+bHD&{KS&5LSR!4B6J>`stBKtd@1JlXBJv$A>QMFjIIiHo zr>hiG8g2V$pc6f^&N52N%+x{OwsVu{0+EwsKdf3hA3Q~#rHVFPd_*ifurd96C$$Ef zsC6t(KNxwl4hmg*X`6OhLf6tf^45uVg?ix8jLa)a4y^@Md-!0QiD!7?%aJ8D}udet0fbpJ-burib(o?T({5)R%G|ZV$5&n zi$Re7(X&xBfK_$q?0-?`W++5H`4g-zam}92Hz)?U3eqKcP7gfVDXlZ}ulGC7<}~e6 zzVa$YBVHOnc(UzRordu#WP#Kko5&K4(E29(*5CC1i>7A)>P*YPuKvfK?E}LM57$tE zPPM)^S;t>3ilfwk7VfP&v2gSnTo{HFuqlCTSY)dV3#(&YHm^mrTk(#BTo0M!tVbK! z$Yfp1Y9gO_-|Rim(%?RNn1h*0goY|90(PKRQfr%2hN>&cy7q0FU!+0tq z(n~;NK=sWeayg}!s7%_vp&x6;pGJg9vuv949=g%#Gq0tI->03YM|q-kp^E@S;ra&u zHXQm1iJf(?y*D_oog>+%VucsBXNWB5`J5r&We#4NvV_%YA>(b?vc6pQBI@V=(U#oHIaHAaVton<@mxD7^i2aC(-l3tzl0t zf92+Qr*216tSvvG5`Yh19HMJAq)ijOg@aLH7zjYd(68}!@z{aTd8Vb3m0suLhZzyr zkC(7Mjp6)rI!#CQ^znaO_&s0rTkw4E5!XLmdw>g*CvC*x8$1R#-ExJQtINd9W%My} z=Phi?(bBuYHV&hz-8{+o?IVkfORE`Rz}0M?dYjk$6qXyFlug^==O6E50YjnN@Av%U zxP0~g$GVK?pSk|SM85jjeI7KCNreip%vTVU~#Fh^RJ$;4|3C4JT!f6a!p zQty(q`xR|7HvAW4i;tL(yQ?=|RBN$+=obpX%sZ=tTKyMtX*1mcF38avlk?s`d+Z7A z%U|?Tz!&d+kYv)N#jveUW$fj|Y^D1l+xWYza>+y@Iggb3QxfksE$)W|w!16RGr*)( z|E=Gdl{!Gf6-RmS#X+?EW4&pO|8sDIjc?~&I$Xu0-ns*q^Mcr|q*j~Jt~Qom9#PhR z{R5t=D@_8mX*qwK>4As5<#Vd*TfT0+$>_RaQB7&Ra_UNWP_3+6pY5Y<04Y5G*q;5H zn2qkC|NjPjIsoXn1ja$b_jjNmJ|^0;VlFxQnat(?~A6AqIhUJFeL8=%=JB3toxyd z`7iLs-$y%m*=re=sxS`e&hPdcoU7+@j=W96Lt#Pu(mS1(Y?{Q@t5CD<^{-PM?0mg2 zksyuUxG8QPF6Wx4MrcTgBGJ2=5|$5su%-S!X*n#7i~lzXpq{QfOCM>1!?k2PwcOS)e1qwtIu)ycUL0oGrLlH1I+}6L z^jD`{MG+43%GM&&$j>hPzkIanAFSC#$aY=bUNzyNtrm9^nM-~=XH+B`&pQ3Z3gMj3 zXUk0gvmBc>O2i4%O8>Wl#zgL=8HE#*H>y7lJ|Re248t~gkq1t6%B=OpsAr>nqfz$B znRkDj6#3gj$(*5H)9zg_58>TL_ucyC3(14Es~uh`u|{R&WI`B%lPxZnp?&Z?Ze6CA zM3I||c~r@M^nbPqpB|vYZSzGWD$(Djz&&MD;2u_ocpcgdso~^hrY%iYmG+`OnUE)wg*)3n2^LDC3-K0b%LUTs>e=E{jU*|;(wvBEJP+*l zt*Sc&wPzlUcgsJC^!@KBZpqQ|awR$~(GOO~?Z>7OLw0vewg;LimT=1xVpj3S2ba%x zG^VGAn|0c!*DbbpsOD=Cj@`^^VlBi8+)?@{f{$|9LelZ=OSM)zTz#Fh)(6nW|2vjD z0QFGfF2K-Bcn1TTBQUyTLq{&{7L;w{%8S{GAiB@t$RHK*5CyGfvF*oBRI1JFQTtz7 zE`xqXFO;kYlVfHbQG;yhAEbPr8?WqUS}nJoSJL7B=@uDV`Cjg@1yN()?Z=4$5MduM z!f{W&2o1W0J!^e0eD()DDl+oW8;=hi;{ot7%P3GL4HHucMYYu04vlMhWPZC;V@LD3 z1+6a=IUX&*HG{1E4~h4k&g~yllP{|yaJsRFlEPj)qzBA;;nG#SXL7WC6(RvE?_bX0 z_7BxyM3Vi_1_S|$zy1hP&3^)C^CMIpK`>HYeJqq>sXTdw+$vRNem6g-Ve9D^G6nD` zOgCQahv;?xo34^}Sr<~2CurZlI!5;R6>3R^v zP~T3BxMaXxtyeDCw#n$!L9+Hg!`PlY5MUb*5r$)bSND*1uoV*3zCHfd$=9R2K&jCt zQOfzS?lt&T@GMP!nR*W-N~26C9g%p%r_MQvIyMQcC*|SG>@_IPTgnPHebHju2zzo$ zuE8XhZHEOV?6ig%Nvm1$4ra>RLaUh-1-`QTjfD>0fD@sR-yDoqvHcFav-8oYs%_xf z6puviN(?*gMCIdE;AwPaL}uu$wL!8e>ScYhkx$b~2mGSfwkYzhOkl%344VZ_Rln8` z_dxqd%898Cr{fx=Ms;u8cf?cZju<^adajrZ1~xd*Z*;uT+|@V8S_!xPuhtU)dRepv zT%hWIds%$$V;I)v7-*f^|#M(yzoqYdYwI zU++W|k&bL{#xOFt{&qfB<31iPEa&2D7lWxULmziUd}zNL*+T5=+y89CqEKnI&Icc4 zcxX)ja&WTiRJq8>t%(6_M1ReZ68HGAkIyPW)UF`92Ic2h4Dm>#&i5Xl=Ka`}ip>>t zAh~J^$PxH3$Syvv1R>#G;liILWg*28R=c$vEJ*nio(*$!Bg)swXJ{{rO8Majsk5u0 zBoFM9lK=c)Bxvpd0KjqiL+J8fOxi22XXE1E5AT6t_DnI?se8qJoMR0)rLS)sFGgdyJa%0F>LkvHs^whR-(Z3%fRKu{LA3DcI>*aBiXPIGe>}U3~kL zdUd6Q9j$Icw3>xQtJ$o`Eg6|{RY2

#H@K%EQ?G$BEmuSRO6KL^*Yz={U{+;h+LAAu^Zh^9KS`FT$pKeBy@2Z zw&H?-?O#}RFWCT!uii_PGu)Ueo_C&+1aMVtx`*c)q@>d;q9qORT#KkVW=rL4E0vL! zcJ;Ajh=DS#;MKB@<`^S4-5~e#H>)xhaZ&Ec=(Ar~aIg<*X293YtHw{u-IpDvDw#jG zPJLB+@00QKo$bG*KXA^Ffb+mYkfjpsu03E6A9c}E0{F5a^5ySY{7TwBJf@^}Jl|Nt zyEIzI+`#jgEYxX+MYY1Urq>64@aVY@;LqNq)r2RoyN+&pm_DfoHRwBLhdv~oU*+46 z;0I!{N4tNNmii=|cK^5bL6*J}?R*wmpxabI{(g7(*xA~pdNA9d!{?b1b$M~YmpZx+ z{5uuZ#a6ww9S5_rqNHDu z)N^aY_6}n|arS>XuDZV5F>|V5)#x{>fK|k_S@!qHD~Z9{`ubMw8|Pkgaqy&|cl-yD z6Zrv@Z*LOF|2K8cM2V<)QfQ5-+Q5`{qTJ6Pq%~pDIAJHGz}4HOG{1jo*PM%>dfc$L zW0Qv467bMtz`H?I@P0q(ft4t@W&1WbM`wsos1G{Q)2SgxCb^4eSV?V*wQXxy*)d7G za?;pbZ1I3&f{waqm@~@0^QH5Vv6jSBHiqx>Nw)Q!2v4ncaxf8hPac)@mezL>m(=G* ze)jk_MyPI}e7i5LaS~S4*i|eeVC@d^;OG>L>Y%WcI8*m@!GPKx=jc<2Y_8OfE=2nQ z^pN(sz>DtmxDA!wnGfwM8y=rVFK<2`k{Khl?q436SKJcr+%^EbqotwA(AX8f|1Up{ zvitPjK9WC!U-eD^qgKewZIl9X5dX(4R!c2Dx3nhXkJpAC5=ejbdil1g8D%Wo1Lt#r z=b6W7O~*&Pd=6PZ&ndLO=6;`(!mI@^K{P>~>{Hs-YTh@4jB!XOMO#PGxUwo)GaXW` z2L4}IEfUn^c_^q5xy%$cNYLbgDhs-QyOO#AGJNB2^h{QSuWfzQk$Q9`r^53Qr zP@qqnzE1Sxw=C*k;}&0lAXi7QMf5gl32iV1x)!{w(Wy^;QK-}U$`Hj$$5(wG>S6$% zkq<_w#Ka?7pY)U3x4ICO)~bVwW+JwXd7m&L33?3XJ%oL}yYqo{svYItAC(&lSa);V*B;i}0F8z}3Cq*^?*}!U`8AUohR?Uh|Y1{FwK*6G4*d8uR97mu{AdGSJ)$$S;V*qkC|fTijr(hK5@1<}a#D77%4 zO^;&8K(TEvo4KI0)I!kMS0h7VY!10uogKQ`i(JCEG5eR^GfOY_eRV#EE*f((8$dJ~ z3GEc%Da$RgRh9S#5N&0~diDOVaz!uAUIK77dhgBe+I(pGKhd-K7+^IVuT1>O|E7>c zIz2LqoQ3iSFG8y%E+5wDTh6?b?IaK;uY) z$u49lBMyi(F#l;@73C^DPWuX_g=cNrAUh+&=hyv}2PoP9-pk;E0mKXH16uf_+at60 z05$IT(k`&XJhH-ZyD-~)!^A?%Bu^p?q$uwuVAZwABf&ZW(}|X$6NQq3@uGpbCeqAqehmOy+Fb-Ow?tUV}j}zgxa_alh6T)J|hWk>7ub(u^r>&Q* zL0O7Hqf79oA7r0o)%r2g)?@uL^%Kbc+&5LypK zBoh9fp%c2nS^nnusyK%uZC)xi*8CN&)p`-FMo?# z#D#g@T#2U}dRpKDy@9$(ptqa$24>xQCD?+OZEkB>^^eSXpd)P$8NVVJ8(>;Iw zN-Q9^%3(na5brruwdv2iHhPJB=hBg@xlBDXY(*7}f=nGrt%j`sg-?dtS`V#@?5`-> z%!j&wbtbTDF4T>@ZSIKuxJ=sEe2m{}@aOE7&m}1Wny$ns@CFiW zZEG3Dtm{gs#Ef6=P&YF-U$19z_wuQ~`dYkwLw#sbqk{yMdqHC-0(mxfoq?DMtfrkGLn8#=k83wxS$N-V z5UpCwy$GH$S z^LZlZR$6nx_MPfD0idTni9>e5mj@3_B6Ca>xZsLem5S}feeWR!gogUdh6+tPQ{!a2 z={LOtkxbl;J9U9){rsZ5vRVF>t>y)egLaWzo^D+(BXZ=P;*Hr7USMwrXSl1tpbCI9L2gBW(+(>hOi z)y4!M4X_>=_4@^uXe(h_I?SZe&V7iL%q6O3v9+Vujv=q<2%jwxhAIBu?;D>OL%+E9 zDIF=_hKb%39?YaqTj;@e=(_9NjRpa$)-fqxL?^nYSZwry@xy>6eiOo{IM=loY)n`<<&s zm6pfa1sm_-77~?leM-_FFxw>1204W5<61sn<*CkhKfzQ+)(pP|*+y0v?G!GUZ>-zz zJP2%?u$iheX0l?_Bkd`A!T7YxQ!LkI=^<*^zwQ(qpyGbroJsbVJ8?ptZM^RgzK9G} zclS3OuXdce9(IOR;IsJ8gugisDgem9nez)o1~vi#qu8dBnjd(`vc9jz&JVeVx26&9 z-S(JKSWpI`t_$EH&fn?#U{imIS`|_k5tUih%;PPTq%=#JS> ze6U9=uzo0(RxNio`pLo*R=c?=3cI(A?)b9aix_(ru#DJiJyajuUNsc1a+-!1nbMrE zp`)!R=vg`>@_oZPyEzQ_^>u!+D4!9Uwz>E@(eu!|eC@{&+S2#BKV2e(qFf;onm?IR zCCVZq_E<^h?~6ZRF4{78oeCZfS-? z#X;ONgT-88>A;*Qea-I{hJi?3CZi%)7~SXm&v?*W4;+)G9#oUR$<&^9aO`Dfu{mA_ z(}!#;Qmtt|mbWug5nBsyF{?e^$v0!;nZQq1Y-#8Fch|$JhRT<0fHslLyiyZStyuv$_Pvfd1=SKihFZJ>afs>rshEY)yX4YDe5~gh{dF zsO)Uad^*ARkyD0xZ;+w)+1s_orrN)PojDr|TyBy6`z(`vCUXG5)?f)fwL9 z)u)BC-?dath5-nUUxWOXA1Hb9Y;W4r=;+87%iVdi>WaS1=u0r**;=vb*Cdqm_2P?M zp=*J)-Mn0p6iWExt;YYLFzK*>$k5OIl_@Jk7cH<(t1!RNL)VjMVWn%-WB4H&f40RD zChAL!V3kj(*zho32FZtCg3H09p};k*JJexL9e(*Fm2lFKp5s{u1T2Le?zl}abW)Ma zHhhWc|6pVNrrd@l{cV(p9y{7bGM}UPoIw4&+bbL6F*b<3qtXq z3aeE0m+d6>KC{_j6^ z%d{=puQ4>c^O34u8wjT0w^gS&C&G={@%GJU?$2ssZ(c>YWwzD z%KX|W*?Q72DobcK?WW&@#JO{vbe zg_RcOag8O;zfCJXHMekzP2ut89XY{*sXSUem^G3x*>ZxF#!})qMdMQ*<80{F;rN3q z?8^e2XtrA_&=^}}UfY%5b93E(^yTxqdpYjoQ3rN9XtS0C_LDE1nO1w-^_NOdaYe_E z%z9+&Kp>kGjlsB?;n!`h3k@3m2+Owlh6JZP`&Ps%{#uh;K%o1ywv%YqJ~=TN!EX6s zYnyhEPi=18M8!VkXPq~iya_H_cEnj+e5^i#h&>n`hb&cNt3yEhBnyt9GIFpl+X~O1 zihuoO8Q;s$rfa++rWWy5Ngm86fkU(aVe_@vR-%hURIS({EpEyBtZ$3XLQ9kjyH1t~ zkP&VJ+ds&1PaVh_f+s@;`nPdGkIHvQ@qQXJEKfY2y=beLhcApred3_Ey?c4M+AlXY zA?;05lMqtwafwJXI!3VR3C(xm{9`UX40epVfd_zt*;+lM z&LEdX%zD$&x90d9p|14fb%okhZLb-;)iZ5$*2YZ!l&5|lOhYA+U;2zDZEjNBP=FLn z%N}9aa-Z?U4P?s|AE4pLW?}soR&imvI+P zUq?Dmm!d3X#8FX9@J&s3zUFL6$RgBNY7v6+4)D~Nbdl*Z>r1N53MOHGa_?3CMPI|a zQtKW{BrqTxxk$|fLc7(vDu_%sw{mMH%TKJT%i@IB9t5Qx5;E`!Byv@-d?K)Dr?UG* zwt(!rt{iNM!z7y^JO~-@UP}^JfeFZ!CTTe&3L8E3XIAT3fbgc-GuP9P>Kk^&4*1k| z65Fa7f$SNe9(}7+UnZGl-{{qvru+3Ra&&Zd7UX=3>cHtglYdT=iJ}iDj!%4@!ERF; zMu@M^@@1Ap${tkH&iOlaQcTrfbL0>oN*;Q|?%VW3@tfsVE>?eBgrM@+sFaF_P@63> z8Wb<`ri{jO{aOIv^J~h8eyr)ri8Ry@>lCVq?fkA-R`$sCh3VMKvqHBT)1$L+faGkjnBAt<9A!ihCXpd3*bxQu4Zf;|r%BzX7C9W8#w63U^s6OQ7 zN9zpJsPE8rb>UD0IwWJSOOiJ$qgw^x+mFvdK9k+_s@P;H2=iPf$i*hqk6OH9}0COH+?YPM9i&t>Un^M9tTX+CuBRFc4ky zr%GR=L)1CQAM$Iok#IVTd-0(IfDugH`>%-$DQj^p|cgwTx>Q1(Abro1|x4Yvvn)oejm(w{q$m$erhkE zLl0Q5Q+ieFf5PcviHY8Lxdr2$xHLE3XFVwo=4U!6FPQFO7k01Jw%cv)T5`p!iQ#Ka zwAa3vH1ym-c2-#|jw*Wkc1S*$wb~%2Op*Da7&Z%;gK_DfH8L1{P>;$8C1Qaa$d$I9 zX|_!`w`+J~VGSatUd-4D9k`dlsxmgz3rh3>7m*eLV4i+JH(a+&L|^CHXKC;UXX@w!Cl(#?^qHyfYsL!+-t*hs z&CZ(_CxXXhweECKCowKGX!%dD2;f=`fDmFgfDhR96}jULIa~Vj4KsS?%k;%2NTN^c zGN)%b#5hdYi8g;ViGJogsUT@+^G@Y?Y#QrL9bJieR5dm%Qg7eFWIE7Noh%XN!N~0S z`fTIMYP0$C4xu&b4-_{n%+W%_=V3*5)GbDKLUdi{Oo~Vsb}0g5IA=SY#-E^Ck0@+1 zu-X^%E+J=^SgMO2JVGUm|6Q@GA$mrxm=32cMn!`W6(FYi!4PsPW4cpi+t(4{k5gNW zH6;DYyJxHDf>b}T{cyI*f8Y7fhk^lJ+a)Z5IK6kXVTx&Kvj-8wQpCd^)k^G9D8KiK+nWt9>_ALW-lBi|)0t565Mh`|}E))k{Kx>z_VLWOJZbNCj|} z;-gNRw%wip&p+VQcNq0V3oyl{*K8C^av(519&Y5&JsSF+4s71XJp#5*0`*-)G-e4A zrA8Ir(!>EAKQ$)sk|J^r$61^R3Ufr9n2Mp-O;6$a_pf*k5wYkx!pxki{^~@cNacdD zv-1fa>=|P`)SN(D^9?=UlRD|?L8QVlGv0gbV$*Hk38M@An%^{t*I_#oiDFF$;t^=5 z)bI3y1>J#E3rGkrOYDpeYim{Ks=UapG}C_E2)8trKlJV`dv%wkeK&zUw2*VxasUw# zxt8WrRTs5RXEYi*dEt1B%j1F6r}M3h07|{->=Ja@NgdtH*3PzjwDW~%%oOhH?0rQ>Bs+; zjgR)0{u%(XFD1m0^b(?fSk~j<4N|{y9FMeI-WV_<>{Njo9DjqxSiyeTUkRCU({;;} zm@QL3J*A@sT}7}CaydO>mYu@1nDTO*n-W~A%+NWd18(h%%eVkUh#b6`sI1~yLH88-TOoOD-1zO|DYNhdjQqUUZNPB1II93HZuiXGvuTDybBW* zFAGeKYFCx*C9l&#WTp!5&n0XY+<_|}eIsnK%z|&9(_Ko!L%;Ene)4#+bXBU5 z8vKtmG{a}zw$X$ijfeu&^O!l~Bkin+jgLF*m|pS*p{I?};6$1+B2Nh|s*~C$S73BT zZ&}2){q^`WS^o$c!_|ZP(3YHg(kzWG@VJeXlo_?Frymh=yHTP{+IW>0$TYU%bN8re z)uonqkZ6Rh2jcXYj&~Tv0yibca^AWkfSLdxgl{EmllL8DcX*KAw*$#u4QqU3U9RVz z_ciciEgzSBMmn1`P#>`TP@TbN3#lm`3ytfMe2KruhiU?=9Qt4#E8cco#~@sT9=81P zRaiZEN&b`2YHhu(U7r*aQJBzE{6rStz!iD7f&3eqdjIMM&Ue~g*1Zb(_14gw$M`VYJ9hw+LYZQ#Ctv) z*lTR7jW*j1-icsc0ToLxM`-;?w>)FY`dcqG$KC1^%lV%X0Eys&?%Q3bdTgMiD-Y=S zd>(y-9Fp2}Ve0i}D$mf-X?4R}vkdRy57OFXm48kbdM^rI0sH!}p|$uO#Vo+oH#%%o zrJz+5J)j2gb<67_woif=>43{ShMDp+l6fSdZBU0BmRIjolI@>pGq%;T%y%BdaizHl zs*mqUyvX?Dw^31K;-l+P6qW9N^-5*T;szFrV>WPeL?E+>c;ZUqJLjuNINok0n0i*w ze>Twqsss1lVO+lNoC)7Cd`CJ=LxC&0?S5d@ql zrm50e{O+6be?P?Mbe>MmC8k7^b(MN4bt@qO|4xAJ5@eIW5Fo|h!%&yV{CfF~&=L>0 zm~Jfd3OxL+zt}W3?Q!W0#%$SLXs3gJ6$NN18!FUgHTPR^e1trcd~8CRFHNhbfSTod zjw#AL^Rdw-o!EsTVD2845dqmxevF1c#&n+B3+QG*5Z%ev$7@f?UnVl&)_JQtJg?RypcIygiX5DKURNcNy^K=67limVztS38 zpS5h25t=ZdYl1AaG%`uJ7w`T)!4I#a^pZk{(|BSm>=t7+_i~Akb8>pAjbS7o&ED_C z(Z7p*$yK%qbEmA24gi=}${-%Z+!bBh#vRU)yJeDM9tM~Js#PsF%Gv$gJ*C;N0ak_< zHR7aCy;F-nMtAsG;cjbwe}}-cTk~0?1fU~3S{vV5sgya8s~m$%Oh~iW+B8uYa9_L_ zRnXXw+*e%qBViYxV;0Zlb?N=+i#ws|?nX{M%~?{pPc(_N81_YKsWLV!$dSK;IsXzfWJaW3 zo&9RVn_HTDB}54Y`QBYwJs$~4e{gEst#`=v6wq?Vwbh{mNW8fMLt1+#L4I^-nl#yT z^ugxqfo$KgS_%{zuP4;~>`8I~I<^vK%qQK5S}R<`)Bdcj-VfBQ`ka|cRF{cMQPl3tC`{(IB4obg!ND+rg)scpY=C*5NN#WPSc4wvkg`+6|5Sdqq;{1(9A_g!jLACz5<1r^wx&)Svx) zn{v^kCe@GQ36!=xP@e``@2hwB>y=^okJT6Xa9_FU+Mq790Bz~?Yu$<=b*wuFt{53T zoUiuk=G_}vc|@e>QsE&#Z6kMm2X5?;r1{fW@xy8~O`pFIW%tT|Eiw_otmS}5@oObV zUMzk&D2k1t-aY2fk&^5zh?JPP7A4SQ@SjNElp-*S<0*nkz3XR~YEjv@%y>Uq&w>Ik zpDklm4XIACTb%F|Af*MieysqCZ7D0K-7h}Zw;HQ_Q;p>=T+>xTL6FJv)av4&$>nr? z%li6$e`7NBty@w`K$7+L(@mlUPO|+XAh3*lhf#g?xfM6XuPrA=Cj*R}tFZj2kqYnj?xkY-tXwH5TZsT=+16Ys|hTMFT{3k1;JDYCh9Q7w`D>muQFD6L*q z!_rXGb)3OlYz${RPvS#AjmIyi^O+#)B&8K*QZOH0B-Ebz$(f))8vwB_3=#X<28ps< zp=F=J-Q88|LvOyuIR$bC!(a!+oOq|tXd!3T>T%NCq^x-}#FjpeyZIrLpM6|W7Z(?m zT?*wjmSonxB^OQoV(UG9UZdoF?NuAcW@|lH?&nmV$pWgA{=0K%=P^+Eq#Qzp?h@Ju z4CzOJyTx<+d1u+OP8yWhvK2rAtAUhNDIc!M_D)CKu_TLu`4`QiXm2Q|k}?X)YgL3C zvJ^EIK`M@Xrf(duQRhVU#C#TcGOkWO?t-~Cj!jV1{Ix@NS%fIinFV&kQ`Db(#(Y_ac$owcR@vVQ2RG+s?wSyB!ocW{R(4s zXUE}r2UJ6}O(JNBOe?%wP{!?(?+56jfe!!v?yP+i^bsDMaSqAttZ2WQITZ^AeKowa z0|L%O>|!l*3QtBs&QGpy(CN3SHUwf)j}~9K2?F!Dw0jiek|=Q=K-{mU)P-AGwgw%e zRfS0H*iY$hPvJ9^v&K3{34z&4zG%QY*__s7A@r;l3vElXIq3T#F{^^g88*$N98k)4 zgHPo^b{yKUnziND__x9J#_L+;9^BK389!igYJ)gi^lZ!m6Y=b~=Bzt-ok)MLN2@~p zJ#gE^d>@)5j-kV)2YTa)Jf}AuM{W~JC*lnhia`g5HJaJ8TXo5vo=;fLT_ReiM4Qvm zH0y)b2G1nj!VYumAnr~PO#r3P5Y;U0&Fwh$v9k7HEgbb=444pRs5$9b6&~nbRbqJQ z_kpDq7yAyUjv&z);Ehl3sa=eN#vUjzZ! z79?vf4=aqj-Pl@5wFX3yq9r(VcFO7~#%-<mu}uG?1tydQh>84IWl6c}jty8-*bV9_q|&~*;|=;X`!GWvXukSwh~VHbTV@gE?@ z*{vO-oww&lFGNIY_c;|iB<~4*pR)Q8v^F$eFAEot{=fX=4p7il5Z)g;Q+&5FwcZXY zy_lvb_+Fw~5fNFMZcvUiL+0E@>P^TWUg}KwwG6Q@YK~w`@ z_iqKEZD6&Ak}Ag&Udu1wB@nAZ622XGY32kY1_JkU1oZCkwgPK$T=p<@8Th)r@&GPp zZahChC+Vz4G@6`HSXxHM5#}3YbU|c!rEV7*MGgG?`KQbeMkNlFUjyW|&2#5|D5@n> zuK>+M^I~Xjk9*7t@>T35&wcPi-}Q|YbF2Nvu+X(%%mjK9n;S8oc0L#Zfh_5vRuCC9 zceXFVGk0HJj|R28@J39dvAm0fvu4G--jk1O1W+L@>>|W=>_WqEUyfi&{1bYf16hkr z2ov~8S2XimQv{nO#~_O}DeqB_i<73nY+ZoPVwqk_H1Q^hC0rVK;8TX@TwW1C&RTO} z!baqlC{K=VUWc_{0c*G=nORZoFZr;FyWuQI^248nE#3(3nO_pK$GM}0gNEx08Y%$S3BQ97`j+NdwH3?Q z#gn!!M@DmpyAE0i(3?`CCB*d(mZNy})?8PJwz zj!Y?f{^M?*2Xsz-whe-K4vJ9_PZ@W75tSl1~1j?~M4I;emwn^G;(UPAZfB=e? zUYX2#>nL4eB{2>v0RVxc;X&6y7H>o8ON> z0gG%n5q!XIkH;n zJ)r7~$oTiae5zT5W2RR;lA|e$oCHGS*bFB6$!Q?B_CHkATd}Vx`_A+oB{6%B^x{x| z9gl%JC$9IYETPUkjE$X#*@|TSa&>2+v}EB?1cl_Q3tZ^iy!o}KeQwIPi+5*&*Nmsk zDz;EFksLV~#W)ymB`LQMVgpi{h@KEy+r@NBr|g(}q2a1Vmtz#C@AnL-6Ko)Bda~*O z>NvTSKi+3DRr|Mfr5WdT2Yn9mX99LmgKTDTU6QVlgKj(A3bN^1(*iXylg&0eMaG#s z8>-5rQn^d2-JO9mv?=$%SfibQZuvT(Ii;-k_|gPQ;>P8hm9DyB##&R`HIYkU8a*%* zKpx`C0=KUEqCOkgVPdM1b6Yh_VEkG4VC1IvI~aaqrXoqKH7O}p>=1Im648yM+Zi`5B>o>>A<+`l2ySE#TaM;P!TOP zn#NvGHAj<*G2Dtv^4YrY7%y*0n!2yP4NB`Wo>#f2m^_l^Ke`s#EC7B ziho2XCz}-5C+-Am)}{H^)qH(BptB7|E{jIv#@tX?pzN#R@xsYAj;EpE_$AZ3K(D8d z0m$}o6iace!z=kxOx#^=Ae(;_re_#OIFE+3SeKgm#de}AmS1hqhRv+)>D9$W;GJys z=8W|y`Ow^@tLD$Qfj%=RB-h7>fq;yfJhw((Tgh)b+J5Qr?w4~8hewF4O zpek_@BvtJKps6V2YqNixXrZ9hd`nQ);*lW&qs@Z#~m(-S$&X; zB(Sw)nC;opJAH9x!?CDJgZmDk-I@(CwXJp;2yA67>Uo_NffkU*_RV=zk?6qzzCbsP z!&t(ww0FV^#qM;Hodxp>4}z+VqvE+lL0iYa~En zoVpt)a$>kp`qt&)PF)~fr`*e;Hc~2eRwwIhgd3qwR$Yqx_|C;W&k&M#=@s8b2h|{r z1$75NXm!13gjyTO8uQL}huJtiW7_#^w5cL2Gs{o!?dg1@Lx>+I`S+>RZGbpLdP$%M1jaN#nUplW86tH1 z9bxZK_jPIqe+hd7=xC}F_Y~EUrxX3-XYSm!cIUH9mCm%biCtViVNB>TVqNJz8#_T^ z=)WiK>*Gvl%ET|~Aa`M8fPq~j$~r&wC6IT{io3J1Di**v(UBa**Q}gkR2b^wBDmhWpjZd}P3;N`-a>D)JA1vG5a|D0xc%RW0f6ABaWCsO4L6~+u^nJP>_6ylA$1b{;pjU%RxEKA%YMs+xa9ZKfXE5NtzKYXO`q+z&H@Xm zif;OM|ZE_lK>Rho69u!SZUBGCGYTQdb%I{Z($_u% znirC`WJuF)%LX44f~cF%$$SqB%i-fs6>4!+ zt({9V?x12)(@DEXGcM@aCnY@|u2l_xf0sOeJkGIB4k zjBd!?*5_S;;m4JmsGgo4rWBpawHJZF=oN9?w+*;|H!08&8rO`WwT`SlVdcVQioO$N8)kSEn|BSo6qvEGA63Axwl1snwJJF2_VkkHG|uJR zHoMnNbro&c?9PQbx8xn?<>MoMn~pJ50jM)~SN6=-60;0d7ZlI!gndj$s3U9$fKm3p zs}ai;7hm7;U3RNAiM6-4KcoMkaI+UA{9ZlLao?GOf?}O0<4@Zt?mL*!VbZ`LTg{At zlEK;0=Eoa85)dfaikO)AjraLT#@0DuK}AK3(B*Hu6e3HGQemwEZGmkfj%-)J{6@&@)v0))Z@*_IgnWRT zkUB&7XA*(t19}{%*A_4lgCUuBQ)geH8;76Bs0X*e&%wT8izBZ$6;V)kWST$jiZj{L z#WE+cnUC?(kXG~5*ahk}# zXP>>VYwv6CJ2xiUutg6S)Fr^dfsaxD5-ChJ70_L^Pbbuy$gLf(OyPz{U{qXtIC-pn zY(VS|;BwO2zDq|~-S)U9ww{FvZ47O4)2~npx_t%m4=(#deH0Y1+mjRcdzJ`KR@yxZ zp<`z5zSBt7J_V{jtZktVp<&p~{?#r4fEm}*M?(=HqLJer(uVJ4<7M#&>&dG?0U#HA z5wVbW^XTXufA`*Jhn7o=s~=DNvINF|g*?$`hqs(g=%?Ji4Ne+QZ%CDslj8~2w-zJ8 zHgdV0ZQ4oqfPc%+^qaARTwY$D{C*zurBv$MI&Pa~n@dJ+e%b5tW7RU&4q)`w*sxH^ z$2o6e53(7wF)xZtV{=v;05GN7JWm?jZ@~mG7*^I(&9evc73w z9ENqu*bOKy)RQaCk8@B>bqVEh?cx~IF_v#y0 zLTf*pc-bQ16s5^gbGOIfPv;)uche3XsTx*WkrA6+y6lXFnR5LJ&Pe5q)cX@~aIaos z>P+M2I3!|&p4hq9rsmhXKGc+Vc4;`|oSFhDFqZl5N~aE5epY%14pFOhyiqGLymoiDq@8nsH74SazcO*( zAYfGb{KJGgec-5kE+vJusHB8vHpKdy&b;VB+rvC*i$S31?BwKBV>5nZI`)-#?uB@BZ*!bKr z!VYQNY+R8amctCkdGu(g!DZd*_Ubf#w?Dnl!p#=CzVIVRD-5z!rpwyxbBM1Ifz$IR zBzmC$6*5_82f6l0i7rnWFgA9*BHJFH2aXznC7#nYt93SJEXOxG;xSP+e3?181As=SsL-0jSf9v8DZ(WCnExeK)C=7x zK*?+suqmlD)f)aPxW+a$Sv~^x`)Oi!P6INhgV$@p3;+ zmuIv&GxE?{hP_gq!-?^6>wzxSLx+y<+6F?1el~=+%M7q{RwFjHyEW&Cssvtu_40K; z+e%$U*6bWTZFy*8(3rR3-0riA!HB0W%h`R^n0;&9x?20oAHP7j2cGKB4_JI--Te$r z4tPRlArp+MV&M6p}o)t^_`uGVjGjao1KCkfex3^e@_FJxh% zKm*%4!i!JVwUtL$L&A6JMwRMO(XUwfnfIG&1GG34k318Au#x-w_uH+V9iAqwFz_|! z{Sl|dc4ROj79EZ)sV;q}tEu^nk*n7o*K5{**cD%2UuGd8eOtZ%wx?G-M>5u8HP!w6 zS;0Ei0!&FskxJ;SuQO-AO_|3>?9)Mi-0t*6aN6yd4er!`79}~zz{MwM<`fv=NA zY+D({n^@feO%!EVf}`n=Qp|W#BZ+{_!f=lBpAuWV|9Q)-70ou*Q;TnN$@H`IM(Oo` z$Oi(rlLb=y`FjjT*=ji0?B+Fx42;4dSi8icr7eZ=eph2;C---b0d+)dnv#11r_(vh z=Q95_u>V#1JlT+(f^UHIv0P2FiWX?nt*l#7)wtaqq%RS82d{22bZ3V<{(5|Jb3B;3 z%DhyXt6^lMbb2}=hF#gxyx(%o(^NwTUt3vGadxg+YX$%r!owYqBj4>X@G$e~n3rmX zf!Csq@Z5BNuf7*r#}Q5D9(8FTwl${Pr{+psEiE4q<*<-OVZa|&ysqY@fqSESfb`4( zRuEQfif6I<>Q-0|MW5LG-HUC-OstasynDoi29vJ+(`eUxtJEs3pYg@dK$n}lics$> zcFQS9B`)2?}htvMh~+ zr2T!D+1$=?W*$o?T#eMGz`@6Qa?0VA`mK8tvF)+r_@LC3l!EWyKXAW%+0W5C;|b|Z zU4IUr`DO0>?j21Ss-v#%v*3`B(QEj*CgjiU@@d*g5NAna{_CcA+Yn_7$`BdrMZgzH& z1DRsw8Sh+Ns;{7@=jZA^K2UGJqesachDBu}=|U6hlql^Y$2q9w!ORiZyiUJLUK^@^ z({48!fcZWjqi^v4*eD>iQTt%@aghZP=oWdWDjaBbWlsLR{OMi)UFW0cDQ*Rh?8rdl zAW@gl2SdV;kPB5hs^`2YGvfZxXvfn0kZFW}1wKHd%TZPQ+oOB04R`Nj&q93XnDlaL zym#xVGfmf<5Py-9qbq>y?oYV6j;Jk!xS}#KqBHmt~j4F@DD~rmKa2ln#J;HNYJ$CP+@EE#1pI3hO zZtod?=Wr&HrHhEmTBLWh|IIR$>}!2_*^<*$%ft6n*H2-<%5X@Q%|)xaCHdH`a;feP zLTotnP5~c+H{Fh`BLz_Aq_wH1QgJphm@!gY1 zR4ZXUE@9vjjhHn9>%x0J`nY+!N0qR{e;!S2by=K$GNm9R&asr3h{5GW&i>yvvE+mVJUk!&=^W7En0 z;%QQkbZxRvQ>ZM00G9#~Gtktq^uiR6nk!x$C`naS5DnM0O#6^eeD&xzqAgS0YiY#8kG)$k?jM!fBzv=Sl(a)#v{PgLQ8VIx>34Xmm5X;A4k(XN? z+HOi=&avq#I8$ponZQzsL5QvABKuKUHYh0Qr0wC8A$T*mpEs55<vR2U z3@^yhtKC_M^6JKhRqYGYtHHeS^!|~G=4QjNTY&vp=!a9ES7+Wn`dO+g$dvXZP=(ls zjkhmD)1%lCKGz?>8(<|B8d`iCwLHLeDOzp<1YKvpw zew5T2#!|j1@gF^l!#cOHDnlPXKFY2T7YRl^9?TQGDtv!YV~c0|OQ#@_&pSP4{CyV9 z|IX8rAT$CeJ7lV~YUMb2dn1^Cs`O})c8hEX{?U>>whWHseFT!xG=k7Z!CS!;uqtzb z5~otsP-@h|$Mn@62s6i*%&! z&LFM4v#(!ky2#Ewi6Ap}**wm?hXRcXlil)~PU&n(uA}kvwqsFb`~%Z+)%p1$4o$m! zg6=z0C*KUt^xEN^euoh!5ym&B11^Mknryhn{P=_wXvybeEnyf<8`s| zZMf*&Va%+H&Iu|qA0EcZXza?x?=}IL7*Lu65>W$aTk$T$p5w8-S7^Bo-~E>5Lc1)p$kI>OKm_DLvx;lk`(LuGXD%>sSbjwSYgF*C5Mb$B|7@d|MrrAK$~Nj5KUEjL`9 zYVFe%o3>qLd1deRmIGbS9~UQ1wnpYmS(iEj6&*lQi9Q;A z`>5d#;(yH#!FQ0xn6G?~ZrI@5G3Q*uK`=Q4KHK@m5BCiE5L8-CT3zRYx9z?US6s4~ zq?9#Kn)f}2Y3S+}nqJIgD|_txnl|VdG2r71)x7nI*VzyMa(~pzIU2J~1c1z0! zoX4)~*^X}YZTJ3{`I|NV6iTE%_(w@xM+syJHWk^rQ@3cHJscb|hNIt49~T_Ia4Q8G zw!Yta!%q36es$Us2Q(-g{obg@50cTrt&@6HftoGC7tw3>^-FoWDO}E2)Vue^I*fU( zw0w|uRdC99U*2?=MaGGqZOJyPmDBPL+H-kY^?)O_KItgT+nRsaZz;&H8&-wK3AO>U(e`eGKkh_P0)hdpZJZwSfDbAD0aQ9 zX)k4@)Ot>PH2mhq2j8P;76QsoHuCPSkxNf?sbwS1Umr+M9iGtP5`mXi8f-ItkI``z zqzwyA|MKP4bZw~i+G`u@h5daosgtBvO{4Ll)=O|NzD2Dod3kwtjxuwDQrDXcC-0S5 znGKKD>vS!G)SlL(#Bv=%2klZl=_ev5xaQ{Os-B*&!sOQh!dKumHZhSx;o|A}#Sch0 zjhwgM`&OTp=~(R#22?ln4Cc}=YOyWPA7`Aa*x0ayNmmF%*iV+TY+%(RDyainUd}0UI;PH|b$a5pK#Bs53JA3~?Pr?KupZ-5`o*>sUclP* zj@UhFl6Eh=qrB4Y%WUM41r)DEW-s23t9i}keM4nbJ4RAf?6%v8Uu5s2og_mPYXNN; zGL4nW;c3J9Y5uiR^TV^5FM(2$k;nUQjyZg!Cll70bl10EdT~rHooSnM@Ph|V{qo#K z;43pR<&42P#0JuD)9cr=Gb{JJ1txpH{KKLU<>DxmID+hrYO2PsqNnHlBa7AA8m}Ny zc6PM$>g%bIl}_2uB{i5J=ktw8`0t;u&9d+P$=fC)gt_Kaj~URAXHZzS^*5h@V~&;T zB>iZts3w2^2)yed3fnR5RR|lUBPcjk?UmkiJ=8)mbfGPIL_3ikM~9Oq zaH!RZD=&rT8715P29K*hx1xe;)o9WnU2FROZvWm!v%l%4n=vzOB&0pn_hK<5Kj9hY zYu5~Rcv;Quo=0o(W{XzIqt>hOa+VkDoSZQy-p%2U4Ky{SX|F_%`(Ab~=8^l~U(XkN z*~6h;at);<{r&xPzBajtc$U{i3ATGvtAiBrna#9CD{@IaU#Y)-FcEv_@|g}s8LgiU zeOT(vMs|!+VUi`2EL%=TV_jaJggANOE2%28-?y*Nv+o`D*~mSUyRHz z50Za37Q5!#)k!Wbae@G>-)lYSPYp5y(GkNaqFBoa5NT;?xd7fSGvN9bO?jrl-xx*9r77ZBM|P`0 z_3;AT8ZU2W=Bv!H!#?Y&sEFH6l&BI{(x!BD&TN&8ygI(5>To-V@jH25O%djJyHhCMb;5o%iXRS zG^Ca_{*141F>c}4u{3vi(y#tAi0#krFN{V<{WA*y*U<^tl=(Uq3@P#{tND}x^GHD` zAbvfs_P?*L;qlUbuGJiu+%7$I?NP2y+z|Wj>@nEf8V51<<^hrn{+D@uADMwi2o)(4 zved8a6a0M@aQJ&+jfvWs;BcdLc|a$nQWmStDPJj{xv^r1yd$c;EqSU<&xsy`hf4UrA^ijA z5OJ#)S0_GIF0S=vh|+s=$jduAc#f5q{W!x|N}l9)rI3H4QmjQ9Vs9+#@89)CuYt!J ze*68;TreHP3*<@SL_5)tuu=4cyf_;l9i|mpTwLnq;1}_=E16>%%mP5876hd2Ph3PbENZ(s4aCinp)3NDk+1S`v z1a2Csf+{E})lJ**u(C!_dT-FU$UA+l%FAQGCte%LQ%rr!hEpo*G3OML=h@_c@7ELx zSzKJC^D2`S<@CSZHkSS9n)4Nf_(O4f?!=V<6)CA&W^(4;Nq*0kKHE`8l^%+*OwRpP z`91BWz>plG;6y!V^Fu-aAoGOCw-0}fO~3bA&T3`{nIi*zUP&GLI&dW~rTV{ItTJ^4 z3*%Qh|IHt>o2ybJ8v=kRQ`6?&-gY-dzK3kXR?I7d@)o#bE&3B9w~+Gu_9+Zokc<$* zMgLVB=aN{sxWu}G8OIFaMcdlO5H95B3we}&e^OD58)h4^Vt|Z)t|dPfTMse58c9x= zVsX;zr^gFchz}KAi^K57-&S9;)eHAvJ+9;plg^Hbv5mM(S{jzSS!~*C?A=rYYJYr#2L^~hZ)CncoS2Z1X|lMv zx%nlr>%qbg-HzGIdWDvAIM_0B(*`x^BP*T+{UE<=-{RGkx13xRXpYdygI%kcQ7KL3 za<0K8rOb1^2@TXZ|I7eUO2N0CCWXwsIosXc?Flzx+ibmA<&3XazH4f|7OMUBoQCGd zRNr0~0`S*xl&Z7g&Q=w%Bn+& zAVstrUl_u`tdhgUA31nh!FUR9ufey=Oot^8o2S+o+S8MM%M*v?S27`bpZYlJy z8QKR#pyN=6z*Izq!UXX)K1-yJHn=AVb088Oe zao=k5-dXyc)eb!73>bsHoPE0Q&3A`LSBHb<_7TCqO^KSz19|&+MAjLQpWV6vvIF;d zYa6?CW1Xms776bUhAi?=n^O7KN6`0YUfVjWuZs%I{dr3#4)AuR2_%(adJF|dmD#DX z77VE40(lf^XP)-pr64euVsG8>9fA{nr!cwXTE|>Ot45(4C`s~UO>$o=an8-&uPwoxLnC=7PE3`%d?=? zbPEE9U%!4a17NiD`ZGSDM_TDPYXA%)H9Nlef04wwY>18Hf-b5ft4AK+!x&ssjDSJ)(qA4+~r`-dN z2=#F+iiJz+PN*jY*_p3b|B!oRVi>&`Bj#r8o$9~-%7g;x@WJunwfjiW{;X1nQsOX` zv4BJu7Hdh9Zua<7(3t=2+pwPAmn5us=YWiNnw2<*;ISDZ2s|`=MsykF z&L7~EO~x2$0ICFe$r9TR-lL3NuM6ZC#$S}`=i`c(ShdTn2E=;-r!c4QLfp`awDx?Y zJ_37hYD#C>*^)eXy?RhoDBI^7mgsrI;BZF-Nk0zfL=tU@@7-}`zT5UhL-2U@bTFIv zH&!Pgr$H~9w9Yg!L%Y^;dp29+v2V1>;map~$e7{zli^T{@5S32vvegH0F2WF{V+L` z6+Rh~vQhf>?VH6m!d)-G5TkzBkl%(w{DC?hm#Bq%@}A-@3@EUQQfT}~R)oL~0Uo%Y z&o%k&4Dy`&$KC0d?xLbB*0wiQ;H!S8U4x_kBA0@`CWCJp@9Rzp@rYL_R4NfXC7E_% zS*`kqnRuHNMq0g!Z01F0{7KuozwEW=dXf%SAEUi-Q$ohYnaoB5m4Dn`gVW=WvzKkI zOU}DbRWY>5nck5G~*LfHY z;)}p=B@-_p5IQ^(a=jQ>sc_%B;s7^x8MWH*rVO*wR)$PR7BXB>x(+L2L5N_xr5_eSQ1Cst!* zNJwC}s;e!1wFz@xazAG^zU+y-RObBB4?8+J%W$>_ru*O;D5ncb17z-H#5s_&`QG0C zy4vk1P5M>yZryA`fo&ZgNyaNDNtfp#4d9l8V9ud%B36dtmcuB}5P+~4>Bp3aQ_|kf z=~o`rtq!mAWL#}u!F#P9 zOne+h`<1Q~@Q(J8wOf+*7*e=&d+X>Cyom`n-~mb>d%!|zL-321e(VFR;D`C3;qX$% zjX=;yTWrgNfv@Qwo9$SKCZkyK_m=}qVaU+=lENU!Rp`@rZr#k|FHQDVRs~DUuTafse@ zlSjE8L28Ni73o_{ZBfSZyhN6?vU=lIKQ&FNn8SzYt38R?^2r>&!bu0g+1c-ZLW;@> z{C5v};Frhi>OknGyLRSE8)clYcM9S*|9Njq7wd?NT&N<>Zq6OQ;>NFhkkqy23wl0} zb}Zk0gv<(}Hw6A3@l#VjK*A7TP!DwOXbu&-SnrDR&&)4H=TMunUFQCp4)x5u$kquo zI8i(tcfXvW1MGkOEMJNpdX1dcf|6kvYBl3 zTcVV;Nj@iDeQj(xt*>_Nc!{q1VUNI6wsKZ*P*?Gqhb}z3N}!f3y;|^#%IV6_Iz?wW|yqnG~0+C+0BFN&0$T zr^$Mw5*QN`V^Sy z<96H*EQRqwk_rCAT(j+5T>sgM2f(fbXkr}`$l9CptQU zdy?3TH|LUum;eP&)o5sFXckZt8vTHZZ8ckGUt``&e6RP;^J9d zN3!wIgz?qz)-zb4!|Grdl>$#5)bp8yi#An`x5(Ov@Z!s=_NUolEw};Vrbx%PYv~sX zo==ciNkaY3jppV}5)WPzf%HIjg$8-lr*?+vED0Ef@DqH2xVeJ%b6 z=m&t}wd385?nzhEL*;=9j;rHS!13f~`rqlsxm=xW z)_~?kNnbfqCo-#lI$G(LDmQBWUaVa{0+6na{nq=Woi6ky*G8Mf0^NnFCh0MOmkH|JikA6Cls1(A2|8uXOGGxgY z-mqOHR&efl&A4yiH%CWu+i`uX}`rzz8QA8OT?k`b6zEFJU0^*X`Fr@|Y~)$=TK(JKxqjnH(;z zQNm~u#pDwHWm;0v0Uvm-{U>x3==a*J82*&sb{#_4c^f0LdnWW38Q@K`AE4W7?+Z&k z2Hz^UXi;#nIBsuzj-KH!5eFyZ^<<7BPT$1jB#a9Y8>*4FhEYVL}i(G>SXV;5^cWN`|lV1)m7e9kq zZ?)Vp46n?TIHM7lN+}Tcou+L(k7|<~B|cKphrk{AXLsoE z7SJyzT-WSmGo}vkYFk9& z2Yjb+uz?pXlHpLD-=No$#W82p+t{77=`b|c5-D_lGRb*=^_VK5_f$V#&lT>m=PPNk z=xM{YI#uM;OWZ|auaYjjy5s0;T-KGgrYfJC52Q^mFdt#90stXQVv0qx2sd9`ixtX` zkk*8qC)rO|U5H6QomerjB&wEE(!Fp4SQycxD3s7?>Ayumh>2a^S z&Db@`yG)o1Vd>FO&Wm+PV<4pPp^RX#vgh!l)w^ zIm^oH*q`yW#oj1*hL1`4*@zGmuQUY;b(RFR($pje4^P4?A};@f`zmsB^7aP_O-(|F zHu9zPY;2K$pO$<=B}#ZczIte2V9-v~V`OBcktL4QUz}N+p)TJ9$lzXhc3nq1i}Tab zee@KS4kyGDdqrJHuUMDlqWhi<5iUm(&ox^d7m+WouhmUVvMvB|6ZQ)VZhzR8C~yF; z$=eWKhZ=alSyQypH>cJ;m9(A)#pS-cmkzZ9VrrhR6Ly5ru%Em+vobWSqa^yrpb)mj zRr>&4-vgLFV(We@btm!e&67cFUqSw21&WWZ{a;_^@p}tD0zlhT#)A)#IuB(hX;M`9 z{XN4`FCSI@I!!t6S)9Y#Vp3uwr05O%b`m}Mwd+WfBjNvS7I*|PONHYto?e}<-~Kka z?#tmr-sgRy*L7lSc>uzxTHhQhK#tmlz5f5#l`QV#BMgjW2+_jS--G$!nw(gB_KMtH zo{_zKfy>QL+8>5QZK94G$~`@{+umV?7a9?fFTz+hJQCH_ec#}uEVoK@c6(K3sAywQ zO_)3SIZm_pW(e)?g>|zZoJ`Ais2>eBcdL5N>*p@8k1#NH8MX=hfvNwwcsVFLJ3AZ2 zMI9Y65&W5<5_9n+`|`Zwt~-ijLV69mgr)PhoVB((8I5%Xa%t0ee|*^o$OY+JUgR`~ z)3(jj8mnKb-uSpSV{YKp0g3tAQrRe*1pKq%_KD#)@4mnMHxh`4W9!HrOu*=izjsud z?06Mpe3V0k$p4&Hpc&o)$D;JjkL&3&xL1lhRjX2^WS)$aoNFKWO6qMl)M*}g|86Ij=N*;ISPTF;=4VY->6Xjec@)@lziO!ucO#QhVBp>POlkAy4jI z4s+V#40q@&N2WZ!SFTrX($i?wb3(C?ihFJO*J9)eFu=E;XyLC}l~^-na5}uzHJxY7 za`7qgq*eD#e|HZ{6h7Pf>&U1-mmsI@dA^-2{}^EFHgj_BrN@{f>G$leGfSWyYxd4w z%=N1~MT14wjwdPe-mmYIy1_1w>s4BD zYCC)%Rg6q9$+>>|58Od)R)q=@fVVXv!n@FJLa5v1XG#vH3*N)Q%|8_HpCwiB1hRci zD@d+@86{h2T6HbW_As1avSau9s-|4J`&1=Yg5V7!;FZjf zG2khY_z!hIJh+c>9L|&p7C|e++jp|HT687!Z7oxbilv-Zkvs|dYeo2h`GK68mMMcU zZY}uH>&#%N{^f#ox3{4Iz$D<~{oJ>2X?lE+8U!8yR+&({)g#4M>(z4hP<>Ef< zTFEHClUE)O-RYQNba4H@hqnyS4qiC!bDaQjPcy>ODW4B_bKqz=S^W!P<4+;_T3hTn zG7eh#KvYVdKrVZv{+CM*2=GGn(L8JFUrdYTws7Rxr@QlaRpPg_O#XE7M8lb$>6Jkl zRu>I|9{$(jO;nL@_}B7k<-M{>PZJ}G+=2pXkR;7Q`D2y@n&u$a=j6fv z+Esu!3&JIM4bH9@c=gUvIt;j;mI>cfle|r3D-R<5YmaR4n6&32l9UheN0~SYxaFPo z%%!lN+JWalG*~Qs15cF^dLqZoi}2c*amEuR}s5^rh|@J1rN+ zYoV$>gA?P-wn>k+7^SiQdIH8&h4}B{x84M&N{3j-4eKf;{BF4+6-QcyD5k5W_)q@! z1tkhuX0gIGjs}d4=dVJO>_wGA?i%ceh@df1*{4nSx%TKBs`+_|?H%$1Uk~{mgSdt< zk(mKfVvjZdAkD~-T9`XL$xir9>REdJRK4dZ4nF(H%yd}c^DFlPMFAJHSgSy(zoY3} zUu1xq=+aG(GBh!xgEB*)d7kgw0@Dv(iemkHZh(qI`VT?@nV~n{uL!WiT>BQN9RN>` zh$!}icNur~5iBQ))hr1wg0f4w76qw4v!DQ!p9nH7$jQye5zTk&8%PX#^`<=dLD{!l zzwsrWtkL9^#MzD^nxQBES`iQcm=}tx=Gl#F^1)|j&F)VU7ufTI!FZ&Bx6ruLvIVoh z7T-;;($1PM`=LJVUC4c1i{D`DnEcz^zGut=FX@5B0~hQ zg!b=ke-OAr$fb$PFH9T0Rw5|A{V}2ER%qgzMXmhv2t{Z3+ca!sGz=hgJRoiSqTJTS z`Ng92CiSl2XlNmx@W3^fX5LSlx0i(1^O5Vvzm3)g_rRC9$n$G=^E|t9CrnXu(l6crPCNew`+A0wgkNKw$LbSdvfbB$`$@L~b68aZANyG=cRMa3-u+{?YRLhh z0Otd~a+#w%1q)wB8T6hMfrriEo~+=P8JPZ_k_i0v&p#Kp!C#=>8_X6QGR(BBF}cMP z^g3*jNWOooBhG8pnW@qEh4J=4QG|ujtl~BBJvR9VKlFPWsR|wVs6>6T3}wnkfViq= zbwL(XCKeR`&<&{7&iSO@4+Z_77>`kaeK5FW!wPB^HW)+amcoXgYrXJfErI^QlK-xS zkV&g8oj7V=VRHX6Ok&Qz{PJnL<8B$18^y8E_yLEc7{*sN>{YjyR^+Z99-t_20uFCe z@x}KyfvHb!tUHfRFc4W6B4i%%~K*mM7%`=eGZUidwu##Bzb(ed~NyN<`lvN zq*3O1Z>D?PoHRL~8}u3X1N>bfr}W0U5*8ng>_?lit7-$-T{I>RNGPCw_Q*pqtafbh zv;*8fa1C1-Ng5h<=V_2haw+UQRN-)1sD)~mR~ADyBl&NX5~r>VnDK+SyIcQ$X<63} zi(73DvDNB076u23fQSNB{xav!TPQ_I>H5Z2y3kMy}k25n#BIy#=nn2Jn=2IFy^DgvWxp{ zBj%cU_Dm!DRdxubC*`LFcB!m?AO_+q)UglBlk7Fi$$86k!})fElw0jZWEb&N!}JPU zeOE{nZu85sd(5#$LMa~?su=N~Dp?^RVLU}7?}tA=6<0DjuPo-V97kFbad^zUD!-Pm zDnh1;SDyCCRfg)HyZ>hg#3jV-SMB9?J$lv8eiuTIl4en$p3?MK>x;?%2P_5zd<@7A z5^=LvP1M5;XIeaEuctip;YywSCQZbYw-@9U{ z)PZ~j69^7n+1zCIJ3BbYs+3_Yn?0mS!TV;Rl4SwsL|A45>v{>;9{GxY&SQ# zeXDX0f7kzD%2|R7paaCZE1Ck$SSN{1_xJd?<_(iUJ=b`ldI-R>A5kh#4o`V)cB)>@ zRhi3g!JC`Z2rBn>rmGP(5<$m>2huV!taa;c_TxpG)_`hb4C>_q6o%F9VgSz7=8l>n z3NCCc4NI1ZCPe+sUeeNhztXTnj-1gNS1N`2w@p)j)@Y;Gw;-K|FuU34h4~q|e2&g+ zO|FUUi4El+J{jG=4ui;%Pn3#|bq9SpzDzw>86cJ|4<1uW38*m zPaYug1{aH$1!4&32vi2_vF2KQfzHF}_f-)H_lZd1&eT{-Jw$s%jk8CprXa#uC_u=r zr?FMx8L2(J2eeB(n#mA!ldCga3a_t!CFpgObD;|48m{iH_d6cEC?EQholFiW@9~3_ z#Ky6LE^oX(&Oc5Fg6MelD*Ycr9MaAbr_|Qtsv|IjUVdCF6BXyDLfH^PIov0Fn13&u zC{L23I?qc>NdQn@t|%ZZ<~b?UW+QHC#*n~k{U=5|zp0hW?0D9iIyIUN@Pe+g2~g~G z10Hg3Pn5<(+tqUARPXO@>9^IcvT6r}a$z-TRCRg@3A2tv$AGAJsY_Yxmaa^kg3s=$ zdEXb+hHTXF&dyFX;LDppd7j%wE6wO@5CqWH5_J_BG*qcBe*>tri6L^#!vv~ESGj$0 z2Z2Z6whj$^N{5~&?r2zG`1ilB{R8Ar*q+G;-^Ki|qH)q%WLwKb6z9=OT@tOB6SY3P^B0$E)1+27#&%X0wT_XeB)vK ziZa0I#$f?9p^zj_9=xiA+Gi>_Z z+uK{i$Te!!?0w2D=$PtgLqkJzvYKo>GEu5usG9u-vlw^{FK)#= z5)I^!3+!SEzjTQJ;`;af`-O6`zKJ$2k8jyESFtb5J`41eU>C-oGFLhs)>ur6)vUL> zksj@8kJY$a-=vj2p`MWh{2!m7Jjw^i7BbB{yp!8VG3%+Fel%2vVn5YTcMfuK0ES5{ zXr+^}|9KP;C??B|89^YBpzl@9*Fm$JgXc2QIpf9JWMV4c@xFimUhuvv`s8GD^fj+1 zHbAY4+TsILI^=tkh0M>JR=ZcVbiK%qMJKZ6We7RR6aRvrxNmcj&(1bCQr9%C_D6o4 zxMhbYOeozS;BlUiUT?c^aUbzF-ZF6|$}LsVm_Q$B!AU`1ir!poZ)m2R&Wbi?Lf_@2 zNzT;9ijb)_vvjB@KH~sHIe@odGo}Wzb1D2FrDRUyJfF)W-Ib35Rzl!2yYU9i0+m3Z z81@<{I58S&k^uuEfN2KVfKXrqLbzYi%!{;q z|IZn?B2QPDX9Ii8_>7C2I}VVe#F1+Y9vrR8+l&_xr3<*caaPD}3EN&-ORP^Hxn|Q9 zh1tfPZ)mbSCgaQTI$l$)H0_AviHMHQ12WoS*R${@4^G+Uy|&orG8=%bL8Hn{7DnR( znAs9$@M8nHi4Nv1~u!4gIpj!PGPLA`{%wm}}4$pE(c-<=9fAYxkBl#^$0|MpVy zz`YK){q#`X?*Y%DomWJBDLN<60|41Xr%80~Z8WA%bT*}fh6F?cUW1$K)s0x-{p~|= z*jkyOIOsckvtO-5oFiqbTmWDc`ui>aM0OPdputS6T_h`arlY+*v{)&IN=)hcY&W|< zl|KUfT&{MUVmT2qny*X}l-BtElaE$PGR^WWpvHE+KC_=LdrZoMd%jx>Nrua0Je*4x zbo(yowxv>QGyc1)OU9ioqe(AIr|u;Su*1c;D*A4{B&yUgw&BL|osXh_5?m<}QsU7I zB|mt#%Km?VD7A@bV39fYXx#{^6_D|&IT*|i7t56IU(#&XPpNlg+K}<9S7;9DHMs*123X<3&7W9U=^S5NO4_Hua+Le;2Ht+kg8QGMZ^nVWJz zH&8FC__mi))B8y@4sHckAP|9^IJ<#t{xDGH?9lWBeu!^r0xtOIvKQHoi$9mZ*r|dd z-Y20qx3^*Iluw_&{qf^R0<${9l506sn_;jSh!HCrR$iL}0-pg968muKdOEMI*3rQM zwX(7@;Pttkh6V^bQufQgO%8%voo7t$8cl%CpZU0m2;fbC4XVT6r3-=jiA1G*>6RvLrRB{S!~3B-b$%~Tn1-}uo)5FB!jWKd8?L}*6CqSFCP8nd;pH86`)R4o-%Cs=;wg0Z$L%}(>;v# zaM~Cenf0xjU-i@(id(|$Rd231hq6!exd@@37>C@CMy%%>=5fHJ%Ac>ECtTHZ`R|ir zBF(Bcld9ISJ-ZhQ)yA!mK*F)9u^Rq>Lm`-Rx%CRyeW4BMFzBH%pdHNBtF@uuq{77& zKi{_nJATkUXK*0H!;3XjfFWggKpbEQ^@OR)8xMnB7c;Y*rwfbJPxb>Fz~>GuYB|3d zVSNH1SlBC=JpjYoiGDV|ch3Cv^QYD2(W)u6^TyCO9zdzGbSqj*=mIxpUss-E5U>OI z{aX?6nTA1FilY&f+XI-MVT$X!HCD*ud0zmfolTiQrr_I2q7T zB^@}1HzV`}OAQEN5HNg!9w_y8xHrl<0B_vH$T14+Lh7-OKVEmLe3~}L;@|E?d?R-; z&0uulAM0ijT=CRO;s0=BGz^{sqc|HaCOOu+w48CFfuZaF$JSRsMcsA%N=b(TDvcsa zh$!75N(xBFP}1EELx`Y&gmi<_-93azcXxL;49z|Byx;v^eePY$%u8dBr#sOweQ4r<3}+Q_B^Ksq`sE4aA#zzl5IP)ht0kBK^;n^ zC@lhpz$a zbI^r=3Lho|i0ttMKo63HtaR$nr1Or^O3$}(=jg+A(qW}PnSbD4Ah$2ZeiS1@%A*;0 zR~~my_94_bRySgww=+QrJqwW60gw!15^M`<9YzrDCp#0__IYlqFGAI!4PWsRMsM-= z0Ed>Ih>b3vj}=wS&>@6u@8k8%8^|lgnRlWOI6t*g!#L$eQwoIw%?}?4pa(4vn>WmY=wWRpsiHw&QGVGX7H0Y+i(-wf zBVWDziB@d4u&Q3CFZRRc2VL*}75zD`DOsjo~UKha^?DJ=8=OuKA>Ac!0 z6WHpMp)KkuU9FLDB5<%H=6!rpWWcrtza~0)Df&+h54kTisSl}%ch!*U2F2h?hzNpO zay`D#R_GNJ%=Wa)(a1=dg$^Jgk1`YiX8{i|{%;?Hvnb63sYnCa-MvD4ey3e>M02XE zdE!KWDSLZ1C{W7VUtc&ZUO%ta?@LOYo|zHkSfRFZc0Ox^V`_8p@C57-QiiL)<8nPV zUc82^NCLr8OGGuyDZ8|^wCYXdTD|Lusq4jYwtl74-s}L7(wVGw$LiK`ulD>J23{}5 zU5U=;M(E7U%!fxuG^b&Vw>Ou}Cwp_+u$S2!9X6iny}6FvgMAnhNPVZMv4 zsi{c}f3zw*+u$h+9KeZH+Lkr;qerRlKuJoBODX7)r<$A)8Ov()ph4x_O*Ws;^CrRL z_S)&d=N1ZH=i3>97|@<%16g9T-Vbn5W~H`sRSM-}IR)D&KS=t(I$>3A(u2d7ZM z`{T0oUcP@;GH9f@68Ndie#z!SZ%nXx!nQ2x>`>1C=gM? zws&^?Z&g~oIq;P2T#d7vgkRt_+QrQ^)6(u*Epi*XSYb_p8_%CHZsi8PH$N`~lscKr z4Od~Xy1I9B;6C)vtpuGtBqej@xTR0_7k-y(1gvtvI6*YBGnhJpBNgg%rtvxNQ+H7I zVzzZ83kn{;7a;{6_|mwqYE6y&FLr9{z?=QMmCGHI!rGbT@mO_<(8? zAhpqx_-Z*$^HJ~$O_pM8>FqfjZ;0AQ#qp9xAf(V|4jIDeBA(5(1%fc!k;P7p2}v48 zpk1RSt6CwhZ@9V0-6olyp4LZco0fR_h>16ak~kqw?WJlot<-}HkW!m~kfWQUg*R=x z(D(vkHCg8W?0pQi3&#y-D9Vm}3+Z0ru!Au-hs7k@>v5)$CN2G;7xG-N1Vg>tC*qdWY$ zhT^Zik&_C_OkJHW4!(>={N_%bJsNje#$u-`@(NC>O?A??2a^u5Rb?$j%#)yhepwmP4z0q*pms>)Ig{=!a!sW^7CLBZRr@H7%n z&2Nuhzi~XV3&W##_iDK1Vm|?1N?M~P`hM*s}GE#5YR;>5+WZB?vNJKXZkO=o#e8U&N;n^$J?tFoX0+ zU0^?K1l{=ZN0+NK-P0Fg^~uHFq>9YKdY`%8RlK-B_delyOe)fd4CtzIuXvqZ`h?nD z5t=niTD8_g(u_gfa0w z-LKb@J)f!JS_Hx2OK2MhpM0RU8(}WQrtusWiegJb&*spx=b4;V?W|q3XL2-0kQAzr zJUu(ntQ4G0OETB&dtvHP3ek2Kt3V*xSuvviOV1h$4jla-etjAr(#jrYw_u7B+(e?PD%# z&;f8y&>_56^XMK*6>d)Sa4s`gu}Q=Mp4^S0JPIP@Ku&JheRUF7B)ZpVv4hoO-1Z3x zV|sHTD5HYuL#in51opS!;0He5um*F`rP8=1fVkPsB*Wgu(eWAHdkLsGTCBs-is1I) zX176C)CL~y)LKdkb&bb$&GNakm5j_*Zu-9Dn`%Uk`BZ7t*)v?@vmeSAe5xpmk#aMY z#p}=I5m8JK`3mzvPw|j%--w0|O-b7BPqy!1_Sh}$5lX2$#E%EU^Xu$4!XCZ41=bfj zl>!a_`1KP3YHA^z`NNxme<~-eGeP`RAO;C|s6rh=TL4p%i6;k|jqg^e#D&;@K|fwz z1{er5&Z6NowVuK>99mQ6D$THc0wEPJ!oGfGM^lI6nX01A=I6zZsk4&LKEuSnN=ig7 zL$hl6gV8Bzb083?K$1j{jm0fC-L^*XgC6T_AqSDN{{+#qvp{`$3m+Ss5U#NTRjFQX z|L(&z-|i;KVchQ}8U2!pvNG+~Iwc5Bx1xiaNm38JB#G?|TX=rLi%FJJQxxwA0x3)W2bdANa6Sy40`Y zhdU+r@#dFS0&!eqaBtDf1}aHSY9)#RgTbTe#XDnJa;eS4MKG3VI4ZlNZ^AD5h5M>v z1#I&egvZ6UNJbTX1>r5$0{W@LPDpi=u;IH0jjSKDfL8&HJT*H5P@7}%U zjXv&}%Q0>eAMh6ztXpz#rvoVnF%Y7=TK7E@6fAqZSo?0iVv4KES~u{HQdF z`yfXb)hhg`U|!?T0Psr7i}f}yZZ@%kyozr7i(BaI+qxk>x07nW zW!-cHMZ{pmELLQ&#_o|l6l=)iehb8LsbfkL6X`!F^K%1bYtp20wd0WMWIluCwMD|y z-}m?dU{6}SJDZRsem7BYL$fkz)+pixSX)2mK1IXD$A73m-Ipww%*B*L8@-HmaX^`g z=~HR0n-5pnx?UDak`K^H*U1~^j+lHt291b|7HD&hG)>sQm8q7`QKYYC7+Io~ib?~w ze@+Ff5$Zg)@c-=vfFg6*P2YU3x6tHke|~77UTvFxwuh7{rou2y`3pk(cv!fR2hIIz zg=%qu@vt*d#189vnW+6cJ0W+vQ%ctQqluvl^pA58HFCxOh4K)1(I+$jUW@FIMr0yv zD|ObgnuAunHT-Rb1Ls=;Yz8{mIqDTfXxj7{cae6GCpjvI1Sr)rLK_47$A+QX44}B9 zv{Y+(4!P@4AcZlV(ckmiBwx}?WmW)s5s@LAECV_}av1N*$;q)9^^$K6q<+yQHT1bF zJM6!9F=3QE(EgnU)POwP+;RZlBMdUUsf6)xaR+G3cTVmW3Mk9drxBix!G)32PbHBC zKCivs7?X6nb2QDhQ=LzA^DYZuK4s1ax;88P4ay84VW-uK{RXNPAoB@Hw%xBt$&7E& zDl(GUs2g=4JS7}_mip&c2OY^7L%UjHAIPr1Vf8T1hciq*WTwgLTQ1k#=iE%Z5=kD- z^@+{oK+1!ad-Hn8IOphpE|RFMC$k^RJ>-_A?El1K^xg^us|pq4X|h*oDJvpTy_$zU z#;4>H2lk4`vxQ3ZXl9Efpbm&S1oHpz=dv;y24Qf%Xdt@2UwEd(;LZ<|EE50zM6t|R zjzJ*xn**wqX3mS1l~pl~Bul9>Rx`6=&EG6E_`|Cd@YIu`+%UmlCi zm>oRTKESo0OQofjSJuG3=LxBLm82SJ*KC8K8h^(K|KI>$F2PY)R;A@4BCk7~k!%JI zeaVX^mE+-Fgj(Yq1dH|PAG^-I~0`@Xmk=hQLG zfk()O_l5RH-xqbfesPPmTABacWFc9MUf{-ORe^^mE%hEhX+Smxo6{T6B3ej3<*ony zW-1XsoO|{7Oa4C9u(oC3f8H1`)=R4hi%^J(V|3;PI7kf?a)C~+r*Re?Lud%>y44P=J*^RqIzjL-d~a;J+J?^~vRaoMHW*LR>AGpJ1z^wg?&X@e%1 zWev*W<}ZTvx1cYjV0}gN!%UgPCtBwPimoS{w3l^1hWp4JOuf~hyP8es20${tF!i)> zq&Ged1=tXgA26j<01-~T5as!oEcuH?KHtbn$(4yf>j0(kHjQlNKUG@*gzqBCf}Gn; zR$JdWFoH@Z>{sTZZH0Uod)=VWXlGvhGlXJN-4B68w?g~P9^fa}rw9+-K7}T~v{I{l z5p+{<4;Y@-9R%3?giIe+0pT>Tt-cHU0eU?y@9w@-Yuj*ganSa;i%X=kpUCSJ47?!E zx&5YvwOk4nogLCOKE$lC11KPUS2s~!U1Ow|1$P)!`VhQF2a-+AD4> zvROwvt1a?UyeFzH4^sJ(g5Ooq_H54cR#U^DNiBArIui>Hhs)X@ds?|LPlM}Qa#XEJ z7;o|hG&u}VSU4U;fO+QO36U^ljqDd{C7(&zP1E4vX>Rb{NQx*GQ#WVlb})DYFn|Ky zVoiGllC+xYyq#M=U^PnF4b@wRHESRE&~9y>?YA(>n`LrgAkeD+axzGPgzZtZG}nAN zC_B6^n&RoOXy=ddqqZx(3Z~cBx};OJui5i8Ma?+Ne1Q0v+Btr)Qm6rw{9^4v(6z%C z)6UY*sXQP`NLS9A+3hZGLeqRZdhq~j~rR?H1fPlRp z5OCq&SUs+s<5&zj+m@o7Jq2ZCLLQn{@DT}$H(s(!MFNOt09cf=0puz_l1vT?r-|o# zg!K!@qmsEjgcZ^SW8~wjM2iWt2`GK0>EO4UO~RSd%S?v+>Thq{7Wa^{OAtONKqmw6 zPc}#W@X?%A`p7xRUl6XBj(RQ(Bn;KeMw$kl7XP)AcHxZ|`ebO?{;oM`(yIcp$#rk{ zm9g;K{^8&LN0Ot1f=oU~r~O1wTD>e#Mk``gOLCKC?rmM_Yfv=~MN{>EuJ57KyL)~6 zfRX)WuyA7nvUt3?j3!}l4?TmYrdy0(_hg#NeALiWE26=tG}dNK@!mgxLIwnKgTjAY zhUC@<2M6CR48=pgqA*KdI&`MLS(V@eg|g$z8Z52qT42bSfrLYq2~iWpGwg1}%ERH|n5OQjEvJFZ^&d2dyDq77&r9;B$L?lEC2`wj zw35VUOn@dS*&tYkQGpZjY<7qns6rNJht-iQ0b`gB3@|ZVX`ad-8-iSKP69kJkJ zX9VQ@N5ma3@G?8?E;DvFieo84Gq4|I#;Dy}#x=eCeZS`9RllRRwE+i>S7fEKH6u4} zV|z8E<+;dltmRT%x-m#|BZO655HI-XCE)Uu78QTX{~81hB!{YOmcGJ6BN@r9aj?T@ zA&)@=KHU!zIaa~shZ{&VPo88s?o2i_qH>qh3(o&K8cKHs_+j2~=;CY8v}rNrX{LJ= zL&8gho$vVK#4OwN;87u06k}_5eB1IoNntPryQyNB0?Z8_+!@WxoB>^p^aGpoHmwK5 z1rvZ$H8b8z$J*1|TwFYqP?DUPlYRE%AgH*)o%D3!l2Pf?r%zeR`RZlv=e7rSqj_pA zhBcA7x_|2f$!DB=D4Z?3M30Vp~VF;;VKT~s9&LRnLZAWFu+dQB!(qVdnX zxR7*FDgsjNmV70qL)rr`Ld z$0zkM>U+9S(KDaC!&d)id?bE@+FrNqEIOO6ErRk-DMth})k!BxAD3IHK1M)-caZO5 zS=T6OO*~U^en*2|0Q|@zE{DrMR0_2Q0D)x!YK~+0oM)Ef-lGA@gYnKJz$WP*He*H`tfRI~E}#8Qh{R3+S^S!4$lbqt6AhIG86gm#EB<&)zva+5sYCEj47|zzbmPz1x zT=0^kg;g{`G*0EI<9fd^=hgFMh0Ple7R(^4_aDh9D5Sx|d#Ivo%9n3i3IIbXNA^ln zBj3BG!t%Zaak;2yQiM8}?~L6j0vYn#pcV4CVsmDTaVHtaH|KV>{n~+oGnB#=um8Pj_o}YJ zZp56BqC)???jW7u=Q0|~p;)i)QFrm%8atyE%j=x4 zNSzOVZtaFR<+GcO5}}yN$jH=)+=Z?ZN0M=vhvH5jrqDZh;#N~Dj30KI$#8aWI0P|~ zpDio?wl%2=qL`b$-Wtgj18~UzqqW&CcSlek{^LB2>M+r7j~43F3~zvN&)DP@g#UyPcp=#_HIjWhw8>glJtT3P2cb4{HH`@|3aLKxV`(ZUHey) zFEhP^Sa7CdH)Og4oQA( zMIJ)H@p!G9Bn`b)4wQk$NHXYM^r4`!miaIkWE`0a=3j&l-lW$McE`N-aYoC3ObgQS zGK(n=TL*`cS~no4XjY1q{51zqlg?nXZWtlnWQH^u(4jp&U;sXVy_pV120)7DPV&?4 z7H+0JC5`Eff0J!w*Mfq|GL=2tDEyk@IpUIjm>vd*b%aklIztnu7}7 ztA>yu8ku`Re}|ql0q^7BQ?>$GLL0_NbG5QI2`)TV&z+B?uPpd|#bDA<>nhzqB2b`w zvXlajmy_7CD2}~`6c^f$&iGgQA%rrQ82FDQ%c#1R4a=5h@6QLh^kD><4xqvLquJbQ_};9vqRM z*J&(p(-Z5hCR%F%tyK*IY5Tl#zP~9_-gLQApjD?nU0VsTJ<}e+(ah%SYiC=2EwGYI zphceyc)$B_?bYEaQ{T&dEPK2F?OMkmkYuHe(!Sd&)$e$+8`zEi6oJ75knhr+Am!g*G>)vQQ!EFRy>cFu+B=`CdVn*c%E^IuYQ>u8f->Aav452HG zSJKGIbaNbW5r95RCV5-VO8Q4o2R|w^fB0RCaZvFJh^+g!)Md0_vKF4IZ#Z`^|AWxs zA#J;u;VGfCE)!{Ae}-lqcXT8HlkSB<_n6`^V>FV{+lOEW5+N5C13OvE+?+NJ2HM@b zgMkm3o9%y-V+=v9y#i6HS|7|Fu^{Z12CqG>dGOKE(HE5>++S*0FseMEp=p^vK=`t~ zEXBn!TwGk_zCU@_jE{G0d!b`T1Qyp>;N!=S_RH<}T+jA@R8hU+t*WZ}Q15=;yhZ;Z zM_86O>cAw9$IZ)2#-eom zI<*m$QZ3Yco}Pj(ho9|}&jk(}_!E#51)U>bqno;n6x1VWv&vA3fVgN#G@ne&;20w# zH}JZ|XalGXPcCaJ%n_(fedld@AL$T89W8Wr6;S~||dX#^HSzba|n9Xf%L!B0x z#skT1bTZEQA4=2g`Ah0DK-)(o&ou0sRzES$Y((muHDl~8Sh3d3K~@`s3m=4zFyuj=C-Hz}VT%DXn$t@Hbh zcj~;npzyK-p$Cc zke$g`{`Q#>Zkx#M%Q35;*?SQVWx87u6C7DyUfBpqb$M2&b1~=&1Dc4G8Lp8;M%^l{ z9kmN@0m&;vH2v9}Fbw!L{>(vAp6rX$TG;-AjfS(tGP7%2u0&Su>FUzcz4egQAbzR% zHvrA?^21MS%E>2bas?oxIf^_)`H3)1{t~oIsg&DegNuk@JYoRf|A+0}<3QJkA}E>H z5^gdYW7D-72kyG_qf*XKDF0TVd3h;&pPe}$9^fmRYmEn-)~;Z8WQK9XPEk9sF#e_5 zzk_Ux-1~-f#dnEkVBTu;J5Wki8%hLIox|}QNQ*xqCw-yK2D-NxdEop|FlG2gOnDIH z8l)|fdD0#d=_J!X8s|ODD1vcw(3+TME0ZSESpKhZPlmw{6Pw_ zzpM<@jmTGS??OgLOjd?15oDqv(QI7C?|uXOjZwlLXg>k+1s2fd7sep%B!8*Tuo5dw zGlO5T!FTDeQeT@Hli3!@W>({q{%P+3Wo17m-#)a!jsCLQdN!fNAxiaE16B5ptnYk2 z?JRdU=AZVIDEfKLra+F{>1avfj#QUVg`Zno0X&EuY z7cWvhmopOz-gDZ6-4CRdBv9rTV=8VxHOA1B|Aq`Oo<69$ear0}Kl>}mcC9Ms*rds}RoOr<;|zkW0$2ny z$)2}*2Zt_NHe^Kild?oF@)cas)w${k%LkUIFgCvb?G`5x-$4eRByNL}#QpuOa$-PT z>;!)V4O;%YRg2t_+#YxJp?>^0vy9Z!0qTt&LO`IuI7zMv2CghK#Ml2oSX4hZJW|m9 zX46U#E3H6JS$;iNT$&Z(y1Ev~XH;T@s4BRz{rdnswZ4@Im1ieDuvN&b9F{Dfw`e?i z#dmI*&53{X=kI#?VwdHp1dm>g``BXT+LT8DmUxge)gv`U#{&DjEI80HTLr8?>-1%= zQ?h)BO)eeZ6Um`Z*u8!&$t}F zx1$0nNdi#V+OpiG6!e_=QXhZzrReQVV#Ylz*P6kai8?w@$`wH&V^W-;KQ96Bx){hX z-!8xh5;0-9ttf(p=yc!hbelbd74PMz&+>O;|G5!IAI}3bXDCxS8GEj#=+S2%ZPEV9 zH=?724|o6Bh@k-BRhDdN)VtMyibFH=QUU28O|!2+W`>ZxzGxU@dqw)U4=h_<$f5z# zN}tfjv<;vrZxl|h+gkpcktX| z+DQiSguNoX4y|DF{MPT~y-+F7I(*i@scgIL~EfF7rpnSqmhstgdy`qO0yW~E`mB4|ETB3K zbXHsQsI1BD2FGs)M|pNLZJ*@Nl_z-RD}=lEq)clZnN?P11zmzL$!sKei0yQ2hluQ9 z^!_U?5g{e83zeR%++F+3Z;AdMBD<$smS1I|nEJGbWFUCk7YsxFYYT$DWRIKH|DQDkItk~gT z$N%QXf$&J(tAxV{s&)h&5?327a$OBJ+NjjAFIm&s7xUJ!{zyH*k3w41$4XBD(MvNY zlvO->ZRzkZ{Q(RY3-kGv&D%`hzP~=FS2n8iW$aw0Q>v?yz!={$kk@CmRgD~Po&@o|y2S7P7Vf!@>uy6*i2l^!KZe^Z1?P1w%k% zf%;b9pg$-4SI9*Q$Kv7{ADX_mlWo?O6W3N}GnZWRO8^egdm$Surf`qvpFc%=*u*%WMd!atd)a&<9`iPfNUKFb+%9IpHw z{a@rfCh8#ZvkeJI==z08DJ!99@0Vk}vc(TUYvA z0b8E4p0992;qUdz`0H!jLH$Np1%Ky_D>^k%C#aRo;PXJnfkr%pAFeX`(SOG3fhn!?tu*xnW%3 z08hN+&EGjyBOCIb-qQN_WZRngpV`lx$xKNV5scuUcd5|36h%4N3q3wpn9&I>i4G>lDgOi%mFXiA>3?0JF*06onE+zs zD#>(Xk117`+=@NWT!axy{AZUk#3?Ely5W0SW~t1~kkBVR)DKDl#Qx#2SxCw_HI8rv zG8(1ou|3_{p~el*9D%w5vWy;;S+*P_ni&Yfq4DS2hMFVH=PpkwN)DrLU;R`(!+)G0 zjQOt=OUM?pYNh>+na8%oO2FEXeJ2oTusN@a@;L4mAl{J&buc}+i*1`>X=%x1c{<d!j#+Ne~AFZt{CBi^(!JhfPIWYwNRRu35yI7CCwY zv&|qN^}{qof4uR{l1&zn2wnmoS4RyQPRY3E6b};M&h*zo(FU4ogA-XhE?xZ2b5jXT zRI>lh$(}|F87{yJ48rSwJTB!J0G*(eWVA_u$+0N;CgZSVf&@@utRFJ>h6DNlECx1Gj@*+azSwYj7YXLl) z%r3$}DILyd*@K>%nrZ_2D2=*f#_)nSdg6`fW#|&)!+P%<8E0^X)tu$eQk_SjvLQz{ zH3_#cXn<7M70HTK&ym&)0u4i zy+)$80{*~@n){d3!@nWBs8vUo%kg#4Zkb1;uQa^q1Nz1Q6jxQvC z+bOqJynMJL1W<+lj#KiIUaYYQww;GPKg|RSI5{}nSOJ9!0Y|k&8bn*dnK4Eucax@1 ziT=To4L!+IzQDu&mSj+`Ps-#mxv$p=)@MJzLGU4HnHwLd%t&g{;el*CTiFYK1W4t| zKU8}aLV)cfr>lub91|A5<=&CSjIpw$;Ga~_bqG|NgN zb|pcPgEvrJ3`A>t(7VKf4xwTHD8Pj}v9j`EC!LAES1a7)uxAe`e$`jfUpn}g{+-3$ zVjf@C-~sgG)DxXZs|6O4?maX*G{P%Otp20(0jOn(cTLST$Z=!K{mZt$G?TsWqZ;~d zZc#`Fxms*x7jFFE4Vdl^s;V*iP7p1Jt<_agFxoi$Iu!MnFxd7PXn<9H>H@8G+RFgf zlH}+DH7a$eEwYn)5`7X(70Ic^)hzGDdUk!-IC^$?{=ukj?_Y69%^PUahP;9u*fh0) zhkn?pQymaf9}pW27d%(rORmex>$vEozDLsmozdM^WKL8nz$7k?WjWtXSHcAN>dU-IVh)S)cM z;1~?UkP-bn>Ispngnup2v#KSqtDz0jZcV({)(0ry&@gQ+drywZ&fu|TZCu>sPmmz# z)FCHe9Rubd?4eH1JdlYyZGaC$L_k2D6bO$z2F@>lhIYnt7A{vyJM4p92s*S<)Og+s z0QgM-c!N8JhK3Y>=LQqryC)3H;i&)*9R%&<=m2-KSdNhBafzoX1Idf{HvH^BZ!xk7 z4U352l9)m9tL<|8k5Ns9YoLy92Uuf*wvG-pXy5#gH6D=75&UkBw!m2jAlmQXzRftm zv4|LiclNuOZL}4DF0rY=JP}9*3t^k<0$o!Bs1ll{hG_vT*_*&kGpkacd6nV^Olb$z z0S}lY0gg_?%X=g+6csji+3I`IIgDMpM6qL*&+H7nWs`0g>K z#-Ydpx@{!i%ptXZ#q*-KNLt=xOlhJwz&Ag#!H@64bk};Ucbi_Hx-(^x5;sqUdgQK^ zh}OHoc`8nXOJ}53<@Cs%t=iLW#L*aBxT+0}t3C0w%uTbiN%dFqJV9vaG?&L4G!s_C zS#rbq5YA#tU@}6u=4))0{PZDsI!c9FiGcVQ1Dhw>JbXW8^r#g8kwBFTl6%jmco@%K z0J1m@n6%aTnvwE>Bb7GqW|@8OOMDgG$%0u7i$TD+FUknvv-c?=5_jBf1FB_N;2C?8 z&ChXOUo^AQ6-_8B?B3@{!DF8Skkx=+f*qn*=-e??!2Grt`5E~8l#S0D81PoXTSjwb zjTC6|c81Xg>QX68=O%kxNZ>q?N;E!R9~cH&*6f_8ZD4%mTMAys_qGZ6*84zT9uCZt zuul>trJceabh?0UNA$Nn5b!voZs7Tbk^@vBKao)Ge#Ixr!eN5+tf?03qU8gIhHJlX z4%^n(=N4fkZMb{>@eb3wEhzg4T)?woBuh@@l&AZ|p}**(NF1+I!BfmknbkcA5h&4D z&~`6S>1Cc61;2=3k`is<7*)@%I6U1bjb7Y?o^A}PZ_(<`{(0YcdEsKa-%9fb1h~j< zIo;;EMuuFAWX*vKd8(`cAgcd7Rt0LMyH?yfm%l^EHd?wcMrv1M8F{aHF3d_QWg``H z1i+((5Y9)s$os0SrpqR4$1J+7WBk&@THs?;0_A4juxe;mWb(UAjlA_)OC{3^4$Pi2)Hd1 zjHenrJxO`&v#K_^Ka!P?0h4wbP+bm!7Nf&)0VhDJ>XQdR_=i-;kNBL6s%jwFrIXMV zR|StfyAO#f9(MhzIWb#XDdrO;@|lDo9oO;nQ%J;sx1B)r6@E}Upj3&|$!HY`L`VC< z$pno`W|?ev|D{gCh*8kd5Aj>Xnzle^&Q4mb2~~1j&yz-^K$4mJaJ>=leqh|~j;HE> z!|aCWv30VmFfw6aZ=2@3#49)YLe--n!1pde5fT!R8@%?-9Zn}=*3#H4+I)@20H*s@ z=6Fvvqg-EX73xwSootQy=dv}d)x*?PrRJ57Qd~KfK{2hIuZU7v^Kl~3=Lh*)pckST zDb&_xx6a+_N}dCbB82S?3<_jg_}DSF{hN`=MX68#yr_>2bs0#-5Q9pZmEz{is#Rc% z@&(q7$tX$coq*lek3Sg8p~tQTrY04fS-UuMs1z7$*8|8taFE|!sY@-4lwYxb0GiYu z@2SsAhbSmlcZ9)(wgD~?%-myFi-J99x|@DMA&Lx1udrTr(?(&WGMm)NFJOIcJVv&$ zIXG^Fd*VxX1X)ep2V2kna5_4h$-+x% zi-GnFRC&$))BTC5xTvI~{3~GQh`S-sfmTwmf_f+ol`B6-<29Ret?|GU8>P(YD#DYPS z1zTny{7rrtX2Ct;3=*McuX=r-TfufD1rJp(l}%fM(U>O2UoI{5dt0XBugBm5dkQ@! zwdn!e4S35pxXaMI^DQipjf<-ROXg#uBLQWb`xUUm(U}o7@rQ$yE5+?pac@Rcg4(R#SNh+xA(IVI6Hd!fRk)v6x+9cwP3xpw6 zUh?}DKvsEI^BU+-pI!oz3=!Y-HJf}Ao<+lRMk@DbivZ;|S{W}fWw#$Vbe~M#*-BPs z5A-1P8U$u3gW99n6c{{LgC0x?W;zo*?M2}W-rT%ws|Xk|=whgN3W+l4C|S{JMAI93 z%_QVSfble~E<2PH^$oI>_?!FGdk+T*#oxRM<3vS%dVD_s3GNrKVW3vA^3$*@%v^0` zj$djtN`)8RJDnJtICGu6Hjp|~ea%@)vy;HaWLW;Tq4-86I7@)!DXK$# z2YG=u1@sLp4P0%9%>EFl5lZ4A`ChqDC;FNP{k(XG> zHe5|j9Us<`T~an}(NPny_CjX;Z|3lC6DNb6=wzpw)RxwkJu_1&5zXzUblZMCL~nC_eFY}3Vbc>AHBi-h^d)c)Mf22D zRE$2ncL|2goc#>wIrUudeK@%10q32pamZ!v3}IJPOI9)zy%$d20fxrhP888+o@+LE zfIeDX+NbGCtH;$~7yktRp|%;%B({Pn8p6)FCYfQ@HCeEI92EaG) zlS?NE^!~0VgHp^V@5-YhBTw>4k53%NG}m%3!H_mTz$LHQ?G_&er0S}bj$MC!m35AWDzG&aEoe^iAD(S zH=_8-RbQzsebtbW?df%Pq%1u+ufMt2g5rIY=`z;`nyiMt2{Xl)yt~atg5+aJo~Pe% zOu~cO=eaPd#Ds`lAj;$#m6LTOJc9DYi$7X-Cq;iZ5cc-r(fqaV5nG=fnIv1{c-_e= z9rlr=(@%4T|9FH-K!W{^Jo*C)RNW?*%RPh0<72Z;W4cj^iw#bzw20;Ab*@@fH1MBI zkin!Z>QP}_!#uPc$qmxH#M?(=rnZxsG9#w?U`iXh4TCn#5>Y$gD>xXUC~v{6WNFZ^ z{IR1@@k=u>DZZG{76|epqNAT%keNOf+alo>0Rte{e|(_~!+UMN{^0_Q?{V$?HCD-z zndq_q3;mLkXNi3)**48GrGs30`#qfZ6nW}=PBxCMfVAM5c&qw}3TY`Y*}C4&knNu1_N?`9T+x<@q$C+~3rl^gx)bGwFMG9)J7K0$ zf(Oln@?wHwE4BefjtS_j0)-{wi#ir(F$&4o)*9$nYHTd2ZkcjWaxng}D7n$`-LeH+ z=KIqj#dK+bfr)qJ4q{g$2W0=92_y8+a2P&WYECJ>=Y)9E-^!k@Xh0?>H>Bon87YVx zK7N;_-b}1?kY=*KE!2ADka4pXs1jz{R*@b20TWe-dJ&_YJazj$R*NbtE4{=u`swME z6{kIUi65g_oO-z!6*PkD$!m=XRK!&2jY*@PI9kp7Vj?AaJ?-P)sG$H9p30y1>GaVuR5)GZZcB& zC5VY4j*k0fugLwi6cp@4A6`x97(rh#($S@W$(^ac0(!Yx$=JA*D!v~)0*-0X7U2-y z^9iR*p3oWepn~d@57q>zpkWvCcBnb`gRA;2Dsv5-x3j_Y-3QEL`|yIg6F&pGKW6Lw zXBG_e{dx0Z!Q0U`dV$^F+YTk%yZp{E^#@Ctz6n}|d8D0Byo3~atTy;N4Ie@RdOScaWk zCj4FYA&kqyLR~#>fk-LaeS)4z%7o&@{V0mBe3nI26B@2=ZqR{5LvolOHVa*mZ2`z4 zMAj1QWcri9cx~upleaH0_Z_Cw;PLhDaZf5nWpJsIO}rkRWoMBgvoxeZ$y8otTiX7H zcN!iU>Cj4GRe!!pvUgd(Kr}U-Oj3KXnQH>(O$g^!l$Q^H9uS|R(0JF6jplZ?gdC~N zO-vrwpZ*iS9v3-`A+0nP6S(l&o_$#P{f&XkDsl{q;a{iNIi#`dAa&h)s*r=dcxB6J zKsL$wx1y%v;Y%btS=KsU&yDVzn-4#CwM$cTMsEebe)C$Fp#7cim8#W%#m^vTVqDp|JGK~4RL-3Qh-h;#Pf9EhEerP1}es}bIK*f@0 zPH{$1u;H7Qvs+zuP1dqBvdwIQ@_OlH%@+0b)$OdjV?;uPkYM$tUPKH*VP4=r0bQyw zyB!j|Tc(?^IMSZSV`U%*6V)4sCX_oUJsXN)5BRh_l^HAETYrM0ZO5Pyd|D$k~ov+Q|qVJIF_RC1Y>Q=eXmbJsVj;-%g?u zi@+eg-c-EbG+kUVawKp-4W{Hbt;+cDQzf6r8gx~_UHjPL(iI#9gYo^YhbdAIEv-0k8Myay z=Qx5|-+sa%dH`Mz=Xbq68Pjx{-b`Q&=360Z`1LJufZSxOa&^^Ed-LP%WqkwpeU6Cs z!+Pk!R&X%hVwM37*=ya;a^FvFMtxRog1R@2J}A#gtmXIqb5I_vSf+8C+iO{qgQqQk z3^P$OT$0*ufOocH8>#tQ&vu$SS)_h^hNZ?E;r;g_czLN{T=hLHOp7HHW6T_)z0z~K zT9+_2+(spC5~XHHtZeW&q{x{C^VgO#p$9Om8r=n=TgVld@Ama0a&~sMU#`(w zXhDZOGWu3H&Euf=4=BjT{U|DBcmz<|0?Xi_=1V_3w5_?^twpqi%E;`Kt9KO3*-T=T zSuxj#Dy-HY4@x!T&FDOnswK;--;CKTy>A*}Uv!|Po?Ftst8?A42|v!_^gX|{-zwc( zbs%^#IMtxVDtCBObB2pdtwZJQgWEGa?W!@kaQ3=*3AWM>iFL4dRIr%2r$+q$e&0-> zN5lP@Jnew8vZl|lrsi1{xqIW;r$d+<76E5?n|S1S)T@56Bp6L6v4-1~a(muH-0Awy zvnE3=DfvdT4^dYyd#q}Mb4U%YJr?rZtgQB;7e#kAq)_}5Jo>(8fqCxq6nu0cii=-| zfhA$UqH;zQj9w3AW{9R?ZU8gF5_Y@&RBF}&$~A%p2^!8<NTnu)o48*RLd@N6_`j)lJA7+R5J^kba((*t5d>5r=<&X8a*@>q!vHoPc$;hcZ z#C&^NQ9{Y!94Ms7ZN;s5Sm}8xTO2r_7QrH@dNVLqkbXS;)Px_AzBc0_#U3dWOVg$5 zk9gDAkw->EwEOX-zKBrlGqNZ0wc4#(!UNVdss6}=xi4FV4N1y*stZH=?x3~F2EOMR z$_%&*1jL@3c}!Y>C76OP5+-BtPh_@`^=ZR0uVtJlK*b5ObD(NfS zW+na>XdXNhs_UAV;F)v# zI7_g|-en%`k0V`jO83uH>Hp4H1GDT1=)m<)HER=;oqe@SwKq_kTN$4dC^<__pwv4x z7xO;aeTx0aSoGa@&fYtQU@qi5CNj&KEZz4?kVk}@dfk0_{DP&`HdTKIQ!rGmL_u)eJU|}i9t=r6pG;(#5WF|>%m;{L z&vWTL=iK({nn|mFd5mYzj)5K({qP-Vxd+)FXHxS105 zpbWUO!u5m%EwgR6`+X`YVTl6}b$`;V^AfK1uys)r@S`Yz@##fBLqjy(4_f@7?FD;{ zqJ09FdCcYPN4uB5u)PorA3wqe= z)3FAfCY9v8!cI45CYzhPa-;8j_X)pZfdrt_9X`fPH$;nnpSQT-b|jZx>$@c5RVErM zNZG98g=}qE(layBDWVc{SePbLf-cD2j|apymL*(xoC*+Tk?>vXTY75-k3F^6NzZ}LS9YU{SwR+ z_n>!Wu3iq)QG&KHFfy6|g|^=c6#J~yR3BJKp68mB&RlJiB`VYw_*}T^1&KRS{ss?y z`nh%10Ud>a>wBL4&tdll4~<`tr&`bM7FeVuCCwUZ@QnU5`2N2a1c48#s{FU1=WF10 zRB3Qo^SB^Bwu(%EeR#}}Q0eLEK&+&^qjGo^As(&qnnU|PiIZ-)kPe%+HkK%9HgxMz z-6~sN<&0yz^9N&CZijYTR!HE>ny-f4F&`DbEj`1;2}=vk%g&ZNZTen&3udiHoJ{9| zB80Pl{+Ew5e#gBu&ROTwjSFSQ!)HFZ1vQ;T6NvPI*KzyRi#4{sZe?ISne@dp}(0E(zxKz4#>_ENL_I>&lIQ4MMEcs_g-&#Zb z`PVEYe()Vul;*~tQu1xzVE+F4-*5k)t8$*e8(er79h`!qn9qidu3 za5x;|IeYIFbI!H)+7e@Bn(!nmu8TH}@m}|d4RIX3m`0P>f|snnO)4A_DAV>8itJu~ zw7AUp%zd{j?EvI?4D$uM`X?PsOibge5BJXOz_rv6ZTxv)FtD|P{{|MKL@M)0n7Qopr5Xa*--mD#IXXzgj%DDk$(M$v@;bsHMcaKR1$1La*p6XPxs>3Mf1wqwG z^qHe7b{vy1Fy+cqHX67t#&i&vV-?{biHx5`OyLV)Ej+Fzm4_(81Lsvw7F#z%rXw>ZN&g4w%&B zs-|-{cocNRvFe)4VohXgW=QIOP}LVas{28`(I$&guOTbTx4iw@Y&8k!x_Bp52ab>5 zydk&;$UO2)-jNEx<2fG%i%p~FOV=!@$%33sM&0AzU{W{3G(3+m@9M^#^P28xbrX=-yPc8j*}I zr)@D8jq4uv0Ffp19^t|M2QaAW6u9GyQE*QP;LA~2R>MKeYrrX`fDr!CVw+yaQx@87 zsZXeqUNOk}eTKqLaeBs#O}M6*nb5(}@ru!A)peUNul$tId5M>0)uYA%ujBr#A|Ac^ zHhF_$0+@)A_W1taVblxnvl4AtGqd7qEu8=xkMp!%@M`QxTQNQ3Aj|YLJ{6)o!~gVgZ`W#bHa9nUd2P%T2~$7l!rrYRaF3aa z0q@~*_00gjTepxSYXcj_ag1pDBd^?6fl`}V$mz23hTW}nv&7Eg*h6(cfysLLgO|(w zUP;A70pQ`$K^uh&k573Idd=w$N~uZ>5fr~CBbxw5z)F2G?yv~|&thFR_{`1C$2Lg} z?PO?J5n&@S=CqytQ`Dox`;uk8(9=e3bxn{oL1{^>bFL2bE&o_8+9LdyR~S3%pbCcg zkX`SGOZMr|qr?wK=)o%6hq3e(xyv{B@d9jp&SG}i0Eqida4^;|zkMyOs*(jc9yhd~ z(KZ9`yoV4{8(9L^IBS6+aeVA@H9f<#;!;xi#u=V7iPZ}pC8*|hYOBC)F_E;6PnI$z z8>&U#Kv4ZQSLy1l9PigOKdAGy5SqeHj=rEt3 zGdMJP|71Iobk;FE?FJC!6Y_PLC#_kzZlp92h7xEQH0TO40s#`zD6L~3&tdp!)TB$r zQRvx)F%%N4FT5#+u;9Fdn6|U?Wrp|d-;7Ow%GdT=_pW(^Rw3ByCPb?F$oS!%qLYZB zaG?&!H`rmNI{Mi9zFh@r1H7xh+r@?vcP0b#nCoxh4csyxmG_R6%~c=KVB<4e;agw<5yqM}&<-x+m|Y>cqb{CfR8 z;uXQKcA$k}HR)5DzPGy-gELga?RvMslf4ftw2WeZQ*+eg`15-h{WHO@;{&r=?v`Vp z9=R>x5t|UW2^f`=x?C9|PpU)Wt`~|Bb*~Sybq!oy$|1YJtMUIIqka5?&qv?XiX_RC z{0OGUFY|P6@AlCmJe8k-FpQh8wArs3Idx$E$mW!_fcH_e)BvEjySv+t zIRLIsPSa2#RP?;*HQ$E^P}$>b&{|-8XhuzRPN`~NO>OPeMR%l@6Pep`N=k}HVmf_h_;Rj?(?Kn4J;0QM(P;6o11YqGE(8~vG z$q66{UPRTpO1kXj=c3+%nb7%8%uVUvRYd&z+Iz3ROB zqW5s)&4Wv2H|sNiqkMo_ac{{JBXU*$$n11xh3lnx!O#0k93NvPnApSU3kDN3A5;(R0R53ieZFTB){X0dUVXLzNG9o2D+PZ~uG44xb`HFk`>0b) ztaqH|(Tfhr;<&;uHKMRK)6aQEn`+vRFaa}c*ZSdtaQ%2N(dIPXcWj@$(&s!*z^y!S z+Uzy+!kuzD@I7IsziCUd4%QRD+$n1GP#zTHUO|Pt!QgbcQ^E7fa z*_a0IyOWN10kXu`6NcMTa{o7PMWbH=2!pWp5*Rv^=ZAp$jJBH>&BpHuXi8-km>PcA z!(L9+M_WL?HG){VL$cQE*qQvSxa0=J#mexNrQ>R32R3xthX)?LP=VE2J{H4tD!3F; zErvl}QE;XDd@=^B2^!15z+dtAMz7KRhQZLp3OSTP)pW1Rxgh{$ta11T+pe^?*-=~0 zwM&csfR1W}f|ta^9yO9ok`MM<650l;I-F9nH$ZXc)6Wkpem=%X9uGIG7Y9opK3*X1 zXdnO?R%8sx3#b?sR99DL8UhKC82E2TK$|fQ7-Gl2JVn5P6WIit1BDCn6{UWUI^-X} z(2(J`r0~UOwHP&<;u@4J-k8jF9YD zwIu*f(I$w;SsK-{B*FNJ@5*7&w%%|*K^}(t4U``WW(|k^^ZPx4B_jTLCaopW9hggH zeB3RpoxefO9)BkDl?@hsz17olz^Hg-mN@SN@W99Qpd_})ekM>I09xJ9DCp@6)UBHG zv@vFIJimvKUlaDhp#6OAJ2jgtXsYbp!fW#QF&d(4c8NdWto1DU~MkCtU#F^I*RdwkaxH(mYO z{b-OcePOz;)=ZDO_r+Nz(@?o-H9htC2-gc>ffRM!ZzH^pn{{t`si*>4eOv}$lnd`<$8$oD4G%5es?>KfXuJyD@1))jo!)=G z{|pA#HzKruQ8e#q_0~saC1!T%<}Eb5{r20q^dKXqpOB)Jk+ts)w(tRmD3&ixTO~Z1^}l_x745*#BlNPkcql#++{9 z56dm%=ydgP!3~};Mc@Vv?B)i~kaea%!3s|)+U4b@v8=6bydX*b^%IX z)|m05^ievwa;s5aGPkWWCctiV=wU|Lt!I^HWDMhPVV(j-oxMHR^H;pqf5aLf^sIe^ zO_gA^re161+$crH`QRfKW;e=;b@>dk$V1|S`ls)sQNu&hIRYWvEgDTrtM{HC39XN0 z0H?Y^beVCz>FG$IRCXN+n51wwN%7X?c6UBLs@7urnPXb8#|(N$D*5guz;PdyE|8VI zVKE{0yiT#t{V(`toJVH^rRl^;j`GyXFKxRzyHzJhn$VneNlaKRay|54M z&eWp&C#A$F$3Gw^kHjmS4=ztM?FSs(jsxN+&K`bci0>>l88UjPJTuJW8B};a6v!vd zZvAlFWw$v&m#oa|I*D~9ygx9)RUTFXJznT#c%=X zW#}6b*VlVG1NnQ*>@%>3>MZ35qioh0-urNRsO`f>QUsD4p+Q5%e~XPkmGN_D;NeZ~ zzK&;hl_1*$dzD&JToO-?(93u_Z}*Y6rfkoF((NAxPrQI+keDmWSInV$B#NHY;(B&( zBD@}Q<_dAWiJDcW{<`EgO<+i<@V}Sy_-9`e;!L}h$Ttk1zdu6`4a{S&eijcstl#sC z3=fm~=R@QWU*fR(9FrST6{st!N+aN>014|YBdi^Zn~A5oRhykY)AWJ$mwIuDupDlM zHtDD%HIlL*zCYud@7&v;`{q{pasEAp>Nd*CES5#=#G13Y6R@VPrP$Tb!+mjT9Z9emC^&5ySr0^UHvm>`~&%1via=+%hh9s6Vmim1jLvo=4 zI#Xw-WE)ddwyl%1m-NMieVkCdJjTGVRT6%;fiI!#Rt zH1pv8?M(ulzCc6mm<`54!-JbSVYW44J08bItEGiiPJ^;VJe_lACJdv4Luf6^zbB-I zXtAVKqA{5JiPhRLdN6DgqHYy7HSp>4-(`p3*RaV?Nv@qlK07ombGDuKoD2a@iGv3- z%4)5x)#`icKV%V%I3FqQlMow@KKO?n0JamqM?ix1O@%KtYfz=FlE2P(_*XpZK4 z3W-s;Z_9%ITjNp>8VtjD!y%#O=@fN|UT|oO_?iES6JWi0y)tq}9C2pz-EU{wqM?iyRs^&{{eGXl{;9ip>q=J#axOaFY4geDD7>ReIX#{Y)(=HG`a2AEa#0QW?qAdHHi5C3Lnm0U*25}lWA}cUr|F}#wVySYj zL}V1c09Ap3LD^i>e9F52)Bxm{@aSI$5hJM6=>rqTK&Q#J``~WdRE@snj(64jTYCFmZ=Z2a9Wjwu-@N_$CnkI1i$nI|gB7u1Vm>7j$Jrni z5Qi#-h)W@hK-J1ke7%(rhsEd9No;A^^SXcb?=zCcyrU0$OLHD|68v^qoV3n*;hQBVOjuckOk0)Jz0cPigUyD54ix<)X`HtmgS?OlWF`KO5*B zas5@{c>=E&(21aaL6ykvs*wfkJ(w(WqK$K5K?-3nRhp%f`k&pig{3iNH^}7xSvgD7 z-))>ReUutp*$kSdzTW!eIQV0@%fMYZ9JoXHXQ1xffS;iTH@J3x0)Xf8M#;~aQ-zoY z4)sWvN6G82tp<~Quqb7BJhrzke-EjDQ4_?YlGZ>VMq6`v$s3o>qN4QF@Gm2JY&TGV zEEa8&=2(qL(VIx%Vj#6H7R+*F;z(J)`N#iT5Ag!x0mwy#k{)im?5u`}#RM%LYf}`T6 z9|Qd;M!YC+dU#^(jh5qZpzSmVZaAt~&eD)HZMqGstA_hGFCmiZ2pwTT)Z1d&vkd%e z*OhW56ORAZ`*n{_uc;$ z)Cloxwn|Egp`qtCjn`W!&oFp}FS)wsZn6?um^c~I;lTfSa!=q%0y;AredAP!RBMeA#NVGE!^L|oB+RmrP5(p;V=^`{=#&v zv(h6z7DD~%xnfL1>$9B|p|0c}g1;PKi7c9lU6JiaSlBG}e&**C9+mcA^x7R^Zf@>* zHL0L70kz1#(kl?Nm|`kUWhKje$^ zJ~|~_0MDmB?d+Q_x7K)w~8q;P0*u#jvYB9}v)00XLMe93@Cu z&P6Lw;nP0w(H=heTh1p&^MzM%Yj`XJ4H!N}Z~Sy5p&HNbYJSFhB!R^M^H<1$K}Dy) zBX?doR#;VezHM1pFzmQ(DQ}Pe$T=YHkjJM@hiU1%=izr7<4ajliKi70#DK9FOm9hYO0{;7k z?dKKegu^#AfQG9(Sg2EJx1skB?b!T~lF zhXGVnTAGBoNM+ge`Tl=CWKN?lgFOIF*=VviD(-d8Fudj8R zXw}ukHQ-9M7do;34bdqfff+nY?RJmK=Cc3^>MK&?UMo|qTd|%6@6Y7#6n{r^fMce) z?-iZRA?@x(=ZS>&$z=_?4y7eu-J6-pC;+`ue}Hpeinkw%YJEOUe90|wwWfV6|9#<8 zPHRYYni0$A`!Pv`7f%XU=$}9I+xg8KrYI8qX#~Xt@~JVg4dmtYT5Gw0pXd0Shy%S@ zfIH9KLg#~V#<^Fn4nv1Nks+||{ulLJJt$}y*249!2w5qEjyuQY;8F6)QZSyljCEVrq z;}RyLXy>>n3Qhdy!?E=vejC)P;x&3>>U1}+8EZK zmI>sK$j+2OyiIUH6ulgPtj*!6dIw3Zn;HhwZps`(7yel|KfI9wrl|5FP=lDR+gp<| zCI6_hL%D5ftLyYK9hL_7&;Oq|0jZGDNAAn3A`3?5<^|6P%pWLO!?gyc4P$Nb-~UOS z;TTD(AJs&4H1jmbMCN&4D*X&UaT1Usdygs={8irP{T~z_(8fQ(DB=|XxWaUeXSTih z+fymIavVzgQ~(uXc61 zWp}HGJ#Z78eE9fbskRRxoQ@9JIYX74&;MT}{11=+_}}2gHLtta;1KXnLGA+ z+Y+FW(=2t*aPLpw=1Oplz*hYAb9#P0RC@`Oov!;!bpYXYx;UMf*@J^fKH>apojyGN z7=;ugiRC)8?a$b~vgLscZ~Q74bW+A}bRM?+gIqd2&Z+!(SR6m+FFXXCZ!{36m?=M| zlf{Jxcg73$#sPIWt#5&cb&Pc$3I;S3p~kA|3}Cy-i8|#=uQQZAS&qc9Lk>!=bO@em z>2zy(T|oWJn<+E%d|4?oyok~1j-XcZ#xwSwk>Oj}0&d@|0ZK+w7v2ZhZ!r+4m0G=V zVm4MNZ>J|LlHap&aQ3T1U(`CHJj=77+Qofl8oV!pg8Z8?Mnawy^UC~|X@o-GSfRYI z-IEN3G@#MrVKr%RX5CHJnJF$f2tW7TJDJ`!gI}=%yWfqYqZ>^3S$WGIu1wLlTWbmy zyQyuA%x>HEzBx1FIIS$Yf2mg>?Cdq*=ev8cWV8 zV=WOFs7$&LwEjVfJao3?p52lN3yjm`@UccW@SA9Fi$R*hj~7Iy*?Zj)3x|ft)OIJk z@(kMp#W)Gm%y06oJIk@fl!Mi zLCK0reUlkCxB-g}UK!j_M+P*-1^i2|tN7MrHe}^Il~dSk9ZJ^_7um_(-c;Hk562i(@CXVAF_D7v|d1N_Dj*L7fno3#Nps=yHs4XH&T4i`Hl8>mdCi| zlJErNZJo}Q>Q(iFb`*k+0Wj!b0@#7?>7U%Ea8i^$g!YCE(;I>7d`s}t(?xIXwM3gL zOqAt0+v+_TY^OYHjTkBM)5w_#QQsw`eXJ&Af*Ol{ND+TWplpp%sr&i{Wq{NgP`i1$ zYZxqWkTm}Rd5{Ji-ZD#UAN-}=IFKWmjvpkbY+8}=x+pwQgsNZAUTYKV*`&5uoQJ|~q z(Q|ncpFBv{+YGZ_|Js}#r)AiZ5!7l=*kGQ!)UUSN5x0lrJha`qH8+-ie%AL5p#*EZ z@74DWlY>HyUA*G+De)m$7cv-S66^hx$aFTd&X%Rp(_o3G&qmUGFiekW8J#6Qhi00M zErn#p1{9@`JQ*RBPm&pcgU@W)@QwAM^x;8-pRrh>wd7>w-C2HqNSssyPZrv*9&dYk z__U_4q&RDd7nWRbRb61ANtN_0dt5wlll2p52892XgqrW8SAAvuj2CC?ILWj_7jau3 z7f^7j%_D*=%0Q?W00h-2zJ@_^T`y-~8Z1~Q>MW|yh!)HR3oiPxWokSrl4W?9YnYci z(N~Zzn23k~JyLGpZZT$KOJDS?IdKBeZntiUIezd%AAt$US*E3(v* zkv-v4`6+;TF>e~%N~l#D9*fFPVAJ1qO2wG_KRaBVtvLR1`+*V&_DASA6xo)lLoQyn zMJ2HtJcZUYjs7c3O#$f(l_||`;T=JZ>{Vz8)VP$ZH1p&0lHfe&DMaJ6A8+Zp8Add1 zSC}cm4c(vQIpKN+1u~oT17>!cvpjtLw;-mGE>8ig(FE4@3)=K`n1?t8$%ZQbq3P%I zN%33A{zEoiOduu}5iD?3sP^s$5Pf-J5yQ)s0%$?@S=+YB9KfQ7Cp!eYluW#%y{ z)>f$Ab0;WoN4Y*Ws|fqlS#)q*c%-DfNsrqH=U?&Jj;0V#Fg&>YGIzMV{CI(NG$x{t z3EU2G+D>O@wU#_~7Cfo1QYCUteVqkSH-RR#4qhwxZArbsaX|vK$!Yk^CXDF<>@YYO z=~p-z_-?0(sS~wj8iMKKN{P< zKeUW6rh#jD(Zc;`9S(l8s{0hcj)HLvzrjCA#1n4k+o|EEo(rDe0uK(lj%vt5y0`2c zL4)V^w_|)~`o_2k41^0#f|!a&ep<+hQ0cpLFyOf4x;Q^AS=J{dvW-y$QSaD_bg=!x zS78nn$uHN0A0f1KFad}kK-N_3g_VtIU=zbAffVbX+{qd=VHN1ADhs+Tmb7bI?D1z! zL2Of>g*@cSH9;-ig#n~h=4{D@)1UNYA9qj96cIA4qi;9ZvQ&GjBbX2sF*H!KBs4?3 zianA_QKF|Z1BCV#Jli&WSUwn_fuaD(hB+nR{bqVPDH#Ir`CPDm8{c=K0g0C+THP#R z48joc(aaVsEwO=LNXA$Kb1m!hxN(ds?SX1B%(zC8J{yolGQlZYX$HBp1`g+$rEw-; zWmfJvPEH560b7yinvYl5p1_>I-==`J)HBH*WAP4p@Eb;oTtwNgNSnAg?4f4Ga zxh^pf>@5^uQNw^iUuLlhDr)?k&zA6fS14Q>95~{lWA-M`)O`q7+wSfb+F;1zC^im} zg9^9bSt|!UDnL92uNti;UJl}bF_UBaN&!89;JwYpl&XT>Ngx-9Oe2Xc4ys}Eu?odH z7tc7Ba+r`L&C8N-VnH0{jM4!(ob2G{%_alG{L^GPgSJ|J`#$LF#KpoU{fi~Qfmmcu zfz4TSge`6Ae(?GF7-~C3VX?yrr9*6OPT2=Fq0kzy?z=C4kz@_a1lXwXn` z1C9DS63bVcXSZa>(@=cN91(VN^YvriP(%hF2LJyCU9<*y27P4vH0cyaG z(SzI1-*@}EM2S^*f>#;O4s4`g^Z{#diRK9yOVs>e1V%7EUyoz8@BC)?Iz25TIb^d0 z+yKb3e&fo0w%dnQ8-#-cxdlQ{DGC!Ph4fL;^RXqi09bn9d+P;1VRY8pG0-m}Mk9j- zpYI+Yi|XPLcJHC7l!+t{dqPqaN++2X{CRlt+*`cx!3}TM>dH*b`;Zo9OFQ(raH=)3j_b(y)S5Vu*MNo`5|4haK9yLvS`dV(hVcvZC9K( z$sgha-@SMDN*BTau0ptC*gH!{$=VYf@psy8mx&!EY=8M;p}mmJWp01?p3!+YD{w24 zO{x$GS7@G(KJB@UNev4_kFKJ?6(fg%;BG3G6!zJN%FxbE4px!^H*4qVHzy34-Id3S ze37*THSQ{>1-!&@M{D|)VzG&_DMB8cXY>BcKxMWDEhixEnJf{Lffj$aU@o2DCYIaeQrHGFJU z%rt$Qlh-OI5U9m&l|5r8gi0u63x^=r;9`RrI)Ami$Q@M;UcXO#dx!G=GmC<0VKiA} z?;}poVekb{IW&t2RB_hWMvQRgfPfm72uUAmMQepU+$@qYka63FQAu#+7>pNLJAoJw%R*p(v5>S9 z-1-_cL^==n3-o~3zK?)>3g|KjmI8832;?MG(U*r62@@GtNIR*3t3su8eQ$?biY)gb$~(R~ z89A~VeS+T!bX>rP<>p7h5b4)!(>9eCKq&Br-Hs0@b*kP|Q`G15+{i1~t%^UAJzEcy zk5q>q0rOuRD$gj&&|&bUtyLsf-Og02`9JH`i`)@~B*ey-gX{NhDy#i)iijVK%;5ab zBk2t3i_nQ&UyKlLQl2X!ZobGRWZ|%{5ukmfWj2w3LxJ_f(s_wz5ET&b%0bu~5+Qs? zSXip?N|vQcQkOJ4%T%~*vTt2&VB;ea%p>_OW{{3&ZVnE1^NaJ{_K_bZN#c(3?|Fyp z4@173UG#MzgpR=Fb^n;vS-lpj5U;V12+i%gTGywbEXqs!4}0no@TsBou-jVt5D}>h zgt{(C9~NkueCZp<7PD0=x1Wc~!Flh8RW_=auYY}?-5Gy(ZsKhNVQ*UxhS|o^@;~raNdW5{&v-m>~%RR!Fk%p<7S9 z#su4{ez_1bxeKwU&u83OayvXtN*NuDXvsU9kdKW+5imudcp1_eXB#kap6U!m-F+Kg zYmcc$f(AKS>z@2X!9Y1b;W46wqIi+;z#Kjn{-K1^4gS>|BWcV%AczZK!mQNVTOmSH zgVkur0D5fm%(q4uOJ3}Ay2W;R{-P87lX~HDQVi_vOH7S*d@-}P9a~a`7y-a<@Z-S! zo+t2N^0nC|Td#NFkEKmX_&HPedXd!A&TntBsL3UeC^zL9Z9{77PM@c}i(fe$4kZDv zK0b*CwFIPW%Mvb}UR*sBZr?$Egj;&r2O>J!#R*+E*#-_TiczN9 zlRoQ5d-+M_yH7sawIy7UPB8oep5UDgz%-PO$eWvY8irQ0v|^G4X0?`@ea%u)_nEQB z8w3|DyQib!m?q_)i3bTU7*M9m2SmC{wP40HfUzj>UZ#MKYjDKWs62e*Y6K|(ymZd8XUp6p;;kF$P4vWhaR|x%4rc#NJ|9;{B)$)>Vax^Y(WD3ZupY%cKe4%6@vb#S}vZ&Ik^CcptvFyKjn6Cu;nqxeIl-4@Z=Tog9dEG$eg;i0h}{WEWkL zPRILA=~&MD(F*&;6`}^DET}MlJ_0q4l`gwfDk+UrmP*JU)t3|m)w2+MI$(+WeCg}w z;XSER!3koDnE1BZblUSy44)(EGK>RU@!gja6xR^q`s+6P zn@bzU(-$k5-zXbhNx#~}?4>-=uTf<9m5tQe9a*_KF6Nu4R`2gq5T!`UAHSevVbZ*% z2d2V%Q;|~cDl80J*>Vz^a!ZYT5AuFv%Dv^z(_y5I%#^_jz4kaHLYqzcINp@gH*wSA zJTY@UHANVdWIKu#sq}~2%a_hLag0zqOc8P_+JkRmw;`)B3 zGL{yPX6DG-ce|36`ey0@*VnN4V(7fEqowG2?HJG{6yD!*vEQQ?7_biygW}k6P3>G` z>Ng0p9NsWk0&U7v6J#}8mbEzM&>ePb{^>SHHG6W*ZGdiOZ$9s12ORX^9pIp@4Q@ec z%h~im-a;g?Sx-7Qn(5rowWT}q zkjxq#3}hnru=TYR#JQwFJ>7;EkO`b!^pUxh$81-~dcTeL57P{pe}8r>Q#C1su=^ov zlG(9GB!#urN98*u@35rdtxlSE&rF`YB0c&%xl7Hh(|Y0RJA#FM)PtPn`ULq+sa96) zTxIaPEz55U{u@`$)T^<~>)YBVp<+AL#{+Sx2}HHfwM!TDw{eofi;JB5t!d<7ZH5=2 zpCPNmU$xjpM=dnOZvuL&HG8EfQB0B&xCfo3WjGIjA$BgUF+Sec3zCzN{Ic)!MBJ6e z!q50WFa)`%eRMR4J6)Uy7db-GIJ7mJxT z;%+2dAIF%Tz>w=uDk{mi+?OA!bZf0l&Pvk{oT>YLl?hb8MlvZvS`D9+HrXHc>;}>p zJRMs?l0pDFjO@S40_U_t+?fJu9jZ|j!0LR2%s`VxDvz)+cG)6{^K}|$6KnHZf&M*} z3nf7RHMhv(j}WG`o-;*|_ot!V%OAimbd+?4V*-(+F)#P8d?_WESSHq1ZCFh(G%Uy+ zpMEWGTJPe+HI}c?e=C!o)_H5cF;!zS-M)M}IzTDRq*y&5917Li4G`^YB+qnm`see7qa&EV8zch0_J z$1T8>B&x($+iKR!VpN>T7moUFKUk0X_(mS?UCiJfa^o+--7tKO*?Ir->rC6!{4}lR z+QEQ+?k0spkPq(iBC0>>{lpcFf8#_h1`5(~{muxyY{i$@8!WRmbq0 zepS8CdGARE;$j*OW9rgvs+^Zx@v&J%n(M6u)eKGbqh9mblV2kFZb{kDX|X@&#M~Zz z#z@}ZApdef>aLhK0nzIa}Si^nv`1uwG1ODDob{6XR~z{SfgSi&Gmh ziQt?_DgD0r1zPvD*0Y}ZSGbh|r?ds4Qy&wmR7fY@r7%VQ)aA6J?#0U|Xp5zvpiO0p zOyBI+DM6X8N^xtjBqUTIJ)Ltn?5f(>4Wz6IX31+ZB(Uw-zOz{G_tYJ7lNK7=MCyYD zS;5JlG=J-d*ye{*-Pp*c z>lL6SU`!fgtlkj09wCpyU##65A6nE*%9B&WS)x5iWr<5xkyXo&K(@jPKyYy=(C#ax7YyvDw)G!l^0b|3MB<($7LjLo0k0E(7R1# zVoBF+YLJcof@JJQRy;`b+jdMk(M?Fzl>!IgI?RX3r0 zFq;SQ(UG0rkcmewdqD!?u%4BJ#jX$%wIn;#f2Gr>RinHBEecFBVLeYo=0k7~Z@jQo zqAux8#=Jc0lS`eC+1KRo-u1k*;XvV2s8XZu`o(F7z5hysp# zp7fbb_GGBb&SU%^$q+^e^_i8#ZPHZV8)@IEj4kP^jV)I>jqo@JdAONb?L?}A7z&YL zXOIXi0T*{7jAQ0-e-1Rcz%*&F)_KHDxwhC_bJo??$-9|-8Di+R@rAGV&vaL(%2qE+ zk{bN+DA+U?`Idt}d>fQeu=?25V6yY!J7WcVFN#e`FJ(ao`w{F!#CaWidNG%<;Fe~z zR8;GO_2QYRqCP@Gjr<`rAk3qv0&a-pkh9?X)=4S8;_`bU$i3cxdhPmJdvqd%GF;VK z*_rOGG>=0kK3xZ^an=!gO?6XOWYx_%y5y|Ns^e}em&a|=oPW)rv*k{$vGCe8ocmdB z>9d`16YEr!7Hn-&=KaRa#2bF@sLHu@^9#y2N@#OAAY<5`k0)*wF>1L zU!ykNr)GdW%(hw!ZI-J2ewv@qPicu4j9J_l+2k*-H{SL87q!=ptUD}S)916=j$ZDy zo(R?!I1&nazsb6d1cEyzisxdxa+_a$n0r~|ubNbRve5}symRI%C;IkQk5`(CADL*D zRJGj`#_mvp*6$0XHCc{lkhW=WJRM3m1q|4MQcWCp(_RP(rDEENS$)J(jafamXxopk z$7rTj3*)$&Y~}nR?RD1yVcgHyQ$ROSL^0sHdqMDLKPKnW6eyhrh|!s-{eWGKf0~WCYC^c{bkyH0?Et4)n)UVNTfy8^!W2-4R(q1t95K)6xc@5G!>8*#Hf%Iu z8Fovrk+c>pjqEo2p#e6bNyjZ)-eoprrKlZ@0FX4K?2WB9lyh>sTEg*<#qd`jrit zreKj#|JJi&Dm$MBR8fq|aw&H;6TGSvzHPJ#YJ~RgHqT(0vCf#encd!9Q+L_M$7f^n z>bVh*go5)gozN-SRTvh8?C0c1F1@uOCYZa-VjK)>4{U4`3VrO&Ej#3V8ML%)dE?E*srAfKmEQWq($#6HYkT2@&0+r! z?@Sf@ppLP<`z*KVs%6XomsNc~1i`%p&GwCOxkm5vrIjaW?yXTX;bLFG{I z)05mYFvY1k)TTC7nI@&2K{FrFe(_}-fV0k-oH4h&)WQ+J(3fiX#*O56I`LYK&D(34 zklGSl9Oj-WZ!m6Mi>`9c8}A^BA^w75zUP9>l$@eAp2zH1hl=)#wU%asV?~*MHiIW@ zJQk_&O{BpLFS}aq`})29h4&NhZ{A2*P6t1)IVM~>V81R;(^hx%@YY+fm_#}1tXgs# z#W&;TzR_q11Tjeg>~)k4S6UL%^}w6 zz>pV{tV&WdZNZ?ZS<&qQJ!NY=9IDSz697)D*l{oJ2pbniYq?&upvbxg%Q*CS;+g6+ zpWd$4@M@KH8kcauyS+i%hZlEo$GxD)?7C}xL;2KX)nlQrF)8>h%tWP34F21DFEK~q z6|pSsii^+o;z0NQAK$Ce;*3HTOC_wK3sb(&N77J>cWI=nCM)P1k_RL97 z93-gQ51$|PMsU#?zd(1XHYWjSd8K^2@~dL{ZN>Wq&>R_9piK{gq+MegzxVe!i)(y332TqMj# z+*e-*WfDs)-q6k_*=4;|aA`k=V>ka{vf5O)fV?V<9a7s0NM1U1Mf;C%z~eA?x8BfQ z$2jK5S{n3l#|CXdcqu7`W%q=#y@U7Q;N&v^XGc|ac0(64yeHBjjPC2yduAgq(`cHi z#-EM2j1&b;4G$ZK_;Kmj!A5!yg@5Y@3R4gp5&5kbh z&0q!)RE)N<#NK5`ZlM&wovZDUJ?5SjxA}q~i-fwgvr8i+;DAHy7OFF7T_+Q!Bum-W z+-#_E~L4O=I?bREzEu&`D^onw0tI9gr%CwAdSzY-FgsScp6`Ud6D+~8L=kkE3+LmmFurkhBa*RYpsfW&-nvP`7X|_ z>G8RvkM^KhgT;4@bxR;snJ%9J41OZ)7_XKnMpIK6J4SeJ6;P3is4EmvFyj83Clih{ z^UPJYO&Lca6;^WtShiUFmq+HX9=q{_gnNKf&HTgrz!1iP27esjMM#yJiolimx`=x5 z`)78Tzq&peGRp6~AnV!XuK#{yo{BO(0qu^se-Anb@$VQxUP(l*Yp-e&FiS>!(1$WKqy$z+y- zk6zBRPJJ0y0^Lmc-Lqo&T%+EqElY_L3t}K|&6S^DC)AvTm#=ObHk?bB!`K3D1iv$2 z6;B6uz#S=B7i_va$+|1sT=kazP+)hKpiEQO$oG8n(lP!1yz${mRN&qk0vVW}bg*cd zyEJxa+dJJh!gt~r3TLuae8p6KUL}|Rpj|1-M=5IN8D76ZE^&^65j+ z1c?trU!BJSu*GJNY-WkFDpP-dcIjiP5>g82jogJf)MGnhd)I&N*KWOF7*QyY38Pvs ztkvaMTvhEqvpiRsuC3ol2~XG`M!~x0QnBB5Pt=f9Vjj3$JeFE`&jd0R=1+T&5xCt7 ziiyeeHJZJ{X$Gw{(cCrbuQwy?eNiDJBKj#^_zdV#g-^5WIDorilX{&TN+q^DJ7(uJ zJwATJh#@uasD9q%x*sz+f}RUT>Medx{tnH$d3n}ju9EU0N}q}Lqws|S({#pO4Zp=Z zLvp>4Xw8r2t)evotci-F2x39IqK!YhEu1`3KStuA2&NRO3FuWjNKV?;IpKqvueNuQ z&39hdV@1cGj(g5_jY_z5&yyZKv>fTSqVV~!=u~eckctA_vA~s~>cHM0Pdh;@<&$1+ zJXygW5$l3PRE3we27dv!cfs?U)*W<|75*||PwsvH+PRTQ<4Fr-NxpXOC7SH=H(Ix9 z87sd&X~467+Zk+|CV_33JH~XW=`Um03f}oOOWWXB!V8>#MH;InbxGiZ$IsOj?g~vI zai^oxH=k}tyfe}lYa_Kvmxw3oM&EkTVW4#1(GyIi;Z7z#HR!@Tgt%hJ<*Ft2OTA;b ze}kP*w$^yUO#5DxEzD%Hb@OS>7UXFP$nR1-Yi|fQN*5>!yxyjb8qiJ|X16Z$YTCS? zA3lH3wagZoaJ`(rvA!-i96T}^=da?o-+pMj2|R+~DO@Avwg0YO!L`2!DK%+=@U=0+ zF5Un*D!*^T%jE*s9J+L|z#a02N+9&x#P3sXp#o0XY89?c7?$Dq6-?x?MxAMcgP5@- z6`6-?Jjru)-SaUL)Zzzq^nHI_x`s4B}-huqYu0o{w?Rs7Z~CU-$|VS)-I+#DDC4=K2@ z`-~Z^QmD^^h=m2U$R?8a=HhN+dhmTO<}i_q8G_m^Rh+HMY%rl%sk>|e)?t^XuhC0v z{hrCmX?zw@28)238~JbVY=%^h{Rx5Bw#!>1gI2r(E^neVo#Jhp_vgsg8#lJ;G}~q_ z>p~b;IyBppn2F}dPrqQR5V?J}VIHNRI2OE7v$C*~rdrNX-f$ZAsc}pr zFggQ8R~KGD*N6^ox&ZoM`UbSa9-17*?SuOKp^+JYomFKwb(aJ|rGA=s4$JO91@fUu z$@XFF#Fbu)T4vmDqs&qn>fkduBz$tL4IzuD_0Z`VwX>hqq;_TmMFYwC4ig8}CntdP zAGkjgL;XAfD=7m*!%Fs|UIHlcnG!i}`cdr=d8sVW-Vr~*}VIW z=Z)X<#>nS2QpBtcIRK{>V~@I|EGa-VuC!EM*B^zLXteh$4{4+u;K4yaPrl~w@W#+9 z)tQwtq8iWb`tRjJ4lQ00ryILdV8q4YU=MHWmOPYG3}_0c!?UNy;(F#<64N;yF&|NY zsMTm)roMjQi~G|<8(*`!WmJcIE>x9-Y2%`-BFJS3G%>Vx-$}U0KG7SI8O;bm z8=`RA{9??w#5zPl2_h}6GREB(B7Tw#8hqRc^3uX}g}~Q|Z5qw2;FJgAuLS}ub_>6d zoCpe$z~@DIn$MjZM_PPWpr^q^U=jB(wFjkHN1UB_yBX~iXg8n1A&%irjS@wgg+GYi z4p)>y1|sLg*LX_FeLqQ`$MbrwVSTrHiy0l%=v7A{IVKyY8IK&^c6C#irF8$DrvR7ul=VQbd~&y zB+;k#FH!#XAKcw>;WHO;o=DZm5shiy_*vuIeYsj?fLYn}%bI^yQXZn11;h#fnfA4S16pYpBq||2bE#BY@wdD5xdG9^ zs-NP`%6*T*eTK17$MsZP%NZEeyOXgeh30OJ`a{x3nlfhgwNUUdu+H$vM!3RI!uX2k zUJ!_rj=JIAhMw{9;GNf3ypWsQRfq=;0Ek>ISEd(6ZFVimK&}v)Z9m%(6#SrT>G>4G zx9L*~&rpLA1PLce#|QV5FQ6~pyfZ1brYX0*W!rWJM$3NzQ>DO}2VC|5n>E=tjODp% zX{34pTgQxy_&G>Gw$vFj#keO@y{I5^R$^d@8ht;=F;Vx{uS2NHEcg}pGTw4`d(qw@ zpF6I{BVLZGu*mIt{pw>ExVAHK+h~eydgpu9qY5IJ-CF*{!R@uQijch^-o-Rfvi%;6 zKi<)?E$4r6Hc$()*P&FDUAypHF&6Y$NGtYB#k`^-(*iq#OjPeU`k$$ZD?KYn=S=}`5oy5>yL`-i zpT$X@YffvQ^Ck9BgX~L$*maC*q_Q3HEt_8@^17D11XTg;!^d;kisPQ(v!-(lfNKXP z@F{4#e$Hm^!m!Q1jJ>2}($8($#EsnFH!|2qZtIi}dq}A4mC^9M$UV(K%Cno@e6Zc2 z^upW*%jx+*8)FpH-u2YPE7B>uC*2Uv?W?~%2bGz=zQ|bNr|^&VTOLZ-ibSTL$L#n>r^eL&T@^!0Z$&+ZN4iHkqdCa zJxk+-bl4fTz!x@G0jr@%;D!8LU2{tT&B5^(p>mB>-VzXG!_Y3dNL~PRjq;@&#(_jT zdB6*@b=A6sCnxqp6}XTRU^sB>P4Qx7$OQ3D|uPD0ZKd z<;73sTPPHt-Nw&gELX4GA$d%!uGk6AZ-M#F_j=@Y4&QfeLp?;P@TPt_BWRAMCT!fh zJyT-h7@&%sCTscnHd0X}fD3ZX-LBn?j1vUjhR=2V+*JPfLql6e|B4cDRwn>SEEoa3 z+_g9uQ9*cjCiD*|L?!KwEUd8q#M0Zp{aL(bTc_uO^vFA9&jKEPU1>?{n#ZNkE=Yyn z2mq776h{d)Q0CH&ps-!1U!>qi!3@V1b--q%Oq29qyfeNN^h*HYi+*?AsCjJ_b968? zE0DiORfX__phHw`t@1UwdM*~6nALW|SJb1n>YfZG*M{oF2+R}SWzIVW+}kPo}1N~Cj+g+0a_h;qpU zO9!U=Nt?YQbaVps+ytMX?Y`Wd16q@Bm<2!9HL&iR?i&uCzg>()6`kz^bL3d*T`fZ) z6!X;`w`+H>=QINsLM>|6)&A$difm`AtALjLEo+c>-1JrftwS{5?4aQ8{Zdh7&UR17jPoA^XwHP@W3kU@WC(~porVd{x+cQo>^4X#k1>HF%V=vZL^>N(YS z-B*?x`Ov*op;w!DXnOB9UsEtYk0uI4Cy&DkT(z$5ehy1DbjfoZAx(-zu?`3Ft70Xq zRy!KEw4Ym!E2#92E9mu+5l7kBisGeOm7fpRRVp8lYgv~v&Ip^s(sax665j5PM^>Kv zmhU3#bK6gf0Ow0{S1t}Fl5dD|^fEm9sHd!s3bGUUP&8@h=as7oVvhJP;ikAvqn4z0 zyP_fwG2>Mf(@VxDYX4SyHx+Si!au->4=VXBoRa+(mWm5p6RrP)i;(d>rKYSVo` z^7qx4t`NyD(M-u1iuAu z^$fs|MvBcKZ-IL4tWDQdt#6I-tRN}N>PHz!`Zu_0j^jptgK3zuNj&>%{&;Iobe)i; zdrX7nm|*R17WX^yTyh!t@m|3{27lEH_R+KdzDr*^FCCs$I?EPL;%&l})8_3Y>F z9j)UID~awMKyiye0i;v$MbYy?SX}Fu$%sub%Z%5)&{;ywfbWu#TP2R-5S9G+ z6PeLDT@ftDeDJp#Z>Pf%6W!jKwhxnA`V6tRK+k&9B9{>5Z^`tvEs}lC{pNY21n!JA8f03lZJ&?mZXMH zcmQiON`_7>>Tu)^p1I7|NdX=2v34*4hl+R{SCV%!$gTJiUzNKULtt^|1Se~&e+`Awh11@F}xGi$-8V&+TH3Y@FFX*Pvg_GH53p}75Qx! z20R=YwMt^syZPk_WF%wubscQb)bXBM>P@qobJu;`J+HUKh$52^x0!$Ya6BtiD4qi2GksObhW01$n3W8o<(zpvAS)h#sn0-V&?mB&g-72%X^ou zZ2=qC64+@SOX$n8I-rJ6|nEkHhdp?r^Y$gQ=6re<#Uv%etsBrCxLM3=;yGWQnn&J*i{ zlYa{Tdgno?m_I8!81;v|#S40Q>A4Nb2BSCzw411;Ag~)oCSo_(;JCz)_G?Ft?5V}- zD$T|T&v61b=11$lJN}+t&?zawRS}Zj=rS$;GJB8^VBPagqo?_IEbALlgtIyGu7N|D z7!aZMNEVn*dNPll125GBlS zR}`CDt=4>&7fCwDp*c}|wXQ=$Xzs(Ek#|SqPz5OY{&XV6O~jS)&>jH$-rwlK*vLVM zqoqesC$#x3L>-SaXlp{{&)6nz%2Rx*L&mKq+1kfdN<5)V&!EqUFWN0Jk zXDjJTSal-3Z3lF(t6I`K`_D=Pt7~hN!F`o1I+bH`G29_i&WSE)=yDuHf-w9UNXj_( z>u-K#g^h(xjl1{o-`gm!^PHt3HrGktgS#352ZiP4Ngi$-oRq!8`hv#FZ@Tt9t%5cU zlbHpb1=i*#J%q`i^WoajA90#sxVQ`QuHk(bp*6}n%eH0{{~WCmG`DH`>*HJCAqu3r ziv*+KT~%6l2q61*Chc{j34lcLstd6AGu7MBr`8p?+w)Ass{56uUtyXaFbR9?aNSNs zek|atuHIumP&AW$8`3-#qy5Cm(gdf*6*8iq$vAMn?lo4@N=ZsUnxs3{Fg35i|NB)@ z&Y&<*7HM)9bh;0xE1{0L49HOx;b*3DJWgCI5SvEQGUtR9BQ7n>;X`cd(MICK-#&CN z3Hpsp6dqSqDweP(#aV=QQ)$vOrkZJ1ta+t;wLF6>C{_EyeHMrQ+y@O8Bjo&Vu3Vz; zIUAWdKD38Un`Xg36}6GI5X;21pANStbgt6efOd~NY{2i1Yzs^AuzI#YF#NI70c2jk za*>&CYF*A(Pa9s#BFq>9BrBw~v`}%Uxy6UoS4t76WB6WuY6);nh6g66&e1?mKV&py zrTN#E>$-%@%OQ`$VOG-bEruvISE_dblv3^gBsCY53&mxDN*bS!J{ik&P8@d|u#~s^ zVIzh{xH$iaxBA#veGpC|!fyxiRhrX_W6Qowv?W z+K0IABH%J)Nm*<0sHUUe${WaAUEv|bQx@lb3scek>kp>Q;t{6Pm{$(s9^ zU~XI}VG~uVup+b7#D~m>c9V^(YF>_KK3=!0IcOyGlqlX>rxpViz0$l>gTy^~o3 zIttrli$Y1pDwN$ZuM7Drn_#v#ot#(KJYG-jXP>I^vYJ9`aI?Lw<-&AyXa02V50}iodu8v^ z!Kg#1P-ldi%@u6btv`Bw@3i;%?#n^k#n#U_4fFOxaJB@aVVI{=Wc2f~ zOR-5gvNgJ>QnPGjK!hKvatEsygs%dz#NMf}>05qZiziv|*R|vwq65 z_8zM%*B(lLnHp>r0V`!3pP}2u7kmhR^u?7n!p$KfSUGqT-eMItI`T@h?1ooYj0b{( z8quZ$${LXzmke4!&nu=Jm(gqbSP0DAVMG5Kf8^QmTx7ha5&FJ><;fdC47h%yv_VGz zqe6-%wRvTD6!ch=y-?l#VEN5n>qp7=KLSR*{{)XYuijs?pl9&gj5b9~^}*3wgbtgz zD>Q?2`HCU)UfgrdM{eSKhJ4A7Z~U^PdE?ANGDnvzZ?on)biq5lbXja+r^JlB2`5o; zmtRk!yO|ZYVIwozx^`R3$mg1Lc~CP=j_@hZ{Bs%nv`B&ZC!5-4V2I5Os8Vf(IQe^7Eu`MLeHeSaA*=Ss0xc zn_38EutvR;-|~)N#P}W`Up(bAGKN)T_X}tv)}asK?7xFLrrG3W4s#VYWR*=A?B;Jq zel%$?J$(apgP2Cl=2y+_8~~WwbZDjpHiU2I=CFT(fks6zlu#nu()PL=X*BKxXpVBefIVtOq()==aKCqNH$o$EO9<&Ul6-Bou8h%yU2P zIvPE{sa7m9bgd4D8kIDPDNMQymb#n{(H_W1mXmES74GQj@@7rRsTx?Mda&|&v2rM+ z@JwO6(5>dcnQqhWB7*odVSl}w*K>bq_>!+8k6bqeGJjVyV+TGx&IsL~+ZIW!+Z+nM z698e(mlmmk9N0q%)SvRexo5t#gh{wq?OX6#uRB$uu-8N7$EtSubT0(~ zo-lL7>XxihSs1g`NI2Rph!_(lv+b-ElFK*qjl%1uslOM0x}(^3X#XW=I81Os%oKV& zWzA$oi#zz%WfQLCRj{qGo>>#UOKe0UR%gk5d#;tvTdi%nDsa)D;}?HtFn?XjC+o_@ zI2E^d-kjL^J@z{?@df!{VU=A*dK$e~~NHWMf=JNy9J%Ct0D>qIZfFmF~H zer`JS)2raPw$48hZqy)kymJb|jW~ZyZ$`N6?MMsPqa`Fuo;&E;kr`YgL5#Odx;(Wv z7hZVi{(N^)e&A^go}dHk)EAx|a!Nc*#Rjc#!OS=DPB&A$S37BSG>Yy<4X@o(hk1Fv zY(M&Xa`NlN$!$ysUR)IM>hX}A*Y?7j#;I|eY^HwTeC1)sRq|{Xvce!{ytF$Qg0MvD z3VnS<%%<3JIy_d8Qt511exk5TnZe7wn>RS^^26V#g@su1uqXVTHT&+i-Y+{n&@~f5 ztXKlT6mnfm$70I`v1wh!JX1Vo-4{`jP_RZVhykvkL26{7;l7!}3nLj+(YNjS`gv1) zflEdcH!n$xthzMNeF8>9_N&0BJ4Rb5>y#&eZqIo=U}hwkDVcE_z~w7>*SoGEMnXy= zb5x(-J+>`+gM8$5p#$*n`P}5$_*wOuKI(7)(q}c|04c zL}(5PY3Hju_i0Wnr$;g}ui!X#h<9`%=ZMj}g`Jp?masV`zmh@bM4#S?;b!Y;xkfux zCOukTdCMeU)}a%Bgzs)z$RN393PrsPQHaW2F?Kq+ig^WC#2`4Cy}y=Eu`W|N^TT^W z0d^Y3y;kSZT3`3Mn)-&K#XtHJ&v&Cd3TimB?!BugqIT1X-*3+PybpSh&?fPJM;B>) zh*c)65US5&P5B7?snNAYNyo|8EolrB6N0l9bx4ZU;OVwAqDn60Sra@t=;TUkBMnRr zvwH!mwefHW>OK4fcsI~ltfFaya=ZsDaEgU6=gTI36fuD5kqf|zOiearQ$`V2SoJV3 z3smVf0rprc(U2N;aa{8LIe#7S|J?yg>-#7*(43AGgMNu2(^9wHGK^MgFBpV0hOj$Scf(%cds!(vMr3wsQ%8~$) zvbMB79AO{No-tCg_Xv znlQ3(!Ni|j`8@Wq2fs_)5tV^6){P@hjFlYleu8@$NRXpoq+bsI_b4&im`!XfV_|*M z!(xIauYuzuGYf^NbhrBBS_kvQRtECA$`c0~pS|?G3l7gNWnJ3s!%Ky&-U#AP+f!u{ zWQ%Wxkx^+p{>ffh-_ov(cNNim+RhDD+HX^5B<$rz*mi*L!F@dSYx+!uF} ze9YQShDwpdN_{@o3}v#wB~8e8b2v|LPUTvY zKaV8EAt_GiyF8os3Tt){6X*?rjH40sgP{nN49=2wJ4Y^HO`#6;HYdVxU>Kh*H zYyNnDV9R>+jvpgSYdD<`_Y1Uj?}lsjX5$XZUrz7^`qo0wGx$3E%WDGqa@indz9SZ1 zT|}W4fWBvwZ0O=PHgzhE!Des#jynNnu}=hZI6FmR0z zQ{3>+w2xBjv9IQ7^~)S5kGE(zi=G1%+CpW?=3XCQ!v-mZ={7EdTklC*9;{9-6ufym zIiGFsP0RnI_?V+f!ylW<8nu;{7DQ}31oam!XRcfoo^#Z2K~{nJq@QkkMSMcttQ3jm z8VeK`*-j)!zTi2M$Mo>NG z&ZTDiR3G3O<3qry^sc^^u_S`8RCOa-8h<6e?e18g-*4fB)!ch<`=VZ5?;}EIS9sv{ zZ&1_`!A28bzIK8tPo#xIp^tc_r?yTG(N{c(qmO-?qFYOflBC5%U0Ce-h#EF6udJ{4 zE{UGn_&Ka!25$C`5B`}QhglB|dcEz7-t~r&o`rXq5{i1hy4{)>)n?#~I&Nqj6i++` zGRm_fZwt4=o(lgey3pzhyj`m+-F=I0c1>^*JmY(pL!lBgy;h-;_f93% zomJHH8dIdoJT6o4;*BcmVhF4r9Kx@bDMlMe8O}Xe8t?7|C~#jr-tI!B%aZl#R?~#t78|wI`*-lY4t6VR&!koxz%!i;)_K&nL zRp$!~vywsT8#aPH)KlSfQve@KfCof9nbY7h1ZnGoI*yDRe&*R zYUrn=W)7Ifhm}2?H#ri~{3)M_J(u^F(OdAIR0nk;J>@SldD-qjA38gX9!ZHS&D6*d z#!YtK^%F4`Nj4Wcnid;{TYWCQq6vDT!%&Ne#>O!bVL8U2ahx}9I18xV9Fi2}Ir|67 zO~gho?wc;|6QVWfPwMJk>2Gm=^o|-dP@GLlQnyI*!krdVcS5MY3YlK|VC8epbHH>9 z^KDwLZwcACrK)~60Tp|qhIJkj=1aE%pSJ+E;X&dYT*n1pxnXacJc|kx0CJ){fPhHG zA+3Sjh~qRDNRANBx_g=bC1F)yA|bqTGcr1*&N{5PK^^S)!rL{^r<$##49LCSKX&Ir z13r?Z#C4qNAz}$gweKy`?w2d%B=Xg?cuSXY`VOG^u?LMChrc~jcB*{*@26RyZhs*H z+HV315%AzPZzgFoZ|LW^^X0<@Z`lyo6U~SPq|Cs8p$e?v*sCII$noW9jp_ORl+LmF z;y9o()>J5K?fhB4Un_PM*H;|X+YB_StyZ$56$0c4a=yzda?#mtwh#NEB#N!`&hp*? z=PbU4?6<3Ptl4F}p4=+zh7yyeBo%4Z*GJL=tpi?H59cqih5a`%YR=BPwL<%HpdPFU z)(&klC9x_~R$lD=Q;+dxn-?eRUvdsLvkHVu{gXLfae2K}hR#!^2}#P3e0aIz_?=kD zSDr1TzYTA5^0#M^l@}4J*?=eSeSvQ@3O@fXXg6RJertpd0E1z=Sy>Op#l{Y zTd}tcs(AEF~wG(bp~Dvwamf8 zeJ0hehO{I1sv8|WhHDqOIz+IgMxI3&`LlFvkt&HXcRcTDDi0%BlOcBzq6iojy&yMt&2oujT(>q`C#=JhK#|aOl)&Rt;a?;Xi(^=2?oxXWKEk0Hy9=e42xEM z3dFu`_Hm9s;bw22p{EI>&JhLp72YigAC>N7u2zE&4J&@ozI^;D8LQ0k+3i*EK95M- z$Olh0jkl7p`~E9CC!6dYJMw9SPNLTY zeEo!ERlu>h443{;@%zuuv9Cd3#FcLnPhII3I3EnZE)n&Yg9n(OXxk5UTxHxSG?f67 z(#r>Lc_5<3E3puFV2$Rlp2DwMqyf=%#DttDW&Zg>j6y zQ*g;6KUEbAL}^!TkJZ#Bt|6Wx?G*V!;meCG&DUeLB_5Gae)qAc((ZH);RLOK%@H*}>sEaDv&# zsI_Nk6oKd`M45Uw6*)!pnn%TqU6D1kG|!J}(%2eSf)+rxEv|BVv6hPlE!!p)pcH@h zyX$CQbMsLd9c9YfRBgv{*Ty?(t!C!1B6T2vmsh-th1+xCs_49 zca-Yjq8w9US20b*m#6fr>(uF9V6s#u2Q=p5dd7Jrb(`vWJ4gxR=h1VM%bTPzxPi4z z4&siwK&j9wx#EhK7`@6KZaPVI-~^~MJ@ydAzST3JgjKmp@BwfMeu(U_90OiRV%cap zXF*{ACXh!hz1@nmv3fKx!BDkIVVo>#~@|TaJ&CP#2O zSiY90n4N!VH&o!?4eCf~L|;+pH70mV3oJOC^}8)v;LdnD(D@u4TdBNE<44%QkaK1B zGsLrkjJrlWQM_DY>>X_P-D|@yofnQ!?e_qcf-Z&Ocu+eelH!{J_rhabzzN^W1rgu)AJ5LRJax#+F$9_gnZPbb4VMsyvxw~c zO*mD{mD%I%de5j&%C?0LBa^lOf*EKsMv~R;k95&?ZI%T6kw!)YVFl8mYYUSU?vqP7%x`p zoRlEX<*^)>*I#&%w@K&qwotT(KvEfIk!+vS^9FTJ}D zGVt7b%50wW=MmkX;y#s@Byh)js&+o|;aw!H%AWn(Bjg7scYIc2P}<|$Syi53Bf-HH zQS_^VU^2zdFFpr14k1f9OY?X2E2ER&mf+3k37uMCC32ZmikB*EJgsJh6^)S>4#uvr zx>#P@ks}-%26lFvH< znx8Y8p+in{H5auP@FSXMt-)O}Qxg+J_Xhi@3*(Jr{(f0tF5iuN{8may4Gji2h}(48T*bUqWsQx?ZIDm{2$YGZWc;4@ zX^Nr5oRU6~#rXfO{&W`+)pDYVHKp_Mcg3`K@iEe&K)=2s|Lsf^M2j3@?y~ulFc2`n*Rm${L z^sI(cQt598r?st7a)ddGmVk~(!Dm?o_q_%iK~yA}Yt58VsDf`hsGCbY_EA>C#!;dMf)WGtGkzjuyo^tgCIR zvwa>r>Jr4+cwgG^u}Ftg8z(+Y1iVDq;>7D53AQ+An9|y(0{V9sJxmdGh%`Y~s^2D-_!eP7gDNsfp4N!_Hm`w&HlTMnL z12ajHxl8;2;1?EhIJ}0PG@A@Tm;C?Wtd5Y$0B$b)Lz5muVS`yS=J(>M@oRCqLP3*k zgQ^cYqy2JLPX!8_%*>nh848;^ zn;x7n7!n-8p86*~4sutaAu0_B&5o#53w^-OM?r|MuQS&uv(&+?W6)bg6<~WvV7e`Q z`Pf-h<;R|cF%yT3RwGDh7x!SZDN_ojuF#2G(>4}v@blx#khOAkzAO;MdS^Y;ws@R7 z()GuNnMVd4TMzV(03j_W98b`X>T*jc(Mt z3113+dZlKHO)ff4V`n|P@agEAbeY^0HeH1F+Nzh0O4%anS1ZEgzv2kG8y7ma7QkAC zL~m?926~}Xf8;hnfJ@7I3h>mWstxr=0>Fr%5p)Ho+d}1gN|y^BghO2fSM=)4IyTo`w{BQ?h%ZaqX4pSVnz)A$|9nlsK0zDh+v zHQkkfW*5vAR$~B5&i6Zg5^{U4+?BVjtEc~7WpJA0s#>Lb07DteQ_6=@{o-UsIb$29RN3MKM{#-GWwc`~>3=Tr=w~02%ewub}2xPkZhCRmj zGYL4;bg>0A)~H2wx7A6Lf&T9^;6)%OIgSDdr&6hxA z4m3r}NFvVSs~VYku4q6Fa6qZ({xIJ4wRe<3wu(k+^xSK1PK*nF1!2-;BJWxARkm5{ z7LvKIQZsamdfXI~qj%Cqvz#|^i)G{NW%KWWoMN<7-hX3J$4IO`eHoI+;w@_ExN2f}B zCSl=+7iXyvQVg;6*-Wn~!Ym9}zRII{ z4q8&qgqxjni8$j!?N#y3H_!we;(p|E$R!`UHzbzhmW81Go+&~mv)8I4oUd;%u>$oham4mv$opYQiW;NDJYaGdhQFx@IAh0RR0=2|(T{6Sn_AU|V-^smY&MXdJ}O zOK(f*ZqzF7@Ev;Ar&xctotK`nJ#K_e?@F}%jq|!o@}rI}&53=L54H)b*>h%g8@L{7 zWb-3l#nlTzQ9j~d0kwqxa!{99M9H9$1^Ls1z_dFl7pqUYBMV8%3XK#sS)2IW!6@a` zVV*6Z&PzVs?(Ml84bL?cv^k z!YJ&b|MX~{%!NA+<{d0l{sOB{36f)vXdygVk2qRt+8o}K72&%-zFl-6pHsL$E+Gl6 zdH6Sy$>v4%vtt0Z;!M?U>qjU1RaUtAeH9DO^5lXYH!!3{UouycD`B6zf6{h2YR7+(>>bT?;X2@x&{7NQ5=kpc^=zI;v6OVd| z8_YH$zN?A@hIw4*eTSl_m3D*7*rfFafsJWwvRFaStXrpKv>f^%3wCyjl8U^b4rjS^ z4Y*#?&XV0p@1 zH@-YG(ZSn0enO$+Fm}ImcRqeqCS^m-GhT|_&BeUE_V0`q-~RNazgUSeU`s%Ky72K% z7_KAne+>_8nGmtvXFFy#pg3s@iunfZ;qNYWr?Vnj-W`rorl_IE$TJALob5blxuNws zUTvtqcTi3nAwF!mm#WZSYp8;`ef3D!kD5&f%3;c}8_7S48OxDd(iXJ4z%UorcZ(ky z$o;|oXN5_I&x#zP*IM|;=RL_L7osIc5^ZueBYW87hMwg#bW;D?`*DG_nBICwLtaO$ zf(b?iy}RJfdQ;f2Hsv|~ax%4i;IbzfRD2t-PX4zR1B(2MF%k|!Ca2BrQ2R}Dsek?g zJ-TL-(buH`eHoE_`;yBMzE*Jtv)_O8y_eZG|6ri1$juy{HSHLp>|%=O+}aeAb8Gq1MN1_vCvt2K*}^^by?e|I1BGcxsH}b ze|s7qmCN4xr=@qb!2^Y!b-FtF@*zG2nM3{F)lV)jTZE|@1wqb?&3!yJ^s1YrRpoD> ziA2yqXyF?Tqr>ljSrD@Y}rT+q=VT#^T-ui)X5mz~&?Apllv{jmj z>@Gug+Q>byGE*2Nokp&KPXRrWS~52c2n=W1%<#7VDU+Xf1eEWpz@~ZId2QD<*^q+)8vR;|XX)(Zabfqut3D~}n#y6!mLoSu z+Zw4*Rye$UZW% zy@_i-P3=~DVTTBL9s6Z>fLdHD#L}ujhFU6hF=&K6!GJ#X}Fj|1(wnbt5b6~7*IZI zAupScl^xRYq$DNuo-O~dc+wq+;~#uWEC{%;R7a6bGBH2wPLQHss}pV9=DLn^YdH$8 z0w>QSfp6>Cn#x><@6*BHD^65e@_bW!9B;D+ zLMW;Tmrv*zbLhf|j2W>&oflF5o$u7+`Ds_XEM)E7Eg32>#@c7FPb0!d_H;R1z^G5g0MCVPsl~$MMMM!8c=zZpI+U%Pu=o4 zqM>nLuB1(n+CI8ZLPTfw2B&yfJpN@jv+`nE3^e$q>Z)9}w*y1!$U1R1VaT2Md5mm| zE3o?PYmLSaWFF#fW+l4m>v65KQ)Zu3or;@fkdb^4z{Qz^icyB7*3c ze|4x9vutygQ+A&_dan{>&y`Zz_}75S92QFXB254jyo}*bn=Xm_ZSOU;AovH z7PIQZ3~*l4!M0YfLx8a2&4^F!I)42w^%h9Pj{9DK?Qp)dQ8n*&yqR9hN5NqUd}DkS zPx#aBp)riHQZBbf72^o|l@v$3RAKdtR5%7=p$tFiX1Vc#SR|Lb&6RR~r`v7))Qjn5 z`2K_Ca#Ke{?y=72if!t4eBC=RNfS6)y!{i}UiL=Y)SYNiisbFj;xkgc2SN~ zN9jaFrT)sPr(NNfCFsalJUhe5jOqXhTQoo;y@c)O7Cn%%^$YIeKbp@YlviZkvFa%=nmXZl3tiuL2spAVgHy z59z)7HG&zrK3amwg-}L)cc8Y@?ZSwM7o^otu?a^pli0Lik5<}x)nHF**z`xr0RnVw zpkSO<=7sB^#ajAxe4hozjRJ6e3VbqVxY6O)T2uSUMtwMh&Rjtm&4Ks(F zhu(8L7z$b0T-16DQ|P0w(SJKC5CNI;txZFnL?OLiN5`*f|`;GOyfa5OD2YzvrnlZaU z_IxB)NUf2x@@=iRx3Ydyv5&keddm&Br-pBGcqCy`yCfqC{z#&V!uCTR#y@3Zb5cfk z{0*4v{(hGU*1wClYv~Smhy6g$BuC<*QhEPH9r*uWZ6z|S=~56rnyuxR(_#%!ze6)U z+tDSB{dZ#!1XsPAWW50Yyezs@7N=k(3fTX{dH7li>!D#8!K4_FeBe8B@V#i>a!D=)cJ$_hLAYqIQU?^CNmks#QSHPK|dFtHxUGP0}-@d|e zwiK>cRG(q^m&ojrgjRsW?+7i3vybd@8W^KKF zk2>&n_Nji(y(apn-t5U4XV}FCW1)|g0%JW!$x(`S7`AK8i*%BT8^~TNch@@Hw^DI7 zWPHW{|GE9&H{xUgb!X{W#!M`{h*tLi9;zhx>vEfqfvy?NYQDpx3P7?9EPMF(Tm1Z) z0>{;eLc<*#5*(4-|K3=<)(PRfwI0IH@>98Q=EFD4IkVl>_fKg5$6ewQc-$^>kEz+$ zg=K8hO0Opl0{>SGjJ0hD;k*9D%)>b2AG?FLC5(lIl^3TsRD%k0UyS{1nB3-Eri+sF z(@I)~i?mvVR(6*%e;RJBeD?A`?hv@)&zNYwH0DRGLvhdeG7=OvL zri-Zy&9s;z?F3ks`dCXIB>m%iU;i*h^Q{Yc!^6-PHCv(DD#88RKzV!9cB9##p*;BQ z`zP7rO=VBj1bEnLB>qLA|L@+lFDp{-nw%nts%3m_+E8ta`*!a&7Vr0yPBf(HRjs?YthiZz3z#>evYj^9ARRk^(i-D zr1|PSa4$7P<#EUHO}CjP1FqvfuqU9}#oUaXlQV4~$?23)Em3@wNNW^McT+ruSDaSKzzzMR&Mf>!`sPhg81_psF%g=>Lmm zi3?-614#Q}kYXfOrrChK;pA(I>m-G`{4BDtpE*B%01GAlee0|76Ex@$dcU7=fr(iz zpaab+?ZCI7W@RaLFa5C0T1K5KyWWPa{l_)H^%*pOt`OX6L_lbJEyI|g-Bh}s3Xlw4oy>=dh^TKb^%R@ljfjWi? zQO=?1L=F*n*v1|M%sa=a&G`RHl7YR(qWc=8_@9>rWD_pq#Tu|WBTK}XV<`38UfcL8 zP6yjGkADgxOkeybHtu=&sQ7h$^=a^&vo4>? zJ}gG_%itTFnK{oUNBI{GEv0Wjlp-^37C$7-j7w>fN(0rT;HS`Pt9KqSgf$hA>g7na<8s-eczrV`**nFn5p;zdzFr>X_9-BT(3~Z&iF{u6;3X9mzw$%+e_SapPT;Y> z$ldQ9Cd1^%(!|d;Y$E5s=&RVA}-s+H9+l5?w_9Fhb`LLiqj|1QrTV)jLHshqU{0?=Rpph*5jlTO%8g)r=)qG zfBxgDNJht|mXJt(ga5vD6q-kjb z;pIW2e=YSCwRvJn=3RB;dkLee%MxL4x8@L7K@Dx6;?pL{p(a>Nb{ClMwkhg+j^Ddm_J;= z@DsaQJBib1dBW!>nz+XSQ)d3tFc^}6fv^|7K7CjUr0S8c!%BK>fI1|c>4pBMQDK@~s*0D~x14pw*fK2Ipo?G^OcvxJhPP}BS@G`T1% zWH=S{jPD0;PE8gS{mZiWQ$U_%9jMRg&~kq8WaF3N!|qiS-hhkl1Y@%rJo74P*64U} z<(Gpz)?q9C%J|1EO51@;jp-Kh5uD@AhH=stL?PID26gSzQ+)F|_O^Hu>-kjw(rfbH zK#4W@R|v4f@4VENDO?%~nVC!tB1ImMD!{DI9!3LV1tgDpn#1N9#ej`Ck0d4ZKV?98 z=~Z;z?=0jj)T&uPK%m5-E)>~=92c9AvJ%w;{^husvxScq1;03WG*Dc%^y1M2EH3d* zR+E+oj+$CFLm~R2&oa3I57WPqS`0A?ijk#(*G&}gl4#*wy`xd|tDNSMzA|-)96Ks? zO~~xZ-1I-*ll(oGS{NMW8-hh~99c4DO@6JnZ@>&>`YxI@&mIGE=g0he@`Gk>;cn$$ z9UM2#A~t`BL7YD12eW!R!wh!H-~m)t zts$7VOEA_3t-=rQuo08Kf&58jzvo=-IB(O9Aan=MI%Dg*yo;)CM!a#%d=&@znWp>L8Mc5wl(X(HD$qSZ+5G>V{m%z<*b-67y0PO=z#JG@ z$e-lj^Z?`6sJ|RJF=WarrT5o=-=H=7$XI*h1BtC|#gN&>B)wq0roE^5l!C=7v`D9j;U=dTclSyOxd=wVFyBK|KV z`vzPRHtG_p--jwUqx7Dn_9+yVOdx95t3uWZNdx>bAvG8)3O57sEoSZ7;8%g!%ztajKBfKEho)LN z2YDou9W&}oGk#{l_qNC*a(_FMxPHjcP^p84_H+As%tEMoPX|3*)vk$O!t420@X4$0Hm4Rq z{o-HhCM+!R8WL-Ba;J41Ld@zQCAM=C1JHH!j+p8nG?*b~0$ zTK>kQzwjsgd;T7oJq|lZPXnC?y?9 zH?b|aer#A{rE-{p;F=@VNu$zV42ZuIY;gY%=LfY|G~vEL@bmjuXZJyoxH78=#}`>X zGqt+F^~e}GB*>_c;P11U3KNc;KoI~r%k$C+5hm9=w_S#F~dW`lS-+ zKa?oz{rdF*AzaL`&qvf3ZLu6SutobB3m_Wo*txjq8-8>54ZMfMQefV!=Ri31udApa zJ=@}nr73?tyWWw(!}j;rf}Eo-Eq=_Rw{WTc#X~VF^_Le1rVo=cPrb`)gnz23T@%u8 z5f^W~8uqa|#oLPBft<3X2G)KQRomua{l;d>S` z(z@Ohta~B|0-jfimyfA7L71|GpAfe=bD*` z3i7$&`8d1*(`U|``nbE=1aGG3_6G3*nPq@OoN;!V=>Zu+;6EWc;8pBbn0z-1SM{Y1 z+j+!&nxrVBoDZ93=^_Sq9ObjazrH7lOBvHiU6x==d8d5AcdSyfIv-k6VHE5*vTamW zko-3R)^!blZ8_ijZp=m^Xx=nmNL(ONcMw5xxkBp4Un%!en0_Z%M`kL>iYB!l3AL7I zePh0fD=zz=`$|E-EzeO@)@klNcSTzi{$Op`=1{M*Xl&%dxuXyiD+f#4{q~?KX19hO z+Bji3Y~v^X*A)vNqPd?`e!{@W0D5jN#emxUtss6YSpM%8@wEY(uDKfjh24>a26x_* zt48as_qf|0kso=}K3ucBN?e)U^c*Nhi+hKj` zvTBOc^Kd+3eT9)-GT?MrGZXb6@7n2(K%qV75-=Lqs4jnh^KF21uNEq&RtoDQVc7>I zVm1+dG}wZ6*>*<5|RN-lP+qD(5!k|Gw4PZ>fRYZ7!mW{K{&`4<0a-0EQ4 z1Zp!}qbpcjwFhS1_AfClTNY^ufaf>sO54Z{UM;}4gxBDY0Po#or%)4NM^_%737Tn*cE@(X zXe-AHvs7}k_pF9HOumCBZ4sukBLBfVs3Z!%g;J^8;yk)SHb?{t`%eU)0K#e6dI^6q zTZ)Szh1U*eII>j%nC3lL2NCr451Y*qpu{XUAL~kkcrjd8QbH+Z0UflBY9FlAZmmFg z2hWz2R84SF@cWxLUCY1O&xJr?yBC!9=PG_t`S&770u$WhSWgmES@MPsJqTqP8q^_Y z{T)v1yrCfkD$;*h{4fChl%;};Ifg5xp22BU)4EdU&)_ryMP!WmMuj`Z_$Nq84}H%w z=>nG+f7nuCV#VlO=`pROVpGAJ0;{o{y=i4Hz5&cA@u7f9|ezh)t_#Ur|E(-%ce zosC55vGr}_x>&|^a)QcC=(~OawLRR-RWy{%^=N-%4D#fz-$hn4-tBrzVH~oDrQgvu zz7)j8r7MA7&6==RcZe?S@<}nOtaW|+FFPff@T`gr^_n@SWz)b)Tt1?1G91oF|FW1pK)@Z@_80qjdOsI>CZV`Rg9hm5w_Q--^jF z>;=2wBK`%I$wRuN4g76$Pm~+=xXChak!d(!xPkCZRfU%?_EF-{9W;;Rrc8~Z5UnN7 z^cPtJqMBG160bhtxp>|y1!3%%Otb%l6?kltLj2sz2oC7gn}fZwgo@5G`ZF9M zjw1#i@9>D~2N$uhQnWEpz`@afH7&$AkVVPcs#}}97SpfjF#GzSKmP~C4Y=s3OyPk- zFkEvYb)*Ce3AV@P7xlo}_<*gfrk{d9Ok#rQF>GPpxu6(CaENZ8h59#$Koj-XB=kEq zK?F!t$-(VlIT&0P@%3=%6xtDimurWABeJQ)u6<#DTf$QfB*vznQWSNL$*#=b-hoL0 zotbb&OpFCl*0GM9Rg#Tav2@3Vf66^^oHzJ`0{iq8pWal=Syg|=0Vysr8q7jcSJ2U} zYSBltTPMDawuXbte4Rfu#*hX~zuw!$k18a{aV=Ta{f!xC3blzA{V_c)mv~prn2us- zrfC(ehA7?v=Erd}lo%OUxId{6LkMtf*Hz!gNlCm*481o23y_X=b^8JREj5VZ3L~Qm zydKZr|1G@-Qh86JokjLTn2~TabV6iLZ@3M`V_ZPzD_CkKszxr2jv1#gd&f#QX;Vml z{@xq#A+m|3MXA?z9X=)PB-!P;vBX!O2*BNNhQo=W@ne3Yu&OV}f9&%OI4ppen0`>? z?khLG0+xuoI{*(ZB>H~^#v4=2(nSttZzPPzEO_|3_(4a9?dVMGBon{;(<}iFNyz<* zIRvwFF&9<4Nk3Lwuy5G+i!I88sNvQyk6IFwf%W70i7EyRWP=(s-X9wGCIEv91u>PM zFyWz$Ti5Ye80SR+!MuIU|uSS;OlU=eY3HjQ4w&W*A(mS zdkl*52NnFo<%23?Xg{dvP`m#W{{+$XV+i>f&@7WpCWrWlEDEO#9Bev(!#8Y3lArE| zbOw_CMH)!kfQ#vUa(+nS)pg|jK~a1y-Z}5ZZMO!&QdbkaqTh$*7_6uenMfG)pD~5B z5crzcL7byfu6u>@E5-8-J;38Cc}7W<@pvkS>|JTAozlO^%YYdQ6r|8n6NA>c>!jF# z9*TKl-CBH=i0vO94)x{Q&RtN5lhRDo! zTYy|7e~V3ey$(L><7;I4FFY0Z9nu#6v?#tK&+$z%OlpO3675@ikV0DN)M85#-ze9{ z;EdgA*6}QcruX3#_g@sbA!WcdywC8V;PbwvV+%&SYf?kk8%Q+8b%3xyJK`9bsgwkTaD>kkO``XM*xsKChlR6{vn8SQg&aZCjEo zW&$PsE6yiwfPs`0b-oe~tDUBY)oxX|ZCrs{OR1&AUI=-U)s$%ZBQ;Y*T^62-p#5cQ zUpUT|@VG+=;%{;XO%e$Hgi9YTfb1Z((0uMU!F&=F=1hNDXV#{$)h1I_DW5;|HI5&A zYir9HFicQVGnp`E74($XHYA0}VBB`aX(P3v{KItP8Kiq4vB$iUo85_a=-+?2W?~XZ zAdxoXTVt>|H_qQ7`Hx(&1yn{~Pzy=<#@$6XbJC=K7 zjskIf0dS_i)D0X$mj_>;w?q$BE8C3D&uT&pIwJ8B&c70F0ljGZEj)4oBzJbd8)T0x z=6+g)A7g7nfF;5jo4Oys0_2q}P`_|_-%-bkzWo=CSnwvGdrbMehc3b0Z01f;Mt=eZ zHtP41RN#^|7JTDQoA@% zk6o;pOMb1_w|J;AuU6LwiCCfm{qHPxYVi9z-h)T(;o(1e^(wQ@qrwxC`@nN&G~DS1?ctT{|?Q`m^m>pxaxzZ-ad99c;1Tu5AAEi0mE5n8S_%Az+>{7bH))Ok5YjN??Q)|jkS|Lnrf9nrQlX)|ygUAhE zR$J<7H7s=4osU-+-4&`-@R^o?XLQ%=?`k z8Z9-Nb^!kUK)g%71+g=k6#_2SWm0KmUbU#x^%xiev_|gek+cerq?;G7Gfs1?GDO_E z%eH-;jCD25zP-%G}Z$!dxK( zy=#LuaF^Mb@Yb`ul`%oZ4qcizZWH_d83qnENXav$T)39jSf-qmZ}uwtha9|C)ml4} z#yVXyfzpAJs*}@#Us7x|YZ{W%*b@O-loN`Yn!yc-Kx**3x*vPyfIm276Ejv(7Jk=a$)4 zTS;vLS-!nWzkAi~(acY2o42>z(JQ|3KfVIf5aKYMO)w&$Fb#-P-|l%*R!|NwXda!LCRRH`?1Y;W7iJn1KZf{;=45H%YCAU$ z2|vKB@>iv7Vpv_3*nYzCU|pYb8pa7qpW~SZV004G57P1};Rotf-r2AJR(VNqAcnE1 zHz1U;)c1knU@V>gK=xCZm;;voX&vs|bGzwcyu%?WFki_M38np?*f$c^Se@Te3tnpn zSzCjP_Mg9%!x&@G7jAN_4Sh3yM-4Sf*jxH>U?IrWZPHuoVJ+%laqP_ z290hmP;4@P>@pl|wPyHMmel`K%Sb~#AtM8rL~Bt{5Bp4g>${6V7nSb*&K2eQ6^+%l$uz6I1jY(M}z zWt@6P&!w42N9|}lftf`Vq8Pw%Nwa}3HoQm#^F9X_ejOgnlax-&xnXC1?uC!{$k$0YZQ%qa19>Ca`4tWYR}J;fp!;GW2~Zo)@b3XuLz~+8 zWhNE(dXAL$x&wMRAs?0d-fd2Sy4@(MXoQUb^d1S*+;)e`K&W-~_1I4J+iAs5NDoe` z3C~8&#hzSWZ$uOKS@@wOOaM2l_;=x=5_+4Ou;{g}V%9YQjbL9gD_7Oj^)Ucj@#>D& z)xd%F_L7~WBBV9V=eW|VeAd_a;9G*Q>#7-z-I(2n#oh;(RyO!j^V#{5hdldrBm=*f zX7n}Z1_Y($JKvp&RsDCf@Yy;|ngmc9S~v(QF#s@dkXyW@2I^-I zPWa8r7?4vrZ^BH*%TVb2s(q&uYVQ9>g9}tk{^45Wh~u={LLJSXBI=9I%<%xeUi5kl z{{SB4XmvP&X-@y-e@WOKM8C9KFBswamSNi22IV$UgbBp!zK22nx#%_QXykj zXCS#SZ`?z|ujSC6m^!jP8|@yV4G(MDC1regdpaHVM)%s`C&g(&rH+A2jvR+1D1X_> zi8^LZ30Qp>ETsERwK`4tqeyv~Wc%ys#;Jj)_eH!i-IuV>1|f>*xV!8+%>}4DbuQ z;3smlA1;gEFK%y)W>b)w`7C{}S!~{2H~-9dYWwx~M00H_V7Pjaz%Y7lZD9Ed=hKQ6@ykH?ITa_Y?29B&?;GH3Ebt_DIeaqxzc9mJ z6*jHKAD1LC+Y`%Zg*eU=61YrS52r&HH-2vW@DB;*9|k+1g6SvS&Bfgabcx;OnBw`$ ze<&@gF|ZR7#tvG=AciK14bogpM(upwEQ3tmeG8Lm{R8ryn#pv%hsVqud8{fST-5ZA zz%lzxK3p#iR{&qxn*e8uICVx*s?5q()kdv-RI?&F)+l)3z`7wH*>Xr?JEyuKr~LM9 z-rbw?(-WCZ{fM)A3vL@~WnFxqtRl^ku06 z+h)3Va_G!EMpZW7e~YMlkJob?67*Td6q^t(L!m2^FO#3Aj+%V|-(WQ!Dnw_q9HQ`_ zQv?HwM8Z1ni(MMn9rx66n;(%e(8iX%5+mE7BB}jAr>dduvl+UCLMECo2#C6mwR5C2 zc5WB$TvttH*{eV*Xx5Gzy>p}&&V*bFed2m#W%6@9Vuz)LviOT@iIbeRVKbHRJd%qt zU{Xs7UJEX>Wc@XtGfvtV{03WuSsa0J==BuqWAuet_v6LKx6I66l!)aB-tEAXr>ovQ zg0F*v4WNAhI=T&o5lu{_ByY_m6e)4+&I)dcw88i;qWkSLss z1y25&qGu5>4EL}3)0wIblBD-~VjO_f(nu1M`3ZW|TKzm~SeEKYh~AwI%lHQryoKcM z^;X{E8rq_(6S13sDu+33hDNXy~p_Pe1+7AVrwl7FzY(dizlHcDa?3&_VHZbiPAJ z)2WY>4d0w7|0s-);^Q#Gl|^lc@;y4>3wn7>pO(L0ExFEsxR)dVii5H-rVLo+ zRJgE4A6XINfM#ugF0aN+`W!RuKZ(pp824aK#ovd1RCd>Lk=nuyuEZ!v(UG%5KEqXN zKL;4!0U6r@fdHx|#;^FDk0Qun8Yh_dP5&EE-=LlC&ct)sowP%{_&6$SENbXc7$Ur; zwugwW<<{D>BvV|T6Cn{k1!u-{D);Qx(!Qrgyi%>!p zo5`9(w}(~tiW)sVN~`5S$?5gC`s*%x>e}-!MqVEGuu74bJndAS)C8ThY<`ou@Qh0e zlM@q&HMM(oYU=$zk)dnA1t{^2$wU0P+!}95cq2uFMt?Uj*;;y={GYuWg1j~5FF$t} z&UnT!>207R9fAm);BMp(HJ^>zDB`T70SldKL!fCQ6#eSb3ud!jhApeQ9cgsBW!yrn6P~zmWWRX3V zvAUp0dY=_pjB)$UqTs|lhQnuswc7xD))iJ23DMzis`PG*LemZw)^=HM8NGQn#pgeC zxtex3&gmN$hd(EA9E&`Kpeg?dU90?&{36PznBsWpQ|znh9b&igX>Zhq*zanf_v&NV z?vc3pEf(UMZ%$~Hx4bnW``uzL;vERWdMfPal>GL)8`WRUv#ijM70&ffmD1EJhHL3w zmLmqYLZoBUUPubRjn+5Ul=2=XFnI2fT3F#uLC7gK=nCV>8M*Ib!zBB#K}DzzNKvl8 zPz5LRIzp+m2ArN9)d~GLOy-|NjR|#6qTV;4-&F>*rL!DBPR-O-2Zefp4{y|T99PJHtjUl%deJpWW|l?=U*i8{ z!MY%h*t6rPe0+~)>AAxhjX2x`XTQIFB=z0gbg8o9e~f+zKOhDcj6_J+{i``ZVK59p zQ={rX|n) zN(gzNq!o&WjnSDs8us(c(;}O1M6Hz&!J@#sQ)v%qt?MW=QopSYmfzo=p$>&!G&i-{ zJz3%^Y;x`C_h$S+Nl?Z(cNd>u94-9>H|4NVS`&Zm_FnCde*J;@QgSDlK%{h&`$V?P zYa0}#*>}>2yNDrm-nKZ4dfaEYW>I)-3}JA0Jrq{-$iLK>=sk^wqQBU`#-NYAK1X`Q zKG}VAF5VHugQ8=lVvlJ`;0~Rwj+QX}R(aaX?wQm9Nz<~HpyMKevei9GWwW*LlYg6Pjr*vd(n*EE9S(p`h_DL!T*J>YGIjIFU+%mW z8Q&PqZA;}gM&{~{#rVQEZ15;Fv?ISl39V_$f?0V#-`Z-E_^Wy6?jizZRC~&NIaB** zQ6nmR$&Ziottsq;d$#5bHsni1VuLT{;iP>}Wm>8-JQU#%`V<*ZOOdhuZ|6UHBXYMm zyVk^(X<9_8kG0xCFG*!Z!G8j>!fE3E4NPqgA~VY>9tv z^!1Rgi?T)rd^vfUtT{BKN@y2E`JRf2RzQ8{Yq4z6NYg)6<#e!TNZk4rg;!vX{60nm zQ-OuRiFCPmIFkBoqkPq%bdw~+q<`S6dAQxHw3bv|&H079VR)OdYhRDAk|?$uv*M|s z`ZUCfi%wE6ChGS=mbKQOSC%`izCic|D&NanIkMC!Q|;FF00-X>8D8_Ty@()3`)D-# zseJR{8eV(!v__Vr<`SlXQjN%toR(+y%v{k=eORl}~g z&~!LS|2Y`guYqU(5QihQHX%J0(`)39VI$*N@ZwnwwY zPAcH?ZOA@ObiQygd~xZ?C+>%6$H4NNoVU-TLgS=fSi{g9@q}|C(%R3lQ(8?gJni%h z)>!8bp{5rN4(^kO?{sWN@kK<^{oHnF0o@o*)|HhpJM=c<$Frrm+_|FgzEj;nzx?J##-*Ol;1rb5%8?cYvoTD z$>$5KM`ckmsbe zzG>#O^6YgOX+>`+_-*g+srb}XaHT0rB=ik7(zke0QWW~O89oo#*S=Mr>wsrK5 zWcy{GP(NUatXPd2!ibb@`=fkTuIK4Tb2LG(42^mXnwz46uswZaCfy)>-e zThNWu%i%F$*&kb+I;jOMeU$hVAl`LY>#Sz)H>*{5oD-b#D5#~}?3r1#g|~GU^hsG? zuhQz?F^T$-%kr4(s}tgOIK*aiH3dA*5~0rnv$CsE+dLg^uP@l0cdXb6alaRLGz%?6 zRa)|=7iN;?2YJ85wqLvfdW5lF7WE6%Q&+rB`}?8r@DL#FVNhT^JS><(&Jtc^zox#) zCnrH6hIB9319883BmCyQxQH4P$yI#ZwF)%}Y<{pQp7jdeiA1A`mQNwWQpbyv#%dEo)7=u4ZF4or_wd?ZvY>bS{W!ESLRO9+-hwiXs(OZn?W7rarRDIoCXC zn?WyKUr~-1(l(T$&80gk+dcJW**ycp%~C=c3~(&P`P2mh^Eav zWonR5={kYcya~2vU4HAl=dUi~`_+1YC(lT*3vvo2uT6LrWS&tc5LT(cHY0alGJN)) zQt2mT)pvW6$NigX5$=R%zn}0{u0O7F+fmBs>i1&lOdI)ce!+ZT5&C%d>*2ht`Ivwu zM42)Bk>kv^qv%Fysk4eD_l;S$F4A;;w6Xmc?>V=;Q8(h){v3fIcD_)N+4N4OC?%87 zN^09CgIslxbh__l5-i8Fo{WdAkIKHW)(!*C#K`t^p{rOyTk;3Dgo?S3NnWxeZ8F8) z1c=__lPVJ0bHPW13r)OHD zq&3xbN>VLq+G+1d8A@GMr3f9|BCH)7{OEl)hWc1W9khJa+AmNH?-2VoCh*Lgi7@dx z_(p7)jkM&)P9_82_61Fg`zJSL*qczw>4PP`xSmEvqa4SSbD~+t3O|Kvv;|nH@7x+A z!REQdd0k4eL=Br2ykoNbO2Wj-rQJ437tZn`7VQVM))8h;)FxY*afYw;lq~o7g>kN{ zvGxEv)RZ^Lu%rluHu*8kS4~Mqx7>_3bl5{w43tfN3ud^? zJ>#1+=wgwx?IBrt#N|HJxT>yt*#mEH8f5OWc8T2cx);T}NSb)kWZe)dOm=N2_@GOc zk8z|jNbwRN#r$)@slAx>MJLaCT7K{sHj)WjXDQDqk{%VD!#2Ka%@-vE!Y3%#$qPg4 zE4gI2ViU6bexb?GO=AZAw2CoOFUkf+Ko`+`;Z z-0?fa9M%=gX`DFZo)83{G2!e=OfWM~)xgeA$VWI7!462tfDIGLN=B$yA^sW)9{+wB zwb&r*XV>D4o^Q^QZ1#*mRvo24HMPe*I^X)F9YtMVH2z!k1?6}pCbSvRyqQ~uTMb&U z{NY-_-*FSsU8{PT+S5Cc#J=iBJ?;(&?T3&&v1Y_Fh!MYmj^#TDrp>O;0urI`9pd*9 zGf<>V{)&lAK|=u+g&~vEqyZdJu3Zn#l2duw3wC|y1?=P*dv?=D?0Yo!1jgpT&W-${ z)Pm6DC}-)RLNrn;=tVROFCC3~!yUBW1sLT;?bj_I`&zcJP;EL_hG55sWBYZ}^^jEqPhm~iGaxfnR*plpJT87)j z<~pjBHEAP%GO96#>gZ1k%BfAoCF?Lco+@$E%$Lw9#k+wqV%zonZm;t3=AF-jVhUkp zSspIXE<%~pYdxjDZR1--RQzOKaa6D?p3>H)#K7ZY$$D6wRDcu8QMgzYiL^>_7Btmi~X=)WPFweh;9zmp|{ort`icmgbXx~-LS^M2Jne*7)%t-#pFVf0c za#$up1B?GU>2M;dKNTZXV;p0a=SnEO7bc)!BC@`7`crUV3{?_iOtT6tljHTEON~0i z+w;a0hD(9GByo(<2m~{<67vEUItzc9{;iH9xPyIjV}$MA_9c{)=>v8CX~zgGRx zgE^#-z0=-7*cN&Wk+>bUAHh=omELYtD0y z@x6t+%mk?=idiJ=x)R(N8*}vQrRE;^DpowTsQ9zD=#Bm0tdN87V{BX3ZI4cc`~^D~ zgx!8=^A;-W8T1rNEw`HA-IvQ)SeE&f1YaEb=)Miq$;hw7ovYq`@t$>+9GQq^1tMjDZf3wuIS(>p0!8RhixuDyosZF_D^Emhdm^8!7N((j+)$zbx<{!2KZDC41-PP zle(!3s&45A$}`&sZEdlSOT~E&mUoX*Y!Unv9N6br{p`}U^NgFTRlkZUe~dz8oN^p- zF48(wuX7%(wB*pt)AF<>vaI=Yvhf%?4DqNMPq~uJ=PR8%GXPT}R8>_p+K6`)tPkFt z!nP!tZ>4I8y^p?k5*ATNfaWlVToB?L-?ySn5sfZr>Y8ktv=m$SiQC&iUOb_OtTEeN z^-x~979}6AIrl(@*!dF2VB&N#(z4m;Mo*atE3jMzttC4J^7MS-_{v2}V{4L>Ia8eA z2K{baQfcLyS!O+GBT}V#FlbIg&_Je9Q<~FTHSPRl`y2KSC_%&!oGod*#Xz&q#;&t4 zu`@7#xc|yG>E4RHkr9F5G`-3XhI@WLc0KSHD~%}*Y<`3FUDoN2oWn97W$jrpB73dv zm{xa-8vWz@Zd%&|x2$tqD~}!8d6ls$Fs`f=VJgp6m3x$ikBu35NXbpQX>ZWu$j*pT znb6^$&lg=paj*=*JyE`?=g`c`kxo^ySlgooUUeo|W(JE7>qI&=tLY@YrBaX~A3_s3 zW!8eq8D&$go&DKMy+Tt+QJXw;mG~Jd@6*UT#L8w06&4{}#!m`H>6^U*q+M6J+VQJ& z;jY$_iiba)q{^ZL#&7^PHA=SXac+4*^vvRH{avyp6au434OKwuJ?zoyJ*Ejmd{yQ1 z00TCwrdm|*d`Qm2b`*(@sC@N$`%|t3W>fod+-BRS>AlI;*<~;3`OdNUR9L@EFFpjq z7YJwTM+A^61n^I+ms~i<{$tr(V(%oI6KG7M6x3ONzYKjgrAYjImprJG&1V~UjhGua z$5EcYbLEuhatxh3p559J5cDn#G-`>$Q)XS8lArLxLILG3_pbgKnQ-!?mw4K>!#LyG zS#-pgC*)24Zen5o3nC`u8|mho=lfE zC%jlHC=5%VQoThts{jQTxMRxMdaIvQ=Q$4nZ#4wrd%k}kkd0PmBB}d?t(JWUe~{G^ zv5D2=8=R>>X9eS75%z;~2un-W(K42PwSvMiLbz6NOr2%8or2E~ZX>aLrOVjpG~G4w zmlCsPDvNKECzZG)Z4S=b&&5;vkc8s1=@#l8C!(arxzR>HiYV5Q@TPg=WwqgZ3iPUY zB#5-y0DPifg3mhU9xgRSngb~-Oy8XNZET+fz-wD zI&kJ}<5f*d8hZB9{Bgn9XvIxbO_dfD;5JGN#eKXc)3_+r`>d25hG^mgIPC7F>h*wAfPl;&C2U z%6xPGlK5jM{?)J#^Q{wA)bf3k))==Y#L);^BI z9_;x!O0tZ2gyYVJHeW_bgf`4*5B)Ct?&XP-b#DDY&Ztd$xjTH`yLZ;h?l~otI#ADc z9=g-MVrRFfNxJJe-Sqx4$5&wH3zTLPN*o=Kp-!GZ!J94n@xa6634^UnSu+hc&v6%S zrf6UuX5V2?8sZ~WT7;JFZGD8^`rtHwz>MY z$WfzX9zSeMhEs5UoageFPNbiU(s%D?&rjmAyjF@2V%6S%1izOako2}W9MTNfD4@2&{td89n+6$gWfN6bBKNM21T1v4`@Samp1~)Qm^=v#JKISUvCI325G=p z;ywe(mJ6RAC@~LZZ|KuU@F0=(>9mLI9HJ>n!~64Xwxji5P)p}qg?7>7%mD$AgUxD< z;!9k^5zn$n$;kC!jH#{aJ_ZtVpJNcRY%tBtRV+i5J@B~iw>$f@PY6O6N8gJSibI78 zHfS;=Sms;kA>r2g@~n9ZscWeSV-`eGYUVH0Nn&J9WUD%)8l__c4&(BX ztSL({+1^Mu-#VyGxaE*Aw;9r+TnZ2MS)vp@Nc+0GpO0$|wehv=JmGeZh0dtQMB6%@ z@71;f#88Bh&EQxnG#x(rs-^R98PA}1@$aW6Rk@u89GWP8s+Vx-t60@qN8!S z@9%ZO+!Mh>*9R4+hq3R1E4>*dQyx3&A&^EnVmu-s!g82@)%(bklo+hhay(D*LPzQ8 zGEj_&%H;_G0Q;IMp?F*)42rU{p<`X*IA;181cpBB&q^9db6BT5(8KNEL;}zc^&l7?= zbX_u`f?#G$fi|T~1%&oZp)>4I3X3nXJah$yRl2uqLlRtqru|~rSbOCb@6>qkN@5Tb zZz|G1vYQMqT0ZnbGB<#8I+t*XMF^7sZc^1*-+p#HY@E$Ru z?5?oj>Zo83?@k@I6V@|-)ivApi0=x!C$}9;AG(DUgX0UU?w2^Y)St|HYEWlL0WIW< zOxzGpH6s1MMvLvw`s_^osYqU2cF3LmmljEOXC^{G)~yiv%vrtd#f4leM|CNfGafgW z8U`e;TLeSOQBg9O*D6-ud;nD`TL5x4AF}V+%*e#Sk;3wMC_$LfI7!X#4@+-vA<6nH93huTtxMEPG)zgPS4ENY=f{y>pO;uL>cW5m^gM-C4mZYIN5H4N;Ditr4mc8WZe?i^N#3ZWe{#S6HekF zmZLO@LVA)?4_lhbK3#L;j8x%z37?`RcK5vj9buWukEh_xZn9<#gV=~g%%?Gg$9;U? z@Ty^*C91P8VNL}*S3q-F09jQ{Zf}xTnw54QwvbO*edpHd^;NwjdhrO>=7X)BxI=wn zO`T)PqTyx)uj?RPFT6z@%fW@LRooEP60_b8XWF+mETr~)!s6WiRR?;QYZa!b3{5Tb zY356JnSMV>&&LAk{Ux6;CBbn0Dj-BtNouamx0Ud4!*3`Le!7Np{s_|aWL9PfM{&-* zMc3xd4vg@Kg%b!l$o=NDU6>ZRZdNvG>NI7{pnfheOzV-5_O9mBQ7U4tTNoA@05m_! zu2IFG&u!ucdBhjPF+piP5AEurai)lNQQv4pt9Ir|+#cC*!wgk&Lv%&VL3*z0D=cfb*p<|zQVU09mGEGb$R|ntck5+1} zC3LvfgNc(AtL&`8c5(?PR-z3Ttx*yLrdOoXV0g1qrV2^)c5mX6plw{p87jfJD@yNv zH>qA0X%Sp0{SC`azsLBN`Bx$$ zmdhuFui~tcLlelzOmTJ?BsA+j71XOKcWjGMcE+i4N&@56z))>{84@i~j+ZDZ+bG2D zobSq`do?4^Io&T)aU2R|FgCfOBBZHsxQzAW+_O_;8brNn7nAId@KlC%C^m6<+wvq_ zIak{aQm-FFzig&_QA-+alXbFW88(v7nXp8XJME$C;))kWKHpbXniqZDE_O_~fm2g(3hL`voa!KF`!ZBz*rRrJrVWq$-QGA$4FY|>I=rzW4=N$j3Ccp0(^&?w@-d&WpSC$mZKZe18 zy=^mPz3=gXWH)hyZ5Q8_WKQX2G3Dwzh3t%Yvqb7qY8fivw@UjNc_ooEpL>LS$I*QJ zWMr+`KdC2u163My@56~GC!Xdu84l_yv!Lm`^E}S9IX6-e`#-d4E$slRIk;%H{~3+h z>a=a5vhmY`Kgq3FZ0PB$?PrM`tZY8J&?=&`ThVP&7;7FLlQ#7$A1uD@YX!)#*jWZu1(lM!4i+LV+`Zybi2 zNwUnxQ1lElP(M~0NNj0tV#U$_nVY`HMT2buFX2AyHUwqh)v}P8wfo+q;wy+jOFmsx z^+>tGq>h(nBxqkHB&h_XN2bQe|+iI!cfIJp-=t$1Y-Um*iz^in+;kpB^RF`Zr z#-PKUaSP3rkC(k;1YFelqfR^>x(x*I0~=jd@+ikxr?F|3loAeBOv--GW3wMxR`bRD zJ4K-E8N%e5f>6G%G!dZcd?8$3czcZopM*Fs)MNmbH+Y2dvjl}hN4DU;U?Ll%UsX2z z{`w>KXqp-|YMq{5fSoH+;xBFdvQj@W)SsJ^y-#g&-cvp%Q#_iaZ?j8L&h+`elZ}@*yeRFi(-PUzu+g5`Uqp{K0R^v3bZM$(A z+iGk!Mx(}d8r$Em&wcJY?)P`bI5~Ttg}LUMYi%)h*@h0Ni;a+1Q(A*UFSBDaKpCRDf&aI|YIm}xj3*EeuC^6A?J z`Q(*2j|5xIxU3v-foy%e8f>s#N!}T>LHabNaA)|phKO44u;e2XUwO2^+|JtA8r4{A z-3PXC0c>`g&gKUcXz01ataGmz_xhui|_O{uu4N zxwga`cIk=x8QhfqQ|6ea;T-c-l!?q6fB0&RwIgMflS^ZM5G11~VI(^mS~7gXmL3zU z`39#i1ZAkqQXT8rEL`Z+j^4zE##7d~RJqqE2A)4ja|%}kfma@a>D4f_BUGcU^V z)>_A`ux~N7BM&^V(*J4oiRz3Z;{H+0c=+*=Z%s%0KsaH$;jI7mE&+Co&Q4CxtCK533tgk2 z1fqEq3MM}Qm9U33494>9pBIt!-{+tw0S;%*=+cc za2HeJ^UXVo-NiWkJe{KN!NXp6E*l771YZpF}lMMylIBW+h3(OY%gYW_=63Ndwq5#MGpR_$qK?cPx5A4hsW>oYU*#)4hVR!x3z3GN zZyHGxrS`5dZ{e&Rnjf5)nyQ2BSG0A1ZwZKs)WQ@F6@F?mtkFiSW_d?m8}h5ED|Q$F zIuEJJ>&lp9qZqQ>B3@C}4RowCjD@U=_bKYwaH@AE+70gOUza7`J%dJZum~mj09`mP_1|g_KNbknL3wQ@a1*^!%iPYO`oHr8WZ@uM)9JM~Eckshk@fzT#5)h>Ix=o9VCmc06A-n{@(w(^Sfa?_5Fu&@JMAz&~ zO?2%$x8Cq2EDKWr7*BuyfS2>lvGTVudt!`X=aAUc3sn>WfAra}c8kH5?}#01I|3n7 z--yg5U%S&ncihaA7puJVmp*Co?0M!Ac14_pT4#V3+qU*Kn3b2aA}=e=<72lfHZ<11 z1xDu^M!*+&enXn2wWOQ94so}$NS7(rYo3!pSLxqv!$j9-{gBFnO-*%q_J`jtg6Rv1 zbU=z63vUA*_>UunCdSWh(*=qdpZam4Qx4NTDO!en`3lN{Gu|VP3Tzo#-W}=_^I-8M zHf*6bK&^zs6kk(CCkB+4_O)W+VBrpb%!2-eLYcs7{;DuQr>dudvgOOKc8-GFjs$l* ze(X2N_?nIhN#iZYm)4HTF^QIHR4w7t7dzN5XK*Vk#S#?|W?rWgYe+v#%V2*9XWE+_ z{Rj<@ypN{5I@wBc`>Uc*TzHF}hvN%>;q_U(T6Of*f;HXHiWpTBU8Ik!3+ zCfeD~BFnAiQ6Kxa_OW=os}!9bLz)vcF;`~22Swd{eg_iqtLfSynailTy@egE$<9&H1mCMkJr8NpnWi$&|EgIWD^$|Vl6GEIG@ImomcIKzUd(56 z<^F?sKE)9EP720%UVr7+;iQa_B4#6G0-@1bBRmiA4)R2^WE`oIpP~9tvc+`kWP`-T2g#rPZJw8V*BQIp>oYuWvi#kR z!8WV~R~#FZIM%0o*M-Hx|KKR)`lDc)PmRQbkz`Nfu9y8B9yLE-VCs*R7tl9M9B8)g z>W}|4jP^c_W9^l&_uHilsY7a`Jd&Z~;tFryh*u%A&B;wTAlJ|G?%vDh!orT{2w+d9 z<#qj%79Vhqb2J2RB^M$yt4I|@!7_Vs1NdOC1L8_Q_~nH!$ZNRsk=gH#YLWq4>8=#2 zjPCi=-ImonGmF)oo^YXFZ=5lW(G>E)E<$)vJt!FtOCl??w-mijs@Q?Z6tK<_x%OHP zUW9Zzk)V<~Yi*7#c9FON>!>9vJukBr3GrJ~jy)(Om7AVfCsTw0gtG>Kp31X4I%i#u z%I%D8YG&l@*uLA9XKEmfcM2`XJq=|s!!D-w==EuJ7s`OGJ*cA6eMGyP;||y^_+4X% z;kxKiPw;l7PDG#R<%`)g1Pt98=)px<$tn;NAXXSwq`}-}V72M&yA~J)I_4`2$W{#o zqY!nHl}b~23y(N`*7H$1=LmPU)evz523#9Cz9)CUwOW7x#`C8`^{n!seSP67(7vKb zb{kTPGW^1#e{qBoD35qc+s_VQ8Ex(T$sJwJ3y-$uhGD+KO?q{4b#K zPqWL>jHgT!LWUbKhBIH?!@6U({>rUsjlK}wm!jzp#~d&V=Y1CW!|)_~Biq-!=efA^ ziz4q}kQK_TzmvB=oQY^p%!$%elGP6jBcU&zlXoOfWc%i?G>d$*Fqu<#ABx(6;v44f za^sNf*(3VS5sk<2IFY#QeesW4)bxjEu)|efi0n~4mYMEXS%Fle7FkN~dgQ4;={GrJ zJXlx;&mx@0m3z?_r63|J6N)nU4u@*KWoKf%n=$MqBgmlRGh^uZ=PEv=$*gju;qfy$Gf#NWNXRElM7@ju0 z|09mhZ{Zdj23k~3WNr5x=S<)V99Z7Yr+r@TkHkU;9O_?O3?xmIfCk`be!^Ao9dX4foLf3MYfIBT7?O$U_-~#VOWhf zbTev$Q>)(~YBxh8rbN#xFTy-bn={Cr(QBts?BlD`0;#-gr`5QjF3UMg%SMat7skOQ zX8g+OE*J2A3xduD;w#~P_15#^iazK>^@V_?Z?A4Y4_Dn^TFH7wS61!ki{{asM#=uf ziAZ!Vq92X8eHFH`;MD!Jlg8+S<3+JhzZCfwWHh2)F)J<-C5B+j-yiyVR8nI2>jBD& zK(RsyaEbS3>J9l0o%=3Un_B15TYl{BHKcS7rHk9XmSGvHv!tt9S48Tu4dOip(8wP( znc!=g0dUO&xjbD)DEQ0uZ&`L6+-?n;eWS4@-MVCT>{?@mRmeMQF|Q^0k8GDkC&qn) z;v9p2XvUd2)uaF9N<=)~pZr1uDt1;XRD_{C3x(9$w+pSkjt|(%MqpI&cwnr%FMSf~ zeW!5(Rm(!N?OBTad16oRt(L~#^@nhZ4YVlxvGdB`B5=i{mMg`mBi)6hhA@F$gzP=elo*r3>8=O zvhB!^-$$ZleT2uuZ5P*vTBCmZNM=KyFkRLy1+oKa$C@oyDJLc(4bji137MC!=>#6_ zzcwU1-0kKPQJ9#BQzyM8tOh4P!NnU#5r^E7n8JsUo-JNc;wgPG`Nl$YCvt7xlQ&do zfl^}B8`l$yM%jod4cY{15Xh69Iprz?Q-YHM`#(gjH)MC{bg}|&xDrvcixu3Tpns08 zfOo|<)DkxrLg)8VT{H)WQ20DFLMf`D2Yxn2FczjDvB%D*+nLb-g4u7 z!RuUFRg6T?hY!n)GHTM;?tAhatf05w#o?D;4Ssk|jjGFBr_y>!nd-pd;n4Sa+9JCZ zz`F*)Tbivs_Idnr!6@rQ&d_kl+t3klr-_fbQZIg$HHA;8TGA%x z&oa@q&iy#6Ic3D&{@H{K_qZ^lq7uicysd;JDl9|(efT_GXdO;bEL}{NAMqnYpcPf} zzK``P83qj%I0>H_^2^k@UP$7^k!V_&56cBHipZYkR^@jL^sc6YYs+y7q_( zxG19lOhCV2D9}GirwyV@4A1@M5_NTq2P@u2`#peLS;NqG?wTd(dM^6s0lrJo+2{%l zZ6@z@5JyBL>c;wcV1Z@XjY}SXxc!MXDavl1D!I)G7KH0E3>wytD%p?V4OBo=6?626 zlDY7lp_A0~S8T$=Pj9cD;}pg9ZSA#yP=c*u95yQB)Nowt*_LlO>LT$AM&>~Mo#^7h`y$Elq1s^Vd7B5%Ye83 zdfRmz_`JZ;x~3p@?`4**$&x6Uv6byd9|FMum2uJOUqcjJrI8LM%=#`;6b*Zd$JslEL>~&g(P8e0<{!r71^{nm8T?>RiUCo~8Q})?%WK=*~TMMmLoBraFZVJ1JEU`~w5f*_N>=Ty@VxmqB zD*b!cxnb-QFFT#B+*h-JT6tQ6r2x@)h*LX#xGF2g(4K@GE4}uWiqvDC9Pg z&kiz3D#z&WhPo#**sPQE-^=_knMcl?s6*Es0KJs6m7w!6$)ft*33?eA)>Eo~u!mXFiaP>J3EW&h}c)byxen#yeW+E~Co)atlE zm{*Er>*b4=`NssoVkv4X`vNNZ=sg^rL$fJXGS+d}x$~JJ{ArgYTg)cgSAdu=5Mh}Y zXnR}25dHdLRVszv2`pFu2sH5@o+%JQU0%c5}3GA8>=@IiEb+vP*c>Q^tg zuTp`#WXKRD-wuJ@&mA^n+3rkM5LmiVJG5>Lj^NSsWuh7WjMGM% ziYa`)eZgSjX8j(swQ7M^m=nK{t(?7N>Li$7?8g*7%ee3)G-8cw+4cr6aK)6xAVVhE4-X zWIU{M*5l6kDd-yNd0uM`H_o}$;M`tL>BD~xJ~L7|*ME1GCu*TP4b5UgS=EHB{E2US z8Ji+9{41^>5X8atPk6Lv*^lW@hDy|U6ZO-KwLas}X#oO{K(o9N6V%Y(#IIKtOb{xY3&GL-Dtrd=fA~}x^qkye4E0Hl=!MWYo3Z@9ik;w@Bqk;gkVxUYP2N&*j77_mKb>pMCkie@=L$7T zOJP4*V{GUYxggV!9MRG;@@Y{qFG9YZZhyX*6|4$i^m}?|Gx7elgc^LMRM|uH=frgG zt+8-8-1ThSJR?abB?)Z_n}8MuV5-nLM8zYulbh#g@bnW)@T$x{Q%3yU1YuR@N8(Q< zZnHXHY2B1|ij-4%D5Xxc!E|_dW2fM2rifskwV1pd)LT-{-NaQMTUA*D3FKrIUU=Q! z`xgT1$0JxF?-4xzd{ob7ed?l~Mm5I3$Vb5*N%$uL6tU&?6y^0cabMhf-W)NrsU18& ze(7HxC6EQcYbNe3SlSyMLXi>Lydl1mN?ydF-CfeN%pj-_dj%x4d-ghbt5>UzuwoxG zbEdO>fet(kLGQ_Yp5to_D~kF${+@Ivr~Fd_ZgUI%F=u%HF0pNeKks`NB5{i|!(vq+ zmkabx?vT9kG!aWie8hMf zv!=t4|DI(2D>g=iyAv|>rn3mA5OIYCeFZnvroN3bsX0w>>|fwe2#)I@)B#5IFLZDW z9D52uT=FOqG*jKk)2nvcy)8E5%$MBud!72NMXn8z7kH? zznb3-O>26rQ?K=0=Q5GxRSV;1+)6Q7Rj0H)q71vQwE!AoFEL8b2QR^{^wI+);jbqi z+oq-{aBg_G>_Jy;GfUve?Z8xvNW^#(?mEBXcCJ5~bzH1C4h}(>I2a&L(rjkUCDQt3 z%~)`4#b&v?;XW#b&0-2A2IFfk4ZODW#o~Dq<_GO#%zA5|b{JdVGwVZm-oj9BY!{Yf zW3J9POnU5@WTCs_h&JRffZE(JPnIljB-$Md0D^l~-E*rVHMW z`G_Kzr(QJ0%|`1z`gMKwU@`vDA0;x6>K4^idtX|8;NhmMNP55ivzwi1thAF{P~AT` zgB&l^M4Lt|+v^4xA6hRV4(Uo|SeO~9&!-=Pj7$+?O@PF!S&0N*sxC{>;lg2%&E~=w zMhj5TYr|Y+bcqWx;Pu?dCxZrXA6x)r=wK2<)0Ibk{`r{JDn`Be29T`QoDV19tKmIqbwA!b@S0=(C)uK5R zUp0h2CHXHv2pC$~5#hnbb##5O<+F=YY|?}T%17q8uvHYMknc#u&o1KTCVbRvmRw`f zg~4}#9IfpaZH@PSC$73jwQAH?FNRUW%5ZwDfQH(m#g`B;!(1A>K&yeGhg`1DmVB`p zd${XI6wH_;)x%f6`Nc^>R3GLLl=*H^Y45ybP(qKEnehQl|9{blD7m+E)9AA!rAbkR z$|03@(#J%pvP}Jk594O?75CKRc4__runk0aqNOSZG@)@=6bB<7Pq^mCe5zCixZrz5 zjDyd2C4Sy2F5@xn9fZre$*4K95~OI@5=bXiCxms{5$=>H4$|C)py(2+qQFQujlxrg z!5*zSVvle&dUtT1^~Z){)a@BKbWfwlnprWBUKD~gSWQgSzkzOHf4Nmnzf zO4M5b!j=rFEIAuB^Mj#)%7hSbuFq4vDThM(s$0MqHUr|6Ec%AP-W@u+Yg{ge|6T|u zDxqIsf!~0m=5%+Racqud>v1j8kxkvGK1;u`xED&0^9AfwxbJxEjc&rX|$Fm9G3B(BhriUl>5Vx&JSqAq>!be5J!6H6#I%Naq+gVve=x(4&%OT zGyKHYy^|d>52FLp+_Bgm>qNil4D)6uc5Zg4eqiBzU)q~2x-$w81(d7Of`<;vjR!iB zSGza9^Xn?5C_jwTGQ!EGlon68JShO^+l%3$#*BBgKAK|FbwUthQ)rB_ejp5zZ2(i} zxc+WVkUvoz?YP$V(_5q*1&{2ChqS^g-Doh={I5<|Qh}6Mu4g36$gTolwKs1J)~y(2 z{oxSt1EzGlWdHcAUg6p@H58a{SIR~VosVj?##Uu|rO=e2 zE(kWNOf}tQ02%dtu{gLI)nPZrhsuzIT(TmNeQhm9vQ?YJrfJ{ zn6UOf6tPcZts#UF3<~nSH@n~`@#J4zhCCweu^Mbb*$X|EFmlJ5AIM9MbQvY-s3-l% zvx}L_u`VSA(h$w<$ZRat4w>qVjo@iGplnp7uwx4on;bS_&DK7m{hgeI31R@b#ahi_ zDPJ`_Fn)JY`02q^$bf7_g^MK*LIag3ue(lU-;WAQx|QrShl*#Os$U8UDO{cfK# zb!ku6&=v7LP*mVtKF8uZj7%ukLIG!hSvp#qMMtiKR2^dAAWbM6Hpy#MdA*t1%{|XL zII~0~J9Kgl*Z+e@JQca#s%GlShazWuSJmxM#Ya>NeUwW{g5u@Nd_(8-dTu)gJQ(@t z1(O=x_+NiM{~X4||0W_miW%kAaK(o&s!(NnqZ?zqxaNK@m1##8+Xz2CiPkcy7klzJ zKXYB;OL2OR7j%IkKoa6j|Y{S=8@A;{K(lM`s|Xl$J2W46>YWy#;>#=-sD++T$H z#Y6aG5HhD@^W)1|h$Yt|?VHl$-okI(j|LR`K!>tT{~SUvR9;{xawS0XK{5a%9MWjI zp;?A&bnC}x@BX;HFU6KW0Ix7vz^=Mp2idGhiDin5gj(%7sRI>Rw5_=^o6rQ5f+5j` z^dipcNK&)g2e}EQsq_ju3hAWaX~_~M((aij@BvtXgq7Ak`$>;ZJm@l4DVvxQ55<^k z8s^5FwtxdX%r|^=KdLt{(kxxXDybfC1MNk=9N$h6!<|Bih9d3B8cM zjh%7Bq*6voO?)e{iW1|80duV-B$DGxium;->z-9a$yavKp!2$t$y<-tOZi0!b}drrw499|RB_64by zxAkUu-V2@FN~<$MJG~S$TUVZl5{$n;38EK!w*vKN8IAwv zqHo|yGx9nIz+GlP;m+cbAWQ*2>nGr6C1O{xMqCZwnjZNPf`BnC(i3OJtNk5QS_vUb z*7)O|#X!4@V(7_=7fJEuA}5WHN{NXO!2Wj^zm0rI?8plffy?q#;;F)%NcBYl87&xK z0iG8$2%7bc8Rx{a2#b}|Fb?h-&xb2E!oDU_#H@S}O*nR57&3bEnQp!2KpY_)X=7M| z-o|CzXvcE;LM>9ShDax|KHAW&Ec>})+jUNiE}A_dcJCnQg$ zuwK3vusH8pjos7_#-X59<@}5kBk{NEXAHY`md6 zdM=lchV$hu{BBncX`OQcN{e$jAz|YoAZhczXu^Y+c|(4wvRz5E2uRQfl$?3|CeN5kT5uUS?o?S1h0Gbx( z4U^CXCw$1NUgRk0WV-!SysSy8w0%oqrrs0wY5f(Ow%T|`PH9s1V0f%>FNRc->?bS3 z(%eLY0-(H4pLD1_#!I$oFvf5;o~+kRH6bt8!Pz}Uok4Y6<^Cg5s`b{f&F}~R7OmD0 z*NakuC3KCmT|A)E5-!xJ3*S<`qZBti-ceiCT;S~$C34__4}*pVeAma(+bPUm_RNx3 zH70S4^P_b%6ac2CWBex!1D4CS-<2ZoZ2~*(Tp;v6A`J#fTP9epxf?N!y<+2oP_NBfEi%- zuj}*Azoz{}OLmb;Vfqc)1a$#dd>zt1%!!%ejL6&cL$Wzx3<3}@*xq3ZLxefb8H7L> z{AQf+=Cq$#$|OvTVMB`ffvfq-g~Pb;YxP3;%vC}YQfc%XXEeNnP+ROTC{w2?0-x+p zzEzPcKd3o+Y)BRdB3JV4FT)lIBZpRv!oRdQ&!(%+!rl8EGPFE0nTc{0`+5{OAgo^A zn_f)sSs6SrUy36N)kXhz7l2NiK#Y4PJqQD>##vqsXJ4bC0G%@!K3wwIGkiycG2&J( zZ6k!^u!?dgiCOq=6bf&P)b2|oI^Ug+^?Q&d`FYNd774oS?i-n>Q*g<7o1K`fcX6`t&CmTDa28QiutX&(d zn)f?+RjI4D@^qQR#oL>2GTa1~Sy**|UWn1*j)lRs>;zo7&}$9z%d^PDeZ;as+h6QJ z8s@w4w{t~qw>66Zs%LFnF5GzSZ3$B#5$rYt*M|FB8M%cmhTGrZZBJKfKY#TA-1>%$ z+1eAYf$&0@JVTG0;BGMY)%-&_b;|K9s9E_F(+Mw!@+2vkLo3|72ZvQ)LFez_7W zD3`70rA?#UDe}Scf$tX>8IF%UCzC-J7br%!TQ3s+GZ+%EF$es!b~xfHGDe4sinB7F zu2~K`q3zAA*+-kIgz(-YqN{$JFr#C~>0rAr2Cj(x8VvCiu%|NLeo2GR77hM_;GgM& zyI)u+Qd`WZbUad^p?9DP@eNx3?I%0Z`-SzfLa24{md0?E32{LXDq(^1)_hXYCd~#b z6yL5m2#5|}1oFVnS}urc(`E&qzw%&L4?)&b=2owbHMvv^(m;NVF%N6ks0rhJZ$dPJ zb2&IxoP%!`=q4n7yTB(Mw7X1#WTesSiSGr_M?I9N5g=_7_mI-~hWRPo1~(lKCq2S` z6#v(xP3gc(R`KObPz*qDgH~oevFy$FcLh1$3bX z@}PiKIy51mO=w6`G~Op}*M7pIv8wd4+I&EG40l+fGX64M_lWSfqEro)pTly5jB$8* z_UfZMS+Zv(R=j#tWq3rg1?OS2ZV06aCme&8sQgL8f*%pejX2pmwY`Rf7ngz!<} zdx=)W$`0I}*{b(D5@U`6g2wk~qqH3c^HT4W_*4w6Xdo7W$jn0eXoR3w6WkVzKP4BL ztNOq#TB*~FBB!^NnE*v0s_hqu-koj<+55g_hAMO)m6s`WW3*aP z-53%lUpsn+kK++osvH-bo8AqwCPQTl`$95f_OJRY*Koa)&&J=k2uK!TH#{9!6k~N7 zw^3HM_75gQ&-E{lPRr#3i z?*dpxVDU5&ZT?HBWAww_dffUzoOJVPsZNo*h>VtsOf!#rAX&o=BhY{HyeK0uzKM+k@*YpwijfEJ~u@TecvIw3{#}NDMLCW zrtoOs5fy#)FI)!87WDemB=d+;jS(5#TiY<2{oGSHhxDiRBS3&5e~?S#pp+mpsLere zk4}}3Fg^5IqW;=M#a5y4^)s3kW*rv2qws=&AhPqlUw`Z$@E zdw^W@6oN~BZM1ia+8SS0T<7Qrp-Zv^j?&%4JbxrG#3PE_a>NgHb{Hv?<0N_u0fOpW z=|WbMc}E#vKwm470z3v0VGsY3tgLC`&eG*XRhP@fD7?&3)o7VzT%{}W{X&8H!Lu-^6>c{n!ca!rp4qfeO)$Y{|ulT^XC4THS{@N@f$JkntiI)pYb*+f%C$MinUu zy#6&Eo=N`Sa(Gz`Hkyrg50K($4-4!Vf__Xpn?#;sk~%?tN`h%i_(KpmQ)42;cG$aFX>7voarb83>sB%@y=%WWV3e% zEH?cpFIhiAkl>JL3@Fab1vMs<0=Ly)Pd817Dd5R9F8{YVN}DH2+1nSMF0?KW7CKus zvdfV`(w>USz!t<-f`Q}o+lxU{-a?*Sl9un;I$$Nt144C}qja;4C&+6h1Sna7cQNl^ zy%d6}w%@OWBjNXQ0NAJnoBdEA`RU|(#2uE$Z51&fD9&^(g&4{Cv{j+5E5!zA7r;!7 zx0?(i=rKAU+szo$2NQ|{iMenK+dubmiHZgBr6$j2%`_4`<;rO+C7an}Di8d@7?ta0 zBOPSRrg-}+MtliJ}fIvwVSI$tQ z&wC7i5qGq(KcA%yW7M&MaUP`Sp=;USr9VB80Cf8spamgE4+LQ6&G;$2)#!yN4(TJU z=G2rMe;}mO&qE=wfN`y=hvt3L@Nd|5)umWW#x%UyDc8d@gLhM&yda!nx}y=^Rl7SB&{d01z*;ZU*peJnmSyR*+f zMm%Tpw|no;_;G!25YlGNPLAXbuLRNuIx=GX!kU_E`lj3ceizB{<4YL~V+eXISAfi9g818WzPRrXwNFg6ftn+xNd*1u2*_EJECYH!#(-}*>RNX}43r`?+S4tOBnz+$IbE z3jNE@>mc^KP*k7RoJPy3P3DT9q5NhAOOi^7WaboGKp@4m9?1tk6-lmYB{S9hW_~9A z;M~8g0xTF?C^v~fyDnEoG5;4-q&+I1fLf=&x$tjC2e|ij>iB4hp5t_KZ*I#;b&-W` zeWiYaD6hE`fSSU|L$1Zf2hM(#)yD(d z+A2rqiqTnu{^BMU)W?#2bvqF5w>!$2H^Wfkj>)0D`Tr%>uvRN7D}0AI&cz%u&G+wJ zTb7ARs$VA{j(Q`_*OQ)zMeUd*nT%jw6Y%4WDims*$%=gL z;C;BRAKhQ>QBeq`DOnGP0wr<=nCDV0M^*l$zaaq~eP-_m6Xj#EkoQD?l=?N$h;)n4A-&$bQG;(6-=rYttCC-|kg2(EfEY4QShu4CmJh&J*KocRM)yfHXChD<=x4>Uh| z8-)l>botZcU=}m=qU0@%|6X;P&>7so%WexrYRNMHn-gA0B~#XoDRcT9h10ZbqLel- zPOPZp*^v{{HFZV;gYgu(NVKXjmI=^j^eVt+eD)Uxc8R_uSX3S05gk3)! zj3Z9I{l(W}yD5z}{eYs$Eae!1t7W{4W;bfdk=UB!;4r}s=+Q4|pvSzACN+A{3*V#+ z9O9s(tT4O8gYd8*Fs0Ly%;9966SBP2B< zsd(ZY^B@3+!@13Wscg5PsURVCkQ;scuKZsAVG3MQi}3ts>Qyt?24}w6rR$6SHh(9U z?-Zmmx;v-^Qx7fzsYh9TvcDHYXPL#Mob-1UoWajQiXJtx=F-Wpi-H|jDvCyKCbN*g zO~F6QZB|UZK)3bz@E@(jJ#8JX*c|I#2;{hoNZPwwA3KWf{#nBkglul*ip~rhrrfdk z$Mz!w{h^4IXsqi;g5T39;qj>4&N}31i4sP(L+iEf4V>HO(HS5VF{@%@auU9ol2PR* z_e>_v+du243NzE*|7kTcf>`z4;x)humi8TfCYjDGUEs;gPs-u8{dAK%E=V`@uMPlwb*6X>0No@xscOEB5wz?p@)q?*iRppzmES;z%4-w zH|A-vAF~ny4Vb`#j2I@BP)pw_#VN#$&7+CAa&uFws;rTdCf+!BTBAwqdFH{ShxS+Q z4a!x#qv1^C40h^xT29W*gduMUSoc;(s5hHXtRbUX4PJ3fi+fo=O9QLRKD%yBn0vd- zdVD7(Wb4lmEeQD`IHQit$?w&a-^1Wk{6NiU9q_ku2VzpLCS1~#4=RrPHuKG;J%)R; zH^^8P*c&A^Kf!2HuZlxI02vwDb18S)(pw<#rJ^5^Vqd)!;Zh4Y4t&N3kMrmZeK;6> zJ9Q$np*qqzITTSTcCYS=!0p8Ux}c*Zsr|!HnypoG!+lki5u>UqU$k%mbC}O>Tgl)I z7G?bkbf0PM$VVp`DuWDC?(?7P7D}F^uJlJmy?(bJYxk!mA0}m@$WoYP*uCE7jP0)< zfqo+$k(zX@A=)wIbdbp|6QsM%aFII_lqP+3Dl_UFQ02WmB%xCKS83!1O82kQ#Z zlKyl3hh+bT-T`4&%)9iCutw~E126GjFc5gtgzkW$fal%~a%4Q7<7nyHWsjmveh1qC zI7G&PdNjrhM7O|m)s6VkR|yv*85r4Te_iCKHrYiHWLc;r8#1Wc_syLpj{e*9o)6$m=R^=FU7l1Qzd0KAGG+K;+H%<;#8VL9P#LI# zcYHURqf6au0sVaD)Ufl1Dwt;|^=ju{jD%n9&Ww@jY1GoWT{!6e&;L{OhBSKXoJn+z zd(Uk6a}-#V*cGilxNXlpVuCJG^5*NQ2Do>c5Hw_d9@i)9Q`80eDi@*KVC6hQK(a># z>`>qJp`#60V0I=%2>(m-{NFi5!2xI!bQXHxxF~vDOmRxUGIe?Yw&tdSaG^vMPZrSq zC-Vc!i;A9u*T^`9Jq4XDTj9h!e8gX|`H|(W2feEd_}S~`3bUC>h3XWHHtjEnRLqt? zwG2?vA^%w3XL-`qJC67n)#CVMrVuUY#MO7)#KEdzUI}l-Vvlgo=1u&oq$GL(fKbI} ze*oF;B#3!}@dPbQ7*AFzr*8cKj*lDZ-#&w^wj&d4Y`eH0BBp!h*Imo1-Ung7DFdh%(Pa)RZ! zNKO-=HVUZd%U09ZO$Wvl`Y4#uoix8g^1=c$;J~05*7~N0*nj;gGExZq1%?>xe4$2! zZ!Yae`cM#cPx~v+;p!dxO-syfC5ded7u<{YtL($kic=x=`z6%#<`|u8?Y4RbE&?_W z*g_B16V_<|gYF%d_x+5E{fT}`f4kg#AMriA1_*u?hEWHhC^k(o`@aFg9S#!t9T!+J zaIvK|`RA893l1$Uai~}W0*$^r`CPPBk2*7N7(frh15?wRKOOJ~ZSlu@pGrs<8tEpq z1ORQmKO)#wA#YdR7aB_^ErWqSj{n*oA2=X#o|9(?+V4CbUHZ9}r5Iv3Ya{QV4Y8xN ziN^M?%Q7Q_u@)mY16voboO;;zjSXt_7&|#MzMGQIfCUsNh}6Dmd>ARspvf@?&w%^L z96nZHHy+k~noaZN{#wSZmm2#lHN{Mr$^a6)vV*-(ZL}YE?GD%bzTQPz(?oeP5`JX$ zWfh(js(NVTz3_9#e>?yPiUW#ZCxQ<&?!~$lZ~;r|(3sNPDE4$ z2AJS*l;qeTM~3QOoA|Fv_OTs^^s(AUaEd&IX?8XFu(1u*0pq|xCTh9mcnStBKi;sY^-msa88@P z=U0u2KjC%J7zKbB@BxEH`>#(!RCs~6h=0k}fCff$ajA~!aJxWBzkOTTsF>$qoxLcg z8zk0dWA}9hMxm08S4C}}_(%X_m8URh{2tIpO|nsV7O~+ujMAvWM=0|8$@L}WKd2T^ zePD#Uim+aE6$B%sM`^A5A7AZ z+U)@6EWu!Zk04`seQBti$eah(dt1>2z~fuxD23?8LD)H2+K?0o0a%xEF@wI1=$-E?hQN?w z6coZ#;=d>K&x8HAvj*T_WxY^*E)-Wu^h5Zi&AV>KAbxZZ5uE>0lK+Z8EtWSqX31o8*;ey-prPrm=VBM05EK5AajJ&gfU`8gEYaNU*D&7{%##2`X zfnFD^82fK%`L`a5jO&53gY$l!IuP>lN}t>P(nRx5U|I-P`Hu<6{xyLvPi|t~5Mkeo zek()Am`oVP*I~W)acOf=E4;tbjepSP42I+fkVB_V7o?4qhUn(-6dAFt6jSy#BFLxU zi&u}|Asn#JRLb1LhY~XGjZn0`JlJ0L7cvPD#7Wt~+J{h{B=;H^|0 zU?Ky=#puFYxnQ> zl!sLm7!$J6T)eA2GTp4+ox>AR1eO|xl7g9~BUzTX=bCmaxVU2ac6u6txcE5KxCKW3 zwY`7tPLU9Dy^I9<{>>OJ*vRp?YxcQE+9i5W;Y%HdclfVV!=i4f~HHH`WnKb#gK#XApm-;BSSQH=SV4)lo9g%(MI z5?2NiGzf|5qSI9JoX%GX0kk1b&TJ&#`t3Idw)}aq0`bdGvvbd`({c1Cc4E5!10%q_ z2qD9Ebhsh%-7JfQQ||`$#a3T{B3BIm(4+)a^vYqgg0=D%#+&sRk0AsoOJH_|jV@Za zoGQPvKd;)a&C+dYN|Yt>{lyahT=74?fhYh*seARLE;%sxSUSW|2L0nRDx1qi$p8iL zXVNgpN>_yFOlpqpL!Kq1ETmq-*x>J9|Bc`OdTM<@sq7~03j7Aq& zWedX%-Pq0;6riKn4?IS-`7wtk3U#KP#v;ooYpOF{wA;hKN}Fo$9d5K^@ju_EhZ$1) z?y&~|o~gRL^AJUi^8MpcFr&c1&tMfLP23u}yq^FR9v@zGKRvMq+iLnhCt(ig{TMH{ zvPn{v1tI9;g^VmZ`N92c7~B0Jl3j5t5^;aJ)sBkU98gO0n*Tqpt~xHNt?f#Rh%|yA z-Hjk1jnbXcrPP(~u2BI+x`u9~JBLQ3q&tRAVd!qYGZ^>2-#_RNX3pOGiS?{!?S0Pq z88T)6St#HQ`gm*aq5~X<@NC>%XtyjM>l%|pp@k*<#yWU(i+vEyWk!Dl75Erc2G)8L z{V^xGloC#_jHIzS;^jF`dYKmWs9Ik<-`T|fQFfOPzY{U8Wsnj(Byph$OxT)QqZE-c zTK<53@kVX}VS}oc1N4>_xL@sUxSz3T}Wc(ZdE79)!qg0bzF4&~G0&}s{wDs#ysapt_UC3Nzf^2~^9A`#U5 zYo0W>2WM>>wcF>bd()t2!`N)`?#MfJZbFkqxL?#ZG_=DX5QW{_ts~k^Ao%`CAK*lU zMTU<<-$fEDaaerNy>Z~N;xk~T9eDKzWA4c7Ikny9tRh%DwS3-TEc$6hw4Av)4z(+0N$c2~>!x1&r5sTjZBd|sIdH*-S+(kbJ!UxVh7Cie`;tJcE(_D+OzOf8^!S+*RGSx zEyQ>qq~m50oBhvB5c-gD%a8-)I@F19744Q|H9oX=^xW?YhaloHP$A;;Pw86|_Se2i zB+N*!{r_bkSi+CWHnP)Nw}qD5E|Cw@)t~`^Ezwb2HpM?qv_8`2k`cpQi&k^(%%_Yl5wH3-?3*jQ?tNzZG=&TL&vh49pKa zYTSV>P;NX*sHPn`fD^$Lc*Eq^mwBJ6)p{&3lP=-5s*jX$50X_-*1R0cIVBhPR>|-zI zRhP4~_uj)o*Kc6zZY#&<;o)6Xbh8ZqJEts~aCy4LpBEaR{M*r^a4LFQ^=bPJg|l|Y!}`E@^Ep2<@@<{Jk{aFf}-t!k?573e>x2|Tdu!Eze+o%@<%r*ULpp4wuc>FHc6I0MdxgJ-< zY&|%i@p}qH(GUB#c`Rsp9NRB|Z3ooS_+{0iE$jUEmZ`$`b*|X%_|<<<85{4#i6V)( z@=GeFnB^JZK4Ep5j&*=~7$K@6ThMv7aLG47CK@L0-#o@lWkHGgr9%ArJ_j1|O|3QO z%;7g&w9Z1+MktH@8QmsIagwVv9DVNLoe=`lx%q<9+(QXk#UojSl6yQ%(%286e6*}3 z^#o2`TlO*MUi2=SE`Gfm5`J9}73HAA;C<50_urBC-%s0X`iikZp?39sNU4QvTUv0ufa$*r@WXj9YMcTsRZ6PP z03O27W)3E?4-zy3M#OjpKU!T&J65{xANA@3?!|Tj#Dhi(ad2b0Bg~zmx=|x%y!Hw3 zK?j+Z>N9+;j&p}8iI-KRP&Jf&>MIN#WkUb^ZOC)frOZnKyH24z3MK|$F+Ze&}>Bse6WYiC$vqzi^dCq9{HX_7+Cl`MI zA4!fV!&`(-ZwKbQ{U9ncA>_At2D@X8O99sWMDpB?{>0L5cwKTma<6`3Q}|MdQqZI$ zOTUsv%R37H82jq}2nG*NAL^;k$+1t)h|I+R%d@2IC>N*L9gShv6sfRX7jtuSVdeoD z0fFXoZ_lckSsqo7cB`LVFwCCUf&Ji$6NAV&S!HxYG%dt6$2kNs3J~J>f@K%2drJ7# zN*wR%8sTZI>o^7%f7|jydKqKSYCWI9%OzUm4-rq|u_q}JxN&-7UfxbrA1Mi#bhM&T zaJ)d=`BdB6q=c#H{nR1WNK;y)DUU3a%Wbkqr^r!PtFJB+6XK+f1PaC;thnmzA6|^I zq?sY9&(t|J_^<*t|11)wOZa4eEiD3#!tWj;a6uEqSsd_Hp=o6=jPZG~*h`JD8M`Ba zav0i3w|0j_6b>kuaDha*qZuUtEe%QyGVk=YKd_`ac5rp6ZkW+OXs^~1X{fcT^gyo{ zFJ{zZ=G)vL!$m}+QECxt(7&p%jC4f|jCnej#M}y@eyYKo|KT3u^#ixB&x33wfp-sZ zy6B1vF*P3a@o#KnP7(ifwh1ijFDk-&f-Oq1kEnev`~;%=xzM}hnb~5xk+?>qKJZMS zBC1MHc}5qiyXgZzkO*@hov(addsfbGa$~gHmBDj9WN+Ku^_D6dLobCI=M!6tp8rpt ztta9LVF^>N?^-?hp5^qP74DwmcX2bF_&COK_;&nbB7P468Cw*1UF>76=298{O~h*J zE3agz1Gi(mvyF9G7ci0x4nwLIg@4Y z^>w1z6z}%$vYs#wmX_;p##T?&=zFgb^VYvAa+Jd+^JHDW?ZB5S_3{+tk& zke%6j(;96u9#j5zGER7BN}8Xq(%c&=C3~Am$hfRyW?ZfkRl`Z;c`>Iy_+=t*HPiCt z*QwLtAuIM5fO{TJeCf2iS3?&~3obyI8F+u9WNrSQ-1?7IdhvedV&(2-no(v+ZRt>< z#7xBpihscoZLvqZpo|>yjG3O06N1)n7I{2C(dO(V`||X6eEEr5fx<7WoPD7jkQ(c( zkKb8{qvU1_ZcnzbR?Ed62Td6B>@&HrPJX$EScXj05R%>2M2Yu;re3?4U59mw?eciq ziS~Hp84)z-F9RRoZ+yQD!8p3$-zAn)3ywW?&*nqCS) zFG;51fBt-%qPnGSjs+qwQL2aE8dlXN zN>H+wzJq5sdY*eppm7bmJHag+=b) zSO3)6(|L>EG^0)msw5cn9;v1>_W9FtS|pb z_J-Kg701AoY)>-Zs?q)P8Js(dZh`k&@wf z`?zAh)`;_OSYzT$9?5)pUcAGR-#6*qBQelGO9?nrk)VsUFLIIVi*-Sc*!L2TiqKuh+^{qnUJ+n$z`|4bH@hKrei#3m{&>)r z#)jE|vHqf0%%qWwAr_X{DeDb%7%~GPLRAVcXKMnfI_+$*yG_yl>XQbqQ~i?0dcF?vVHwL@7UH7T#e}* z!zc3FC=?6QAO*^1y}1`Ea~N^HOLBRmIOr9Xo1f z;E-#;G5}(D@sT|#7;om5fz8`Sp74(wag|a1i87PWmW&iHNS6KkLY32Yg%LX?(H#bD zN>8(@M6y@HKptfPc_i~a`{&97FD~B-J*`rM8K&S5G8UMZD+yos4_wj)X$0JVC(%8G z&IcdgztP^;DLQh2hRtkhTjF8n;@5z~E<@|X)zJ`;tx(Vl9J4&?T3K+8o$Xdhj&Zl% zvhwJvbN*S0B-&MD#noXpGPS|-$g0Elu;1;%QLV4~X;dEN&(1Y_{-Xm`K{I zEB7_SaqEY)NWiB!x)9q&7pz~#Tw6uV?SymtkvvEgniS~jTX6an!9Z}Op&^f6RM*r5 zN6%Mbsk!#_jn|A32ngpG*svJTp6?8%jYdTQmW~HZWu~~|klSLQZG8|G8Kb&Y-W-va zm_}&ko910RGr^-bDx?e)*K@~uQMC(v~jfN+=BJjbwiX;EM+P!#T^XXG$$ zw$xxWic({scJ0KGO9_Y?Wt7O-uYz3k)`bQZ@*KN zA@1HkSY7J)T%OO{S-YU>I+%d%tfg47{|qS4pjHB-6$`sYKHW&R{r1(=H5i9* z+f~XdNWOc+XdoKnKf#iczw?zy9WeM zZ`VJ{Fkg)P6LlabLS=-6mHkk3NV;iZWhe0FV0_!a8U1oSx`GEP01ns{1D@$`Dokt4F@bwDwMP% zEyec%%eM@osVgx@E~!AO>BBxsX!I7vr1Tp=8%%z42?S)-*J)}JQ4Lk5l~;M0(>ip6 zm84Q8yg$O^G-N&p4#<8rTM;wL1?nX!lY4(6K0+!pmFDB(s1XOtkRDsK%N3?7x)uQt z;<9nqr#}cvqh@q7E$ePmJmn#weQpqMXXIb2oy!mp{s^!JqfaPDNK8`_!cd(fmq62Y zm^1FqYih_}?4#&I9zQngHr+HhfeJ}Kr}XnXL?LQ$zTwUY zut%>Q^^i7SvyJ`oaQw?nn@J=hP%B4?YkgtLFvf_##-_wS%lfKrb2h>9AaI+|{m2Yr z`mHr)sqX;eO$Z-15KZA;2w?-P%UHSCVFOLJTNaxUW4q6##XIq6{>JY+syz+YbZ)zP zVG}Zm@~*LhjMv4{#*QQ-;vh4M@Fzgvnsqq$cWIENjL`7rbNql5hpSvtFGb!L2kV|nO<`>$nPK-comq2KOGQI?$c`hxAjs(G9(lv=l>0EMwwLRuw2_xDFiW8}=kE&7gF{sn~dMRL>>eq-y zvG4)`)t1y%CRdpX#pT$wYnzri>)Wic-qyp}YzO_r7w;Y>-cmkiO9B$_SCWe2gPi-$ z53X~}<)p$yWJ84$$G%tCUusGlXltZ~T*2L&)hs&nN$&|y8;ktHS0$Y+duOSO%)wWG zqEHuP<{Wl$o98*^yIuIHASAHh*xFa8T$M?%wKkNR3lZ&wm|v@|x!!$?2yxKRoZpOg z-K(Q0$*&ukMooIfGfzIhqcWXamI64l`g# zibj-tS2lQPONQuil^9I(xRR{W;{63H8W61z#6e|QF=O|2B3;SH)o`Z$c|pgK{v# zr(G&QpBC%!<)&U7vhNxueIkc@^f}J23L3u>J1JsC^59Ehk}k78>)KG<@W;<)e=Lmt zP;g9}PLnqb+);YY$!EHgb|pTGB*GMUY^`=Stmcf&0{G0G+9X)Cnwe_iaZ!w#BP4tpoZ_7*I> zUJl%CVhCQuF#)bb5AZ ze?y51If1{T5ClDDMUiZe2^j!K)E|n*<2(k~1_d?p6~rJu4b@Jc?2;r=D4A0GMko2> zdkZEuIAq@8l3m&zJ5^_0<>gg=s=3`1sm5OOxFq_mD?{;e7eK4~35@o}Ark=!v?>dG zNp$Bb`%K-g@gnZm@|uvDK|zh_u@R-ekYXf-oKSz|07`bYOS#9SdGL%Kn^hQQ^`$lMjbW!glu*uu+kq%z&V4*QQVqg1k(FD-a0#I*<2lI zpZ>S^3&tHpssZ#6XqAG$H;(>N^`4d4eK`+&R6zPGQ81C)j9TyEKYZ zVJz6NcR&7ZAtEM_{hSTAC>wiPaV+%KWZ)4C;~Gcv8HU6W=X*xv~#1ixW}3guwt4Md|a<2Y}s(&e3@@0Y6v z6G-rJf|6`Ga}>bn7T{5x>@1YqhmE#UP<78nl`=Z4i|6)W4rgDBGCFmGh2%Z4qwF6P zkpN9sKZU42nn~!dB{dkL#>33V=iM(+-FHwkN5&>X1Pyt{28C)EMQ(ztdOENY70M&; z{|Ou-e9(TA4J6ITc+?R>OLd#5NiD))zzne{1%JGUxWU}AV0~ertMfR}PgB7Rl56)N zDaw$V?O1A%#{YHyB{Z;yT_;ekYYRK-Z8snv+%54PgYroJIJ+uR4T)~AnZH7wMRW<- z5Cs`I`NYWHwrCU((K3kqvaPjKtC6}E)xCXiU_7Z9rx^=)`zKdHXDFdv?;_hS;!V`I z(0{O6APzISAMX?XzyXSn1LDjhS%{pZe-xvgs!gMwVWp&l9_8}LbARt|zBI;czdM?6 zdqh~m;c8?vDNQ2uAb46BQm$enaPz49;Ls@!hv<&S3nP*N&wbHZ7&ttcBEIef-XkD$ zEqiF{lmY%u_S?Lo6weSnPPSjzP7wMl96EOiS$IRA+=Q}M?*rU2tLC1u;)b{PyV-ll zCqFEr1Y|X&t8|WL*+TZ6dI^#0^W}J9H<`DY3tHsyxoBMPOJT%4b>JS(r=Hs=uz|#L|Beq z)IcoW4?W;JK3HCgQQce=SPJb?MMTV_HabzDbDkDcz>)sc@=w*|WwOOR*8uhl!ZSq8 z5r%VrPrYRlO=7Z#UHxK}56xjL{ z^)2*xSQ@YZ7`Fq)Ffq3uv^HRh1y$YT<=v|8fnH)0?QBO}%7K5l!`bQO3of*(C47K) zYXND68T)))h7a`0RoX@ahW0dOwp4%icxi6}zvy*^BLq@Mib?XjW{A$|@yQqam{mKW zdP@@Xs~-#~x{0BepIf~_mV`I)Eus`9EOK;DwHVo=1jw0GuQzL8+3)moUUvVP2?BB? zyYe?H=nAzXD)Lh%qXUAD^w&8$Xf%eRB3a&V7A4PJZX~OUx52Rfjd}C(4$%Yld*;7* z9R&)?O_f{l%-OCsE{1!sg^0J75K@h#>Ky}FB4o(@5eroIPjS6kDu(A8joS2mBt$JC zh6&Vv3Le}&6e@5Wk&Sd1)>$~4L50ZpOZRKSFrF5fEs-TcemVtw@}rXepuRJJ#F}w_!pHGok2_%GJtH%Uv^zNzwYQHd-0}G&-2W4>DUXx z@?rlsh7yH~wr9P15Gc;WQcm77#$|U*TpOV*{-a|Z>!;V#yP*3!9ELbB5-8Um;}1yz z{vw;!B&1vH?INN*rEIljKhw{=n(^WBXDCZ~xjp0;OqUYoCg?CkNQ?9M3|ggX19pp_ zpjfmMXQF#NxlVuoKNkb+cMkwtJU?jwS^f+gPhZ+lEbX&5<1;)MVLWqYyHh3JKkfP1 z-scyz&={n7=tNXiZ+BolLbV&)m}s0}Se*N$uD~eoWy%0lxBmpiq$IEcYtb`;|NK?} zG@*$BPHn2qaXL-#PIAdROC%&a10``7XX_9B0t+H!&}QW+@xNHPR_(@y5GG5cnXGAZU>*sD0Z$z6ey1f4r2FnUx8po*+FhK5GnK z!kmj)f~z@q=mY#P3BAW<%eAgEqWX6MAOv(Pi@umIse;l=Pil+UeujH0x`%+E2nmiv zfad^I@pv>7Q4c0c-Ps;%Tg}fo1)um0{E6hhm;MUmVuI{nEeFn{P}YiyBOj8bQZthT zg+N7sdaxsO{Q8O7wAmk;7@(Nu;E*0XS3g>dNIntWBe9vDLVRq8^_OVfp@no7TzMSL zV)=K@l{jbscfaMZ&A(cJTxYQ^@Y+n=Ec13H(>JowLJvP9~`iPuk3hu0r;j|Dc;ukW6g)%JCIg!Iq@ z!XD5pFFe2x>y1fXQ~%s)?$Sd;bkM~*u*-@ykSgdFv+`d*Q-oI|n$3$3SD!z}_)jzq zpg6^?1QNwL7 zP|`%DmY=}+FnAm;fIguJSK9)22WraQo8Srh^DF?D^oiQ45*89#LLv83E1SM&!U2D~ zzq<$)HAFZRQ1cF30*KaMfnZpCVV3)~!(3)y_qXF1`>LP0c?GMA+{@K0B?7U}(v!rs{Ro<&K{mSEr57)NC$J=Z1%j$vmdckpFP!JI8}6@fV&uVzV5NBR;=*bC^vbFljhs1#i)?yY7Lw zPITy&WfN9Uz$f8Glg6V<6Pm%%rq1mS*DAX|6bbqyJH!7cTRAoT;%nkPx8L#ov0u0f zBJ<{#m2b^tR9NhBxDzc!P=Z*;(`zjo;S+0A8?H@KF=HU-KsP9u$(05+BKF1jruQ!f z1_#Cy#=RgvOYY}=!dkwZ>iN5~aL0od1jReqh{GSgXtFF89Q4h&P#A7E2 zJI7v;(w*-_8Ag?CH>vCGD~yJ~1WhF2-Hu3aznoZ!mXq!Ac{zB&(qJ4v*7KSbzKny-YbpAEvDDwB?gm-wuv&zm`rnd} zRVnJ(!i?1*q@Qwal4K#}rDj&q*yykbBJWaBTdoIC!bqJA?qVZRL?2tNqQE9my4O5C z-nPBUZ_fM3@bfJPVwh9-MHOeAAxT0oynWAaNc@8@756ZfsC5!Haim=<&+qEWFp|sV{DBYFA3W zxJ07Bm{t?o=A-sai{c+j_xuZ^*rE#$;m%51xc{%S)E{e2DKnflR2O*Z;rSBDmI$zz z?=)&D&RgfX_0lP1a}qe;8A$qlOE1nNhuaPIo}z(WT5_vGK10hRI*=1uaS)&bl`u3|E5H1t5iD+TN?TEH+wu^aRhLrQ4rpL%l`H{ zBlO?-Jwss=TtMQk>Grzkk`qLCKBj3H;b_Kcw!rh1Mc^*L8|>XX)V!Uq z#a}GNO3kIRtc!T$vgnZCs@}6t4(>%W@e^%*)7euB+x_IZ;G#=R`WABWgIs26itvj7 zuPt=;Dip|hn%5Mb7dxtME1zYA?)y{w`8fiDvvYq{8r{D$Lc)*B{U!%&FIR03`6x;E z&+k&7x96375)SxP9f6%{SqG$psFDW8`Keo^<5FdYVznY$ws0Uxm2FfkfAToR4a|kW zXiOw2l2HT=7%#_9vyP4`+E-6#3bW5I$yi=%&N8Je4S$T0c?k?F%7PFUgs5Ch_S7D- zWJt%L0oejDdmSB{X9_MnwKJv}U&97*6y zTHs(cWjX+I_aJ^ftgGt=bSdv&i38N~17QBXv@gj!3eM*1vaKMg7v=G-bqW}B(|YlR z_5xD|&B{`q9gpTVL75GWr(yXMXM46|X%DzMGZ6Vq?q5fVv3-Y|(bx>ai?{s8GV`QoP zM3;c|DkRa~?GHa7?3&*VkAodN7dbZvTB%}15E?}q%8{RS_q$u>rl8GQQLoo+63ytC ze3Eq{K86XY9^xM3fZQES+U>OzOmHwmZ)|FhEKasncs1jHWIcw8}XL!F}#}G z1ARnu6T-1jmKS%D%MXwxun+umUT*z9JQSc&KoKVgkz!^k1&a_Bm1Ofj~M`RLz zcaoLNRk{D11P~}?Th!F4OxN|O_|$4r8i{Sak8}h*s*7*Rrw(TD#w|I_Xnrb3dPokp zJ*4Y7#k>;jh^gw(bw2pTqQjj)$ps{<6FJQ+zV5a`g4K4BAHVI?_Vd~Hak$%BSLy6} z#}FuLmlOf>0u7)9g$gL>8KmM!4NS)jH(sedDAuLre;;f>^_|x$dLnql?$7DY5 zD}`^P#q0P(`ep6}5phvpUUm_wBFBOqEMTjQ;t(E%4VH?);SUN4 zVf<)rJ1t>hYCRtNYUXn(A>mj9l3#JBc*IQCEpQo}tAA*Hpg>?}7 zfDX%jB|Y|<=7vC)qA#|rdZaGrQ%W5@a&l7Lny&hEIa_sl2dIq0;`PMr|CYQ(M#@N2P>91K{y?q6b~4T?NM;Lnk7@be(5 zyPA<}SJO&5D`x~~sOl@A%Z{#&oS4jcn6SXjC4OufBxdSE4s|Y zJfys@2AMfa-z>9FQW7OR9=s9677M)-vH)3of3?VAH?O2CihPU4*TXg6>9oVOK(Zw8 zGUxPb_q<3FgW;@oUQ$*pcGrGj|c-i5Hti;){*jnFK3*6(!Qks21+ zNw`BZ+d7G)g~b2hKIMWPC@J1^a=dgodEzZ_8jvyFu@yReap34(eRP|dvi~c6D8sqk z>k26RTYF25L{>I~#51p)HRxxcD=_i-e**()Y>Fh3A>326A_z6obi zp--a6SGFC~Dx1&C_a9Zf%)_Vur5?6WBF3QTJG2cQnzYu*LQbpnO}yn^&Tw09^heG& zaJLb+2m#cNk1pq>!x9^9AC@xg{$>?&k*0l~PBER~H;cw$JM;RF0!IaX9ugiNsP|t@ zf>6xD>rEDQ($uVJQcj!npes-u%t3EEMlS8`$OUt*R>*A95o^`OaUHBZAZgR5Wu&mz=V-cGCxc2`#d5v-GHY|VAjjjh+Lb3_br`M|g{Bgj43zhF(y@ADix&6iVj8pNWwks6$k zHUSY@f4iT5UnL{EdLk7JaK=x+Se+ysZHL->kG>x9a7*`v6%%2LwY!!)(^*#XeQ9yN zZn!KgMbm{>Zl7t|h2ZhED8)avd7g5KTu4|TL%c8MGunN^; z$EIS*V^H4sXX`8rH&L!39H~4KNhAUXt=%tFECY5#TN*aGlyA1gigx|>*)$isdk7nm z{eAoe1hSTAA;UKvUp|wnlZj?Fyl%_h3`%UzG{b<>3Prgcl1@NPN%njIrn6lJ5MirB zt3*oRt-hr?ANVv#j&=Ie$JeZ}R$Y-V2-AEw=V@DYlSf73pBL@@jG{GPpUp6{_Jx_g zGvDLM#}dGn<^?*Nk?kNe2hY*1

gZRGKsqlfi&Ei1$GB*vl;@9?kw{PipR$s1 zYr*>2l}lb7x0iI);Vc{s6Xfj2+%~*BQhMk6S*;3GC-OtBLRmi2$1U%C7wb6nbz`S) zin~1joTplT$@6;moPnMN>)RwNiMyULtwdDYphi#FJUywQ2b}=Zy{@6Ax^CBIIaO>a zvTV*~`6@3-WxllJteJVN9NQRg976~%e{ay3*0yA7cyIl3>E@}L-A{W}hfVFsPrJSz z0p1JTl;7(4LC}Xq1(HDd&p^Qv=as75o{PGbcD( zvsq2b!UulpUQl7&00JE)k%r;K_(hV-()~MK>>i>d;_?rj)y~dl)1ttF`}Ufn&;h3{ z-92$zu>7elo=~Q3!1HrHslIYnkxMFjb9(b^L!ry^rS}C{H#g9ty>>dQ?D+B^ z{mKp2t4`K=F)bKOm1N?%?JPMz)Yhc$90G!9uLt-3Y$ScDMLYqmST!5+l&<&7 zF*wLtJOb0f#-KAMhTa%;u~>`zZ+Rh(P#7p799=4N0()YG0nDmcBI5 z=TW`MLbadZB^0+-h!WDMbwT>4~9MxiY!b=i$%Of)>)|zHg_sgdY`dh48e12 zk{jy0uC6VhcB z6*?eI?xax;JA@kI@5x;68lOJ5Z)|9AiWY1z3Mzz|W1pTSq3(D1D7r1g*mXttOC_ac zNK6g93jNqN46#e8Y`$P0O1BX!>KoU|X7i18W6mE#8o$)$GIe${sgJIQ3NJAhg)1B1 zR>&TniuC@YTx;G&{O)``3ioWwnE6MHa7oWDp^|;PiPEfu(4Kd=i9+04O;Sewi|8!O zxy>q<0=7n#t2U{raS5mnQqAV23ll9k2s5+j*gNr6m15Z}F7?Ir=+#Q9)-F#B+X41y z%#F;?)C>3d^G>DMCA{05RH0PcVXT&)+~dHhl<;^sAgnp(jET{t*-mR-&BrW}^YOgl z-pePM_XtUBXf)IH9jt%cNTNA~ve_fuooAlM9xX6O`jg!O<$gi9^5)pZu;cL2_g#rs z>9w0tpL&75jQL|!H+b9N7`gQ3Rd;WRc$b=FKKhF?+u_)aaS^^E+k=95WEaNKIVMnc z>ZB>u#@B8nV7vObrj>14+_$sz(Op0zAXgLa)kNaLbc<`bM(M>6w}K zIs~rdCUtCM$!4+^C+T(bqA3)bSB>9XnQa&xw7L{cSUx~KrpWhjZa7R(H+23o>7C=0 zmo0cj-6^dZ-3k>v4zJYTG`PtdopWz#U~3k~}e1rs)Vb&G+KUV_3xcx@r7WSp@5F{`Ojn6qK%5R7Cc`$<~{qnG!hj zix%fg+)ac_93<+1>@dm#eRI2e@)0ONMR_->+V-ulBkL~jZY4f4qWD2c06%a$uCYKBn)N?r;lWEC=@ zL@Lzh7Wa6Q)%t^lf~kU@-(D~J+_$Nz8p1l|*_et(?bg*8vSqWRD9y;9_x9#>X}JF3 zf>h&uYI_b@&6CP9zR}h73AwNx&q~hRA^C|Z^sx3I14d(cMFXYs7%z_Ot1$%QBGuGA z^H|`q>_vuUo8u>9&TIxgCAo2E<}S7Q@+u(sF<{7pgAoz$cPb}EyPs=VF6tIT!bEb% z)eVnV5v~BK>k0kfZ6@iHPSFoI(#4|V-}7$06Mle4@gOr~j3T?7d5~kKjGxW7~09UnIfhm0OuTwC`eED_y){6c793UZU;D-WZ$ z*s9`vqB}Yb*CVq-ibeOAkMU+`kRurvzBkF#@XlV)=Ku#XV7VKm2S_p=v1Kyo*)3KW zQ7+-XkKoXw_dXq>;F4;f6-p(LJ8QeSPn4{+3mWYPgs1H{ILD$omn3U2R8$SNDpob~ zJ^bTljE@@!*o}{>B-e-HowAAq3C8&}I6oKT!p^o&@~XU>qt?w-0ZmU=)&=Z?jy zveW|g$D?OU+N^4>V79|I7l#?4S$hPi`h6s#5#%Fu0 zsCw!;qfKrjoRp6D@-jAiVrW~ZfgumJ^YajL_%I95AKG(XsX=KQ%|2G0grdg_YRLrh z4H>>wP}?E9)TXRfIE=yqxuGOY@^0{C>E5oaG|7~}%)qGpZizLWO)e5&d~#`15@V`S zB7c=P_?D<$!Bp*{&uP|x#>yAs#6i)(we#9&?RO@>l~iK(ByMpn2qbvC5-+4(#!BssKWSe8Xm? zyD|!T*o&8fDE`&!!M`eiNBO29F2Fi3-YJsitIq44YN*Q$RL|5rLN~KDRG_Jxc zK(w>#0Z;&L9_BJhm8gvz4A9f=O?8&9qBdcISAerGTY2@;0LPFS|HxQv&HNkxovHEx z{CE09U7uxL1W?0%x(ephYqYE&KfFsiYp~*V)W{`WW2~;xd1RceHnWvO?^EdRC@+)a zn1XaYzE<^%ZQGrxC#B@SF&3A4@C*U@^AY{y_6Kc=>|~YNJ{|C zjM{#_%6c@;pl|fyqgQw^?V>7?X2^V%s}oU58kzWOcJIu_i+o+H`@lI8R{j+mIFdI^ z1x^#Yu-~d%>^8iquX3dl5*pUkrQW_co^H9(7%m5w$wUqy8m zj2qrI1q~aC6x1AkaK-bmK)>6iyxWqRJSerNLJ3lOVH_9)v(DEXFfO=Xe*Ee?Pu_xG z8;xx$(x~g!x`eMJdf}P+_n;0_N25p=7(Y1qS_q2WcX6aICmda8IFk@T`SG|hl*w5n zNNEEWaA$`o z0L3N8|?ww~hZm^`X~0tT0~%>g(v-l5ak&-$H8)r)(Bn z>Z5WWu__5sHEl*z8*Y}iz}itegDmvFenn0+U01iv^xg4sBh67Iw6(2zaRV zC9++s(VZ$}-#Ru0b#Y9c_hsAzA$G8keW>wKhRR4US(yzZuoDjcqIi>Pe8@#<|@JQT%O+8mu|n+ zKW`_T(&|Lj@S;oHZTfsn^_H@XeHj@J`(MHjuGIog6g;d?%DI{|UtN>_bL5++&x(*n zenI(nS6GHndC}MTAKhl+Pa_}fZ-9Pae%9hVxm9|0v)*Aj)t5G#N&TzL*~&l8<*|~g zSb;Fm_iuci`3%}GHw>z^rVVcg!UJK)v9DzT;wrN&b}CiiBWpduWq)k``r5*0?HRCz z;I3*^`ZSizj$KtwHv5a1tbV0>5zDpOa7mKdMb3dETeaEg>BNFG1C6sqvMycHQBE5V zGN&$p{=H2%mBQVK{8ox^_!N|T(VwHH&xBF2-C`ZgmEc3vl`154G}6ebBtmu3BUgxT zW2s^&z56AG*8Wn@h7oAcyJt<=p`7(#w%BP;8^tMlKWh?cRAf{JIHTaD&A!aR*iLNN zG|M$u9v7KmDNIZGJb2WFgygmE06!mf*Q6NHE%f&hl4g|K^*`&tLQ9Bw>RWumYk;e% z5gMHzAOBgFUwt<=9wG$Olsg;=lbQJDiu>AYW_5+!DT`R#rw1L6^t4q`ShPr>aM$@K z_PFVM-JlDq^UY1549xe2vhg@Ed4_3|=Q~ghKX@V}4T`u2^H4=%UW^A#0w8ulndO(GpBg*-06+hKZGO22k-_usD@uzYh zoznH=r*?4(GR}UL%_j{`K6}N;8eU8pO$`{Yhn?93EV!i(H7YWL(dJptu zrB!g)(Qi}VPUS4ORlT+cB^;}G4f6(1pp2+cZ{F@b{6=egJs&auYg-xaOAtNzn$Sd_o2X9+sOQ{t=Lt?~86 z3byDFk%kwnT&1I-Zix8}WMJLm)np#I(=L5pZ6>FTPK^WCQjaS9Lhz zL^vMgaJ0Yb_?BNF=^l3kCtS2uISQ*m-cZZ8;HVLS&*;sCmAJ z!p=Uf-V~m5{_6fjd3fNTW(Y_K{!;dBHm+uq@?oEBktCp+KM2&fH2hANfAx042Ywa( zZh-pH9mnyTmy^?azn@c@QGPkF%aY|_EZ5uH5Kh>me*qI2!C94S`@zp4M+A;dd7i^3 zeo8$gMVi!?Eot5`%)?x!QL6}V2@A2b4zE|`qR7T^LA)=ut2L(Fx6Qb6rfGAadh0Mg zw-Z2^zrC1@3a!PGrb}F+K-@SW*x`OtK}>J8n3mIrm|s1AZzixNZGU8S;VS@x)gd;6 z9m}77d`{XMJe!amJlm37osm;j6@ARi;kEiH{ zT&;~{x5LlxO9R+TTI4?`xwD0wafC#^*}i!5r!fjxO;gCXn$ny7t;Htw40?tPpOvYH z4`A-3devWj^z{48IE42L@$#ld9|K})BQ~6a<5V&H#YM<2$7a9M_^Y7BTlJ8lVnx_- zO=HtORL#DEPwcp%*mG^x`Ez~Gufn`?j>myIYY7)Qd#yE&3s|;!jfcXpPy+jnqi8q( zN;9zSI=pdU=^aC*VUxupf=)0e6` z>9_0yygD;v+wcn~20c1+X`}8e%{t~~50rS>x12YsN@kQ^IU9mfB~D8Cv$m=NI2cq1 zK1~PpbTPAla2*&jTHY?Rs1q#SW=Q+^-f^FBy%gcx$XHTzoAqo`2v4M}Yw^Uw#;Nak zWbky?a(9lrOENWKHNr0gkAiXp{kHF>w*c>cmy-6*hT z_qdv)Py8;&h2Qhm56e8u7j%y3ocI+8guv^y;^V+(9{Cwy+Q3~64~ZE+na zj%e46i(gqU%!~h}+03dODN0;Cp5-tW;L$1QS7{udcLK+LtPY&3Cs|%!J7Jfwtp%vE z?;NDGmR_xEMgZ6cGcEvsnn;9*$IUT310duyo=I~XfeF_}*9>A5YMOS7umnpqKV+y> zwcSg;ZJE>|@w`YYVDaRmt?|gTZ{bpuW^igzZa`nXn!x#{#c!^~U}M?dP~dR(!z$-N z=H)4gY^9kj8=u$|zZ~-09w`Xr`sR(_tw|JvnTc8%k|ul4QtP$p$HFl$H}>ZLs3?o`pMwILnRI8n2Bq~G3DQ5 zjscSeGyU?P zbs)+et7cN6&V2V8M;;jlsLuqRSu%h11DrG1d)U6ISQHHCBFS2BP9I+wxh49|TIT4Z z_2BB2)g8y=q+y3?peS9PTyx^DcxVNkX@NIeIpP+XI2l$tkIQvVI0DzMp7R`DFNHq! znm_OTT)hjF3MS8!Y%DG+n?Qz0W9f@wGd)QijR}Zw?Ucqjf}PQR?Q3mUziIr7Dt^+b z%11BwSBJ?86&XK0&*u-WS{g5HH%^0N_!OrDc7~ZVea66&MfPQ9cUtwE!{H|Z7RL>AZqmGoOkK+*B!dmVfr-Dp?kwYG|1rEGvAGm`@;>=wTZMsU?Ns}2(XVEAjoO>rD5qFl zU$WWY(-spFRX1XbwP}>^6LktiNZ8T&_&0{`C%0HmuZ~WNi~wsr9J%6JR5_nNlXS;~ zJ~|CtRJ10Aq$`kZy}?X@qHE&_0{|X%EQQJ#!@qmHBvn;H5d%9=l4&b(daI~J*e&pJA5N*P(V$JCugKpCC)KfD)v zR)8?aUJ(Za;p*d1BNdN9%r0baU3}8kZT6$k$oD{@HZFcKEiBHl(BFFII4z;EuMHAJ zG2l=be8iNAtu!sTs~Yb6=3!MJeu-&ntqfHxxz2%jacKOt?f2LK{n)|dR*NRocJaSJ zX}>h=#i(Lzma04jjCLIDZ3+V3k2=kcuFGr0xF#*&U7~ERy&&8pqNri*e!gZ!;jWSn zjSpK|Y|TuHSGjBttYmazKs9O^lG5HXBP1ddd~W=^xXq zrm~u@(d0b%;jEyvURw#rX-`VF>R_54K;c&4^xwg4K4NtKWD!540Rg3W?OCi@nlus1 zYk5Fp=jM!%f&Jd|NqEM@@5=7G?FN&QRNkxh_F$CyZ5qH49d))g%|WEGifbAiEj;br zv&84K?|4)PEy%XA7*p)49T^yYp0i=25cglM4%<+3NYk4o1=?0THNyNJm_%WxOC>-L8R1>PBze@rg5ZPdAm_ zvGi1t{^zHG#I-bG0aoes(w{^vhxA{g176es@S2qMLh4BvajYA_GG=`%MYb>TcoQ$Y0pIdHeds%F4Yt(5czvwo=#}mj4uq`^0TwC z*^3i2RK&kULp<*rxEM;{K-34g(yZbWr+0t1eR+4I=JQ#QQ`MRVkEY8PgQxymw=y*k z3tkm&c3xP+PS3LVUko$e} z&Y8yceu|@3S~J(+aodoLcW0~+3jJ4=5b4!WlrbbWYz?qt-QQY!dC&F3%lstx}CKyQ-Bhy77ULFa>$8>(lMqQ(ty zhy$1|0_*mK;L{oMb(9DR7O?fUNKOgW$!zSdTLX!*_2gd$={=%(G8ON%o7SmlJk(t1T7Gpg4=to-V4M)D@pM_soY?TvI`gaZt6Aaoj=k<47zMqq1 zatztI_sGI`qTV+CKHxdN5&8X-eIOeXOW>C$8GedllfxSHVueuWqqsnlPe56;I)K5- z?V^$&8z@y=0ab&xxQw0}GTtHvg9*Iw6&>yt?cvaq!2JAqtyB{y4#o}xbQTNhw<(|5 zT7MIDI-{`daMv^m^&uP`9WQBMFhBXEP{u|XZjn7!{Zt3D*4ozi=BkT*x(RjOVV1KT z5J^ui0K+_%@6CR1mY2$Pxrkly*Id3x;hNr7$gJ@N!=nnC7G>FP8%3%I{3Br}Wj|)z zw~kaBUn()ns+B?NqKy(`AWhP|vx4&;euTa9=(mQwO$pqf{4AAma<(`R^0gXhH~2-? zJUwG>=r~jX-edptOLKXm)?T>kKZuf4 zzlPs@`}Hy{`QHSrxIujfDN@Y0+%IqC5Iu5;wO zDiCgb_zUP|kk2(35=bL2(svf|g29G~RimHHI|lw{>P+^}vrx0ICtk!daFVbGyy*ps4RyGTLx$0GoAaY%tb8OjaF*G3W+C@ZMQ2RnR(|%V9Y;OZE)q$=m$q-g2JE@2iu}J0agEUedH0x}a#z z)F2tVaStE|7Z(oT&r>2RIJgP8KohZaxG3&5Fx?dVsjpW2qbKvtFTDz76?)&oPn=$# zEKW~GRCg@#bN^OY!842D%KX4w5vA}fwLx{Ez2($^{_e>{lWBiPBR zNb>Yk<)`Pc^pY4bt zFUspm#Y&kJeEQoBkq-QctP0<*Dgn}-d5=a{3Y~d7sdUo_Ddso>$%xU|_hFYLFi3}y0<(wE)KuRD6q zZsa;mTebli*J(_@Mzhg$H-hLwW|v2W5z7LvUt&Pwn@`{;IaICPTR;Xl8_hWIx5ch`rYPev-4+(lxwQ|o-jQH$cD)4Y|}*vfK(94d@jNV$QtUl zElsSzhCLPfXf+_VyG8ouJ)+kO9yQR&`&8Q?j0F{veIC@>{uYLP-0e0j#s7Aj9oZcQ zfPePS$d=gF0_(E!_A0KT(NstObJhtr*oYKR(akO|<`T8G%HBD76~SkKU>5T-H&w-z;x(xgB&xYcUw~!~u zh?RlkPy<%u?^+4*mmoiW{Z^^qc-*H~T)%sjJtfj4mj!697ycS4(gGI!mOa{S)WodX z$O=H??$vBW_Tt;-`}05>=aR13x7?LC<34!#543)Qp#{2$hq7R6x^{L4dLV$U!q{kUmsFiT?3UiZd618~sr9Y8=Z6^el6xd2dy4X@% zbIs?0&3Fv}7W-}VDJCIQbnihb3zx+Hv$M79qeU?)RnrcuZ~7E!N5ofjB_1<+#27T^ zZGAY&298?XiZk^3s+)yufUOXIhB;BDGPO0@HMq6U&W1m6pY`F~&DFb3&@sPP9*j3D zr*#46NwXg6cXL$>@URPjbe2}q+`5a`MJ&(`EDSUVl>@bye80WChD>arEvb;`fPl%i zUSE$<47{351|$mjusl_E-97(dsc`@bODF5rs)5EJCLnEK1LZJ3X&8o%{{>Uc&|0>E zpE2o&!Hy<8bWwyLmG$IGXQUXe0Pzxq&f%e(m+H{f%hOH3*|LZ+lc*{DLU$XHX}?aY zq9LN&Ck3yHRMudEh`z`9fpNFCiVmOx_U^uey`S(8XgnR4GR4>!}*Ys_{KumQ4C zO$m*bd8nOn4r#C&cdvO-L(JbqshN}wFi4n?~hbhG2l^1pMuFXm#KS?$+C z+`QQTQWi{CA9QRqan>ohN+C8xyrDi5(;%VG_ciW&E&jG6Fxk}AJxRpzPh)~%}Z zHDN<0v7J%-3LU4~3MspdW__yjLq!T7`Ii{L*wjLITf_7~1p&B6)$ZsRbKdabo2ULi zSsB=k%Yg(#rhCxuQfyheStSgX17(BzsV6KfyUHE|)G0?L**cd+3RI=y??+b474w*( z3#uVW99%d5ho;4WC(XcFfiQ9!-vS^Oi?APQ@(ZWJ^*5-e>>*RPuUsP6+e%F8fiZBV zOV|h<0UcrsfuiKRk(?E05jTpV&R=hKO3b=+Cc8%&>@fYm6~C%3{dRC8YnBb0gjfcWQw1!@j_BG5j86K3&0V-j-CSg~&(aXK)3}#{KE{&$q~-PJ7S2n}BY1 zozk3(=09piK|>fD8vt7WiTIUn920Q;&EUuW93#Bm2pSL3A0P6YgeZ=gIg~D9CpRjv z#-@3*)%|#>NR0|yU;8$@#y5^PmAbOcZ6=+r%sI=Y%>)$th6)C++y$o2&YYY`4N@)9 zmha^L2oKadW$$#y(a5gX5%YyoDsJFVuxi_~y#eRo1NRg&rg2NT+<6k#8oa~KRQa08 z_*Gc?5Ir$8=;Bk=FHZU5KJXUb=Kcb2wtx|3-`!iuo(3Z>N9T8CZ5>W<3V(Jt(%gUA z8{A3>anY+E5w19%A0MXH+B2LRUs2}<4j8-uv+XL^PwcA7*B+9G0~Qrs-TEstC}ko6 zlquH;Zw2x{|NP(Y+~NRE&B~YtQ2m)T`R_;nkH4t^WvWqD-4{Op!~6d0tH2090DK0Z z>Fs~>;Q#gWe@g;7sDKlCKjJcI|3AL)_p4C@$lkl%`P{Al-8W~+fgmCEIaA($Pqg?i z@}Qq#h3p8p<$v^4^naw`f4u17V_@Rg|0O4y@fY~XgMVfL{0~$`ed+&__Wzuug6~7T zC$I;p*)dO0iC+yBx+1%xlLNUQBPsYm(;73b7w?DY%Kx6gaygd)Xe*oU`T@mx{i@T| zJNvneeX$ar$2UW)BUCB$HuI-$C##$%D7{r^eB z9axBop`z+)=8mOXHk~xtnN}MmI!AWCspO$juz6cxLKIMks=zjwqCK(?c-a`EdHDG8 z-bTaKNhXoD+f!U>hd>+KDlb^pg|^z#Wu;*Ai=r``v!Q@b>byq!|0 z%Xhez4nL-_Kpt?Iy$w+=kQh7L`&}?I?J?`2$YH8=*3>JDnk7mPB)2t7u=c=ogpfWz zp>2k}55K2>{nXg70r}r{vqBceZnJ;-n$93t(0o?%KKi57?tj?US1PmW?lBT`ug?;T zlzXPx=>4|)jTw89lc8_ol85?cf2rDYfH@Kkoi(i=Am``(2e976UwUwd=tjV~`7bGo zg5~4wvTY-i{G|y4loJFQCVO&vu?EMhw9@uJJTZ2K*V)!nCpl$&PPj>Y`(qH#y|C`E zkKSBd`r$G$2?OtIlAyU?R3g5nN12xwE%+Yh8 zeBzt>4^ST5gvdT;Sl~WUCD2tt@P64#iht{cI{J!%U)$tRTIm~S|MS_2?*)$jnP*Ep z5z;UO-2u5xAAgXe#oYzw7WRFz{QSOiUV6|-r2faLDZ5}IOZgZ7`*=_!M~IhXTCdk1 zEo|cNJQKlP{Cql2q7N4?pIWo-%wbSEy!ywXSqcSDlmulR z2|zxwq9*Ut8O4?ucD7Hu88aV|9Hq89ixW{Zd^-)DZ3c1aGdkkS` zUk;^n+(SCG3h&cge@N-eOg1Lh>9t23x6s}KCgqd+6IRek-XHIkojleNU%GIN3^@Nf zJkqAKiAvL=QFk=iS4>VLN8p;ms76{k2hjS#fwjl4HDO_l2E-hX?J6AWv(iJs4*ZY$unw2 z1Rc(ojkGgP5jTwfyxH*lhdyomltU^eH$dDgpkT^}zUxs!_4m2_O=}KkCfD=sN33|yP@BY{bScG$llmEW{otI~HFMHZl=pdH9tSP2!4xiv0;wAPP>1?S!WC9n_8Bf}DX0%PfuEy8 zT>d?-IhFznU_v7%-1O~j;FTCyAQqQ0c>9xaCtF7BnpC);jE;glN}2s`WbBN3YVb&B z024v|xi7R&UYHO_=%#S9Ky50mzM^b+2zb_b==iv+;-A)c^R7SsJ3QqD$EK06 zb2F4$li=vxvOja_{la`3Nf1ga!T4{-t+%*Rr%U4zDkry_$4W*~D0s{SUUY29sv`tB z;!fE?9fi$I`xQSUIycwz#WYBf=miJ}qf4xDc_$*U5L@uq!6%v{MHH^0K9-DTTi`)Y zt&=}0&%f}t{OPiI${WYuA7Hk z)0*#E&{KgTwL}y!g&4G6j|j+x$Ey%HRlVX^PC?tD)*Aru_?t75VV#oI+PKBBvih#S z9c5b}J$aFi!ZTtHhNDRL zsov=Ahc&mddKlKnC{#Ruj94?|f=zWM5+i3Cd65BZOeV`GUV0Nm-LwZbnQ>jjZ@3Zc z`&kwB(6F@DWB{j+71WFL;NpBHVg(x!oHmk-e?Wj(3epf7pD6Z&uC~&DI-ny~JcIL< zpVlp4Jz+&)%l+c=j-!|f^PMLijOACgfQe);7yur+Oul$eXV$n@SMmAXsn}5SI19?d zm)2}Od*%@r%9Kpo7m>iS7R>PH@Bv_Ml3R<6=T}}0#M%9FMF>1DTzjpH=D;EDZCf1m zmXe3rI*0Zi8-_WYg6h)NnYM6#*aim2O(4TWxQlyU7Iecn?{^+U4G4F5F6 zxMX?p&5+&Cl4`~Ct^Pb76ilIZ{sh<0)#_!Ud_DZSVQ93$h!LM_23o-}u;|%mbwrd{ zE1W&Y(Dry-pyg-D+INZlpBPOyFUOTl}LC(ttuZeR{Xu zU&|jOc(Bud6Ed2Ef2oH3_9(UY_e%6hR2v85t=T>L_W>c)5yTBeag` ze@}w6GH&+)I(E3FgarkHJSTC!H~mNxaT*?r8(ORToh~edtQ3Bx)U&-qH8W?Pcxvzc ziO0oLG#w7Ty*=e#x+&__YjXv+DZ$&cWND2a9{lhTN)z52K6>Ia9N$dk9-Ii++#xD> z1D-*eL~8yUk{NYDtF@_npB8(+(Vjnf5`g^ z19n>XwM1FhOj%loZFob@zJSn=yC{M_#J5t0dnL-b79twm3)x7KJZQl_?D=*DHAKYtL?44Mv8+yeik2IoB@aEQ}e$UhHi?;YaNu02iJxBc}nuR8m06vU# z%~T$)47bG!HETQ}=`z23TZ)k2(B^enT}9VP-5rp0449@^iee#Cp(F{7MWN&q|?a6_C)mY}U8nS~MRHt^<; zeD)^BDa*RiwS*{EfU&3UI5<6sq+IWdOh)7vUHaollG1I1_zl05`X;!q+UOK(?(@GD zrKCXFk0)%KJ`FryxUIddUzHmR93?Y2SPqj%1p}U28Q{*)3_YUOGgXse7A4RWmYir`6)Ns7{}sQj_+ zY+4&_d2ixS;5UH*y^xUNNs6K4HJI>C(jli;Py|Q1C3o<=JiH#FQZwV=R?*d~~P%25!5cOSG z*&h>3EV=RW(-M3&Ieiq~BmF;T>nhi2p zcN$w+bIb(B;uCXY7&r(tpH(>PDkWTGmN!`^^~+|#lPCchU4jB+IeWh#jC-mz3pq`g zurn=CygLHO_MH)?`A0~fHwe6o^{$Y-JKtj>Clt{njqeoIe}`s@ndvU!;N8rZk(Fd$;_M1hS9)!((@&igtfW$bLOazb8!uf$4e` zNpMUEd(J^(2+bVB%iOi?Q;F_sT8jIrim2tHIAR0Q;D1^ItJg1X%cf2fU~ro%P(g_v z@q-21c$xcuQQdt52^a!|zHWCJf2>#`2soege4U?#@5O>p)7&Mk6PTUcS*eY31@X{(Oby@8{I_>;^nDUIfcTA?%k$OJ~r z7axaDhk-6K)G`flry3Ctd5xnF>Wx#=`E85glk_-*=u}U?L2FNQoUk)I@nB83-KhTO ztqPkb)$TH1kzT=T@ai(vk8o*V_L?UoQDM^W%a^4YPe7;LrhGh^kdsS^Z6S(lUuM;Q zlnm2_&{SA1G>erKLQ(z_@t?gZo|Wm&nlI3djUYAo30m zF#d~yxi~Sa?(-L4GZe7V?E^tW`#HlEumqM-2f{&ndUZeV^sSIRO=jT90Id;nN_g0d z)+ad*oT3Z*Rcxk#=Zu=gyn8w?QDz(Y-VlRX>gwMfD&FfRf_#p>dKF^Bds&;Pw+2(x zK96ZWK!|y4z%UfLEgWNdBfeed((X9)riQk z`t1@_zMuFm~fSP#07uK$`)?d2k&^Nlr&a4d8-#+ z!w$)L2pG?(d5g75)Ds~*S}QMxw3r!WfSnal|9u-v6n~9Qd_fSwKo01v930n zeqdh6&Zsl@??j7=Jm@#$@JyM zL>n^;jwr>E?(7cnVk#r&4f%Phz~g*1G@<~{P{W(5L_E~Y`%!32BjNEq<%g?h-eg=G zJwWzHH)^}TxE>-z1gHeRhave;$=*uc=}S0F8f0yRm-*?FPUYD99k(PNWDZfWhD zJODIUFzRgcRbS5X;7Ti-^-V<`5mq>5jus6gPg~_m%YV#Ewun^m^jgq zrvGE>B=64Jz^5%t!|Z?J+raIl(ZQnIy;n%w&{mVnl-oLkJH1U@K;LXKrM&1S~rq7ARRd7kV~diBGNO`g^L+5xGB zk?#8I{)U=JETa1kf;##sa7_2W-Yp6lq2%p8dB%>%hpVOx0PWK1%tcoBpYZ2;I#~44 zP=uo?{UF++o`ed695Jut<#?QmnGmu1H@(rT6dhP;(qvI16rj|ce~?*( z0v9ly0mx||0hl#)cK0UuNwtA$-ts>ygrz=f2*_H^wQO>#3Mbw5==W$RB`Ams1oiSF zym*!BZ&-TrT!`lpGK=wDOo{tS#%tU|)=#k2B14&r<4zS5lspk&O*zkk zeF?rSm=G08-xrHvxP+6SRx@mck*qrJlV^5!6Wj|)(Kyt~0vRiwd^$R3`*q5r`(0uv zQ4z@yF|u^nP0;Tx!u9zANS?Vgg#zpXl(U0-bri?39HqQZm}dc2mH%9}QJ1}KKUHya z&o=~b28~@o;7#oHN%tZlquy|AxD~Y2kH5m{F#}pTPs!vaUlt?rmj4UV+~wRwMo66|!S{A8t@nHc|1?&tI%|cJf_{xzAgb@)E)qTQ@HGE=+Bfaau|o&CG)ly{ zN#N}G2Tl|697aAx;!(<7!KidF2G%5rSByX5oj&q!nrXs~ZK6VTMoLo1sQ2INX2v~U zz$MhqqkVdQz-@+>%+pKc1kgm|y8)=#f(4Q9`jpv0Gt=eXu7J!*TrhKp$V0#WD!G%y zQ)*nNh<+#_#sLIY2-m|SS!eZ%vz*XJaws1f@}1zjm~wQfIE8fU9f(wRN}ccGRsIV0 z@l_pkco-BU0a`4A*)f-6b64E3Y42gc(RuR}<;9?^?h7D)2@~_uciMh$na+*Z;KziH zs38FajqdOrW@u3u)c?tax=r8Ir-WB>fO-V=8j52AtlR7E?#Xg75N*V= zTtJaDM%HQRiNpk-x}9!pNhSh z8Dt&`x#h}M7(z^*U_G@Q)`u!-@Byit>O}kwAfj_Q>5Z@{Ng~lq=L=|Ud_IAJ+BfoH zy9#E*>vg~5RYzYyvvI3L{}^lxtW!#_0H1DA50|P0gp0w9rCN8;C3*eG9|kxKraYO! zQ#22JHty#KDr#KbX>jH?#~Y8(q@}wDa@sKDKzfE8ctD<#SQGh%$xW>WU&zLq z_@q+&j)hD|1T7Cs>G-9Y1I(}q!Tm0Z5wY17W^*r>HFw3RKX*Y^GJGnH5$j zyTU(qnfwCN$$?6ZtGbGM`(6XH3+?K>ofM{rj8XdyD*~L~Xh#FKQALcJJD&x`*6LiB z%Ty1_Z_XjFi2<_5wvab#Yp{@L!QndJv8%kA@imy6eve3zMjyp5WBjS|vxl0z+l7U^ zV@|we6ByN5;0=UwFEJ!KW2jfnQ5YRQ+tn| zh}XADX_5p=_3I4(0>T;TL%S!Vwb3ma_eKX7_ES496R#+7IHrG~DW5D7*P=Mp%2y+X zb*a^K_$k1ABID7P`;>@tvDF^(;5;OFhGhbqeCFW3GF8c!C%Y7T;JO+7+INP6pOo-{b#)b0VGdtrjIq?s?iuqt3)>~@RX!K9 zZg$lDOpjzT0k#)!jaqOi$tK-qrWAX3g=W{+*sgMH*PQ2;l)Q9k^ADlLO?e>x)P%dt zw0i=v7*Gly`&Xd5rjTKvngH`~(BKLjxM~A0kVE@kH}hJ(SJP|8pfwNb{TF3^S1&Ul zU$_H!2EB@#5+GXa#_swiK>3w~J88xkSdr;+U}DL=S)Et?ZOX8ByzWkxRAhjFp_tas z0NFqo+YVn>y&7i1)i#gHAS})C*&Qu ze}~q{+N0+>PWJa$AcVOJBoPro)InXLvStM->IYYpSPB*#X=OBC=8ndaBf*dEHORJT z!ijn9d`-ivTl}F@I!)XkR|8OhLr#}BfQetP6Jd=k+;1|ny1btz7qp(X7r)1KFPEj; z81U`~i2-eTMv}23z{#O3)<9Y7-yx$}lKa(9#kAtf{VvLHe~iV0Cm1zcC*B^}xI&EG z95BI7`LN3aoM zet{-45;&!11w>qmZ02;eZDH?VehL9c@W}G#*U^eSNpiC*wuh@|nltpN|WI<4x7rHa!zmWo@)OGN~}PGEk7b%w0f8T!`M~-l;OaubLmv zqp(PuQOB`>cU(=Y~3TE#aO?<|E z!*fW?`qHFGC4Du`L=v+MB5#Io&etE~&N9Nh9Z=e1I5n=XK{xL;n zQ6M0r7E-6EA8rvfzc>lR7Iy$uJNezMEqUK%^_{ZavUiYHIC|y8Y8YGxpJ1YmO4pz4 z-J`s;m_VH=@eA2P-LkcXyx=<}DnZNeOLa-W?lr1_Taj*0BW9-Eud$oGtczw90g2=G zd#_^B>r3mJnVNqWZM?Ww_TqQqMb=bspm?r+bgK+Y0_gTod;9tii2!gB zrZ|ibeKfRKHs+Yyn1W6a(jgt4#(+JQGU=H4Mt_zj88qDt0lGb@;i_8IXyt%u32uuA zI_n=ab~4%NP%iVKD6LaVHby4omAIc~s58zp9a;rI^$ZcZRJIp0xG60#SZ11v$_j zKm%3mX}4j47)710_I|bp12!e^|H2>eg-Ujpu@Xz=Y|e{_XQPwF%*hu0ZVlC0s6Zb# zCkS@c>})euulCMuJxfApxdj#Ao72?;KrB<^n6D279c<`3=df>I-*zwbNV2Z8_-72S zvOkC|$uqn{ZvS+LM&_Kb`a>QX951f~d}RQd{Tqx%_U6?x6ZR317+ zKncUF_N=-{;gi5;dP)J?+f6n-{Y7Yx7x=X{W|#IP1oP85Z6-ZoA%|iGZ<0+_z!Sc> z-^nToQdQv;?CVns?aj1Jjgrjm3@qD73;p#5E^Q~5=7%=C`SAkLlC@Ne?x`bwr$E9N7%UX=fpjo zbYZkHp~Iiz^URS?E&$}i*lf(|#~S;>w-9?4yM>bN<@VTbR+oVdu^nVGB-J$c{^sEb zO>R>0kwxC>t=%@&5!n|e(m!tSB7ND>SuI^kuU`O)Hqc#s^f*Bj{pjnwg(^h=nka{@ zGxvG{=`w^8-5x2w+5prBA_w#` z$|Dx0Kc4z=+JG-hvF5~|2*5e6hDzDvK1Xwop*#y|6Y1QY;&HwVfcPps*x*k&V}fi;YF$MW7SW&QRii5Ni8O(R@n^4#lNh&&^yoiM610}xzn z$cJ$5Dk|5Q<1@{k5TqzmVZ0<+&n&#Y$J!VYedJW`OR~$h(0E$!Cd|EX8vu}2RiAN# zB6;sSoK1tZfg$Ty`)l64Hfof-6Ffdi6T)4Q5n<9ORssHhLFRhe(t1Skm$QbkB-?hu zDi!db!31=lz^zTO@%y{3@;kpX8xu^-+My^0Z4m8U$c1JrrH-ZA1MRI1>!{j1QB)W-lakZ+f||++jq9k zv(%7fDFCG!0g4Yp{j?6wb^?Sn=uvDZrh+sY3hPgBo2~b2Jz0@@L;}tb0c5#h=l5C| zu9tVvZIiP5^i|nZXjt+h+kC!}ythd@3LBC8bJLVMW!{*bO>ghb_mVuRJWjKGmjOyG z9;4@{U)FVg$uTMXo6!U|q4aaJ@+|oSvqG(YO?U_oLarNf=}NV+&sy(=x6J`3 zN#r_7z^xTP9F&$XcT*dqMi<<~+2TgKfWcGHquPWb1{TJ__AsIWs0>nL@qhc0X(j!q z(sYy8H;gIf=r%fjpu?MkuaNzq%c=Xa%|@wl4)~y8FTfGIu4<@0NQ>7^o22DRYam|o z*Z3fnMf`(~eeahpJ(Qb|w!>D~Q>2mE%lc&ji60s}=Uo4qj<9kteIj23q0xu;^ZM~B zlwM(ybUaAoki(iU#{!D8=&YqqBQIx=VovS(8x_)(gVves47{TaQj)kw6qwNM7O0KY zlO>2`^^pRG+_OCe71k18hn$8q9>*1C!(JmpivgrXokS=i$RQKKCN}UD-gmKGNS__x zF*pzVl~@t@=X|z99?8ASYBbX+68hSLk045GDST;2h)0c5vBCmM!EB>%iu@Db1GWZVHNP%g*M4 zb``eN!-|8Tw&$_@U6M#J<&7V5M#hE`&2_Wzaf`lQaBL-pBMAgn0VVsNaP-i zygMFAX<~kigUuzA5{1uQt@oDw(EUt62xb79pAgdI=;{&{&}q~#QIl1CIe96G1dIrU zC_uN(G@sgx4i{((F<1cZSS4tiy0%~E%?k!P0&+stWS?SA=53xYa(^>C^o&uTxU@st zJHArYK`Wd9mjO))WO`GSBXP~#orv#pkI6wWB9m+TBOcFubd->NKOsmDbF-R;X+tsy z=R4=-L)3+vbk!h}s{%c&iVW?Pk+0F*CFaGbvD!|l{-ynjqSRu)ETbMpZd&YZ%S<~3 z!AD3ZeE`V33nU>Fim@O7p2{-y;Ih8$ytb%3$(gK#&JkUKK(VLJxZ~to{4zRqU&=Ss zzx3Dt!%!r+O@R08l%;PPFqf{}O;frwha{A8Ihrf+0pId(rL{1e-pa7sLW=k_;!fA( z4d(eKCz{vL#K&b{GY3&?Dx-8!WLacse~ekPNT?_nF8#;E6JVq@Sq^j~uR8(iR-vG} zXWsjcO*)pVA8C(c_(8$Q!a>xrkuM#{)nKt0Hst(bpat{kZBCT)PV9xj#cS2rSajX* zF)Q6AiQVqc>iq1Ff3CFX%ADHe;dfmH-4jQ*0`kt@867cgn2~{p9p5F^M!IX0<->*f z;3^}5`3cFP+N0p3GLjgj?1h&3K(w+D>(NC$<1#=X=>@6f z@TF)?myJOchoYB_u;2Vs5setdbh90Fd__<1P6@E z6LzgyXq70q{ZvYF4sO|=zy*xFdY&IrFqrD z0`Q)6JrvmAuHbdP)JnR31F)}`o2`%cQ`j*+s4w!Th9<_6_iGWme}l0$jqs3e6M9mr{dIbMYp zSL_BlTR^u7`gV&*k-6fR_AD$KgfeVcQgpm2g5j~J&GirY$=Zq?3myIl~{Dm07#sB>H7yGULV8Rj8OM`(iMT6 zvgI829iQ2#!0pA0_Hw6iziI9{~*i&7Dp4bm~WblkkY(EZ0@lHHbj6ax}0LAWc zJTC*`i7!_Q#y_i@qlKEDO-*b4WZ7u6HV|1Z224Eh6IHOGaGe2LvR&PjNlq4URaPr9 ztQgPvU>1O<)kUG1#t}9Fh*5QN%bx^hYeNv=#`t2aT3>y4!2TW`q+npZU{gzDABVP6 zlzDa6Q+Qu`rWx4)g;!PXtMpZirsD!7Ok~H*PdhFJ7$gV^Z5IW5Tn{nU4Z((6KSoSW8tzU?4x<}pAidZ;U%X@Fe#As7@VZTVaCRr+VkrW0VOE8c zy$k`HYfNtlnA=V*$m}W4_^R=hU&6pH$i$@0$2*bQ2bT?H7J|Ut*p(;#Jiqr>yE=@* zS1x-d5`%n0Q)%L7%z1aJc@29d>B0c1xLT%tFUA>gqDG!DynAlR;4w@pf#3Wg2Dn@U z(6l0~DaG+v|6AbPLj=z5HX{=V9i|Irq6w%nUMo+^$$p z@EKueOjP&=4*L)Y!RLnD>;wFU`ogrcniz_&L)>$pF5LF@^K??$H@5xywH*w4|3cBr z=Uezh?vJ;d?dm0A2abVv-pyr}&L_C7s7UJdl7Jk^SUw%P*PvRyv^7pUgfTfXnd%*2TOhEdH0|1-!$TK}3cAetEsEV#oSL-{qKfK)>ZeuS*VmE@U+Bje2IyU)b zBN2J$m97a+`7Y`d4`{mE8oiH7<(t`AJjpa;q&E+V3Wg`-#$H=eFuE%l*X`dO9}l>NG_q?V zi>Qc^mKd1}u}HM*D7D_yla5x=vl?i5yEibHKI}eTcQmA2=QUBcKTs^y{^W@)f3s%= z!Ej91=r<`>$$96a7~Ml4 zw^tZP-`{!T2aLoZoJ(IQ0j4SyExvVQ2G?wdtbo4w6%RFd z+>DB>^F1s!3>QL-oNpP)jmy=!Hwk+dCJpuQay}V*b=wSu( z*yP6E7Eeaq1~hKP?fbsLtEu)1Rh~kHw)bq7l9o9Y$H%LQc`FsF0~OuVQ0QynDH%9f zSRTx%)HbtLz~`QRiXjh-kqY@=_fRyn%qH~(pR8<)@VE+&Md2Sq$NF0JYi8-SR#`hI zWRH^LaHWHXv~K&u8m~Tz5p_>cQ@K;o-jNV<4EgPC=VMk{thn=BBptOD@_VVXgah0# zYfBj1<-@L6dAMh*>%t-%JZh_&M38? z*ahRcro^5G5lm$UXE66aP^vX@8cTb`83n>MmFxeKImBgrQ)kq;5_%6uG$tfPZh7FK zpW(Sut%X$Xg{rOZ=ax?jO<=(g^^jjdU7RmG25%?l#vQ8iNJ0M!vL8R%tr9gO$5(`^ zyDIzaQ}O}V#l?an(hFwnQrt1CzS*JUN~g6i;zzJ7cWvX0+S2UJNx~oJv$nCya8|8B zOTCS}y+A$jkxD0AqfW$3miPvh(1s_=M_bmAGc_zRM*iR2Wq z={+7N&8>J44Bkx&bx$B2#Yj)u5kMDJWQmtYu8#GT*Q1ftcV9r3ar95Q|BTanq2@n7 zUqt7($m(3fPW%kV)GSwF64K})uk0WW{_xizBc;9o>-cPB3D=A-om|`vVh2ImmVhDs zIyqxAKT=E3NcI%N!gTNJ8klF97QIye$jQCEsb}(!|5MuYJ=e_BY+9|qL?|NeowN1q z^f>iKQV>PxX9zrf+!DoC;QrK(=5#ms3neuZ$>DTZO6e#WhJ?feck`rVnAb}HkX-+v zwTcARW-Lcozo7&U$#G7CzcYMyWEN?L_25cbVC?T<#_VTq@79`Bp3q^XNa&0F_**vO z_oCXJO*sEjK2v&*X@T=t(y6E*EF)@B5pzICsyps~N;~x^A|!qncn#B%gZbnwu|W_* zsS>I#-lmzM^2s`(_RlB7`|8-+nw8d}Wq~fYOt>!jNxW787ut(}Y8M)MeB*jn4w|?t z`fTc>;HR9YFIj_@xbAV!hP3y!)^71&wsHc?4G_vw2pp{^9Gm(_ZnkeLrRErv!p<5A zBuvmD(m-~^4syUXE^aEeib9Mo8NnXJ+NAzR52Jybaq+Rf`98qGDEZ^13@u!Q0vhrf zKb8p-{=Eg*?sZx_{Bz9Ck*)R(OFMql7&c^X8ng*;8Fn^|(Pl$BDBH zz}oad*QAD}8lK4AX-o(Sq=W=bPk0e_YYkGD2v4-;DY%p}skFeNl`j8-SIJowK?pwG zO`zbWlC*nilL zpB31j=<29Q;5lvDvibm7t%9YDPpdaK{A#SAGCqm(pC&1Bi>w90gpvk!=QgObD~KN{ z0}FL4qscXZmc`buda<(8sl3MNxrx%~TGvV-tf1v#agZ?eFgA=wDA7ZfeMbpd^RW4} z9s+!qxxyOGQg|Sqb|Ufi9fUkQ@T}&}Ar)_Do4D_bUuEXUildpI+~{!2Y7emFNoQHb z+RAcTYfQ|=(@XU@D0#cj&**V4H#qFnBt$)pLN_En-0v(7+RHp+(LGtaT7SmkN$BBG zZPOtjElO)pF@OJIYhN3bqm8RqrSD^Qpw#Q;;8dczxEDXzfz50Zc5oes>$g;KpXu%2XG0XWV)1YyXG!^FDhoQ(K5d!fauap zi7dV?w?Dh2FEs4EFF^Nk&TArE4#y;WXpkymcNtkKudgniW&L6H`Vlr~+k#!z z*=~~bm9-_LCaLk%qKGRHspa~`t-H&S9+MP4=lEE#X7Ypfrho>Iarh2vm#42ZQ<7e@ z5m@LU$zU)3BJYU+6Pb%LjQvnd)Q?r~w_}2=`(iIoVW{yuufzWM$j^e^Lo4IhK3oDv z9jU1xkT`m-?p@++g230aAhD1$7)q>6J9I zn}O3vlVji4!-F&!scY%9{ZimS@bOFBvs2z&M{K{}Z#Pu*R%E3?6)XQcf<1n7M%*HX zq|1RdLs4r~#3dl;x#r5!_J%ZmMZw>+CGxDpt<#PnBrB;sLH(--Vlo8TXIVStapJa4}N?VrO|pY|w(y*ANf0 zi|NGRblDROZ6^|6`U`*y*_0Zf3j?Hg9_pm+K-JqyS}iQjqkL8Mu+JluY9Own3A#Iz z8=HES<4Z42x8EL6^_(%WVUS*-xKf4nCq1ta70w>DrsA$lP!JaXLlNhTQ;Ej!hi%U1 zKEFH6jbW50Yw`Eu)Xt>4h<7g7oRD*}2N@{BLpq=cTg&o8J3kh&Iu4Qobgw=w!!S*9 zP*tF>S!K{!dsvp-(%tU#yE`>5Oo_PYcLAwnd-E4s=!3Hd#ZW6Y_C*#EGDwe0a9qF* z(GzKK+d^`fE=x~v7tL%2#Bl&RG`_Rl@VJ0L$S5~=ZT2QcFerVj!g;KZv^};nl>=AC z+V7r5`cMrenAmWF5mOG`dq!PG>||c^JlEvb5=*BRFtV8|TmWs8*Qfy{zgc+AWQ4j0 zIIPdvus5-%mh_Mwd}U5ji)LmG(&EK*bb|lcpngjmZ&(a+3O0d{{GRE;ux~zdDND=x z-tbIg8JV%_X%~5#5^;ROEM4icFVj<>*OXXENh6z-bHv`h6U#$0KRFHk{|f5}$A59M z-84CKJL38&II$tUj4WiTz+xXd5Enf_Pyk}{7jg6aA-(-_m41FW{q6bTH6tjleHp1h zS~(yp*7iQlBQ_YmKg4N9id7?R2%XK;l~PxXQ^LYV1{sKFHW&?S%&CC;{1=c4Q}?j4 zntSYwnLKL3#d2rwu)EmJ-F)@BV>z~q>vY4-0X-<9ywU5ootQhQ7rl!0CJ3cbNWP}2 zT%J)Oqc0rnn*<%RfH8$I>3-sdEN1xm6u?}fj=6_Du!8_@n>VWy?JW}K|9tr_!E;>7 z+lADg1tlzNJ5ljW9mlf+I5EFVve|KoS#Ld$p9$Xj`$GKk-%H?Yt$7cyFD>)U6hBtF ze&^RWJqFt~?se01a$r}C*o#ZYC{fcV+38_vH9sB?&SF6OoPiopG8+S-1o`Y*j?fKZZZr^NZKQyUn-hFFtjb(6n5G`mK^4qN%x%*-Xjvm}$ zr;cM1(ppx_GfKyotHbxrUto_i4D>hwF{o?kkat6Vo2JYA$~nbi4+%z25mzSFta268 z4(%HC;kGj&k4WPa4i{?Ab}Yi5RWahUj_xKWwOVR0OvvyjnMx-WaV!`rhofNl-`7gH z26}vb%2y@MowkK$9YZeElp|@|zP_f_IxQ4!>ne?Qv8T81e7th17tkvp;Z@Pp0gHO> z2Zdm=66`6+>t&a+pS^N?*G+QRY<}E^t=#7Ohm&U-0kytBz{%ef&M1US^T=RSbM+7mo*+U-jQL= zPuE(l&xif;6aRAqqZ}^*^N%BjcVI~#&9?v$N|*aU7fp)?wQ>#4JH1}X<@r|$Al^1; zsGS)|bnmk`zgvc|#x}v=LC)c?ge==AoyQUykc_g#7a1inGLfmOkNPw~xd1w$N6CAn zZQLM+gYHhUZn+HvIsZX3eb4H^h*DwWAmX%09gIoWPd4?RC>?tMq&2dRUo9#(E_-mj zpVtE{kXq5LZ1Y#@w?Lw6z#@=?*@eRqSKC}C>>GZ6=jMvdM?bOCY(!5vBG0hcWAhNb zbp1IF6+xP5%4JSyf1(28CAUq$2o)ZBR;@xALHCCsHA!5Xbq+1ABM#AkAk=GXxA+~R zIJN2Nk3(i*9nW??{sN2K_{KZBQm;d>YYEk5kJU_35#u;tXi>iF55OIYIdwxxisCU$ zP=}9yf86$0rMYv{F7D@A?1f{+OXVpJ=qu!=jeHbX>$yy|F7{l#CbRibxFZ&;K+?%t z9YCv5Az@JmEdZowP%hcINl`~u3Aj*e%OpaI{VO)|Y#_~v?}CMWsY3-BV2zw14Ux3( z#vjLOhF>Ly!TUnz~RqCmkdH9ye* z0$NnN4o4M`N1)!;Zn1J#59X46oI4kemB*JGN;MRv)*e9$Wc}Et$shnv70zNHgY0Ce z&MXBqWBU6~jv{!c_t*%z8DCDR3114%^HUN(MMzGywR=S2=aYWLCTrL|AS3UL^UoGo zs226+pKR;`jiE32sz&pSam3a}s3-l!Nc5R8P-QMB_oiahvy2i}G`y$6^X zEn!(;h2%6(ICnn|N3w_wG~Nf$#aCo8V+UO(kX7hqSPW_tFF?+t#}15f|7_Q+3_Zp< zb@A1#kz-5tAOsc35<n}yg zfKw`R1!8<~L(HO85AzPW?LOrALyHTIb%mcEdPCmy8-Xu>Kc` zAjw(I65zXHd-#DOM&-10FHhBzQ9G{cf3!*>pN!SYsR*Tue7^gj1CUHSnY~9Je=n~x zFNxa|IzrRK%C2v+$K@7id<;s zJrGd+6JTSheCWzm6oU{{f+sC2J0qZFwQF}O_>`hlb+JmV1K<(G1WJvb>bJp;MzC6c zINbV&Dlc?x$_`Srf0+Mh7tqB_%^$0t*HQg!_B!cls06ws^~)b!eM@UKu>Q8yT1=h) zHPs#d4<3Hjzph395eBqzEysFD0N-U>y|I zeWr4P^<_fq&G6cs6;El76o_Tz9@E^S5DaGII{i*!u2?w0uHEk<&e?)US=)8eHe~-< z=F`W26eok~Oi=f7PkQHoFK_3j=*kd4aZDw;gQDiT*u%UijTa|Y0Q4k4FzX!pozcwU zYM_~13AW9LJrS}`t{wi{LPCL~>0fr~Ass{4uY&xJ1oekPoS0Z#7qs8ShP7mYh75Pfp2@=@v#>N)sRy=*FJbrj~rS_~e(A|bh!cEoTY0~5{kzo4g))H=b=#kl%j zmE6#E4{s=ge09C@5If?SGk`3zgqryZGgJ zH^Wl=s-y`$X#P2gHo#7V%8yXIr~8JLu)`m zw5q10ikAnNzklF!d|sRTlz0aSKfgQY(j8TW2dM?w`HI0$Giwouiutba-piIn%G_9G zW$#f$+FZKy^XnMd4FCT6)6?Szrt%tc!jSqShojw_&*`y?KwKsQ67egk?q?v^z&;pB zd!g(8DWvw|@H-Xxd`v{x1PyT+a3Q-h%g$L#NUHvW#2lB-PHBMr z%(_~0Us*1csQi!@H#iXzPHK_h;pGTZtqJZ|!&jM=@sLND9YKmtMg5N(G+(i2$V*<~ z{7?HEX&8RH9cNT%@SGMevlKCjUWK^J7K!(kwH>NRanr^uE3RjPvC6bcel15dC2z}U z^OV-IlScYghIZ4=;+tK2Pj7<_u|bJioN#Y=w7pjLy*DUNpXkOHb)}4#Oai3ffKiN9 zoV!<`^M(sDLXeY`EUO_li=&RjXAP`)Qzcq+4(8uV64+rKSAw zWO?rKX1N$Qbc`~D@V-5v3rd`X)K9$QE)9P$;KwyjWy7dkW{L~0QatZ<33WV4t5Ldu z%Xidjg7Z((LLifDJ2xW_zahVD%vQvxZ;Bn`MgAm-nq-B%>};#uA^s z1Be#B6kgIGVfbuG3UAV@EhE*_E#&yLEjD8J^3ew@?Em^aiT%#=@^w$j1qQx34ZR3x z-WnAwNV3b2};{zS~41-xVdQ|!G4$m16ZTaw)!qxdN5BrN=9i6hNSSJ6U-+t&g4}ttyqWsAy8Sec$fLY^Cf&TaEVJlsTguFfm{FlNCp%wn5sEn z$fHh;14(J*dzq~Y>Pj(vcC5r+^Xt5C*)_*Bux$2|ov;IS$n3iyMPF~5IhT0Tv}zRq z@T0sHj$amYq7J+VX7R=DMcwwsF6E`H@z3S6O!o;?Rk4P=@DSlMIj}k!%zH8E6!2BE z%zEu6Z1woK=s`Xf_(!h^a=lEaKx*E^nOW+swn!FjdJbH9koc^MQXq+$_3#X`j11c% z@-z=2>1SF4Z`O}8#m1_Xg2?c&WYn)K>8M6W3H68Iv>LAo)?baj_@Fwe^*s@G)+&J) zlc2=7?E?ew!uOqckW+{|py5r_xMw>O4Erzj49-+SC8L)8N0XYSE)siTQu$$SY2Ou* zj#?IT9O>5aI>)9D1+X%lDppdjib*QioWws$d@_bzmZvB%@6H^*UZyo|;_{PqA%584 zod5Q`cxLN62h5&`hXDP2+;+@{vz5w?X!PIc1x2c|<1%r?AC$zBDK(7skH{(i17nbb zS*;lft|^R&Wx&N~0T)G$`(@!_9#t7;shXfsk_|tg_Q0JpiIue9(fjhQ!HbxW|NIdC z`I}rb`+l{;$1Idn?{u$VR~*zh^;c_dWz;JVzM(yNFNjJ(rM#)QH{X2*I)k!_EfA(7 z%|=LWC5APNR`AsG8%MA88tNO{rMx?;pew5`==^U2L41Y2b{NsJ_pX*BE6=R8ar)c z$;uJ{Or{yG45e%3;bYX{a?elYr1Rwrrp!4tilF9&+?vG)euk927Yka@ zJ6Z40&^KEZ`%QuEu!%mgIFo&yrjQ4e&RC17W5Wqo}SGwMP2mRV}yf_$!s!e^6+47MVwaH}%QLGv8 zDRaWi=bmHI3mv0)66aO1j3P@5pbSMi%H@INt)rAdbd#WE-59f+--lc8FS|~TP97gl z!WpGP*P~@uUmoinOsTeFO{wZ7|=g@AqG7 ze$g5uxB1jv9FZBcc}x8A<-0}b%{Cn`V#U7;*>fAsY*Iz?JFDYPi9#^j@5Ppry!%X( zf7IeSZIkZo>?8KmDx(k5WqngvWPPr*U6qj;c{yBr?<;Ak>$uF;6Df4#j7L=ny-@$) zQl3+}sf6O6MhZrk%D~;#rg@j9t`6zNx7my<^CRnDXtbS!*c=T=CqE(rkBWx6Zxv*X zqhRUHr_)=A0rc9c?NjYo!o3H{s6&UdMGrtd7IbVOoAVE;T5QFBK32Dpf&Z_Md}Tnb zk@QhxoVuF6C-I^WNUtrTe^*;?2Jx*`FBR>HBpuYlCl$7)*^d2MH}(i30Ma&Zzz>?> z9XHrEj>8lW?rs&-|1F5R527k?^W?xB#!W;mq0Jf0^7=)Neu_xSFm}srbv)L+5mf7@ zXy)7bIQ@GVIg8B2l}~Up8L9hzcir06G}8OG94MHjt&R@5kI}s!S9c(;bYmrhMal;R zY=00}SIWMbaxJyG58m3(6$x$)+K8~cqAj=aDB#>+@{$6a-rOs;YVln`$Dcoy0k zw*QLvzz2!gfu(;odqBE{Umqi5G&m%9W5@ur2{f60QcSZm}jMZUPH4&Cf8nv-{ zaFCqZq;8f@qAYw1|D_9q3d*N_W+%YAoeDl)UH&WI_1BE$UV^0q=x4*NJ@l8_TzIf8t!OFe4)Xu=k{fVZt-@m#?)cfw!h5jJ3 z6g}S7KDItur#S8j`bm>1x)&7UE@oMunV#4F$`%3B^fjTh;3LZta%O?2w9O z+b2+!&GN+m{%UC~T6~}Ld1uT0vtuojMk4t81}ef>|Az-*A*I#WWPfjFyRPxo=T3P; zI(KKCclx`|2e6PKq3Kbf23M0tGtN4j)_od0b)Vz*-@~J@%rht^F10pr<12kCxGO}asv)R( zQ<*}(f7@$UcFyu>*0Q7c_;0aP)V$RibhE8O6%cD*u1mP~AJmQekI4W)6Vqg-X5I%svJqd|FX zawVC0sNmDD?pKTc#Ly(ys{>NCzqIC>nBZm%1AnTA_1cTFH-3b%S-kK$YWR8RIv<1j zJL&y5XjZ_|fo^Oz96deoP+W7Kb?$)ja+iuNu zMs)>B5&S_Cv5FQki=fx9fBP2hL-Jia^zXO1xXc6VZ-3ZXxAy@W8Si@j3V8z~L^Utd z8JpdebRDxFi?kYi;>!MLe_qCjKI}SZ=HB%U9h3turAf$UOv_Cpy=GFP@Qg*ya{HIi zlie_AuwqTw`rX+ zdO{CS-4!alMnxrFv^&#F#-QuNSQp|VSD5IHOuIZ99L>Ob!DDfDPQv?2~&SY~cwFwuqjKWXwqd9m=cgRku#_MrGD^ywGQU|=ecpVdAaLp6$ zAu=yTiVUqiaF)vbDCiWHLdAx$8&HHN!Pe_m^^g)mzaG zR?rs@s|1ZzL0^5+1u1bKPU(M2%OeVlU;ANkL{4m3NMwA^!QtoE{fVRXFhyst21$Vx z6T9>QX}RO$pI3^e-8Vc6`$AAfG}7s-u4cb}1a0c7qs^Bwn&x{y`$xucVM`AxQi zd#%jjgb>RCDs&&ErR~cyC^haCw^ypTFYY0E*3AwLUBB<}2tfXkfA zwc%AO_Q!Ltaf-D5qG6&{StNLTIl^$7rjy(goN^4|?ipu#fiQgXQ$^p3Q!&TLIi>5Q zYvUw&01i+Y5zgd$92;a2&RwL14-6c`K&9lD_9u@+vZuR*MwhM^915A|y-vOR>%nOK zfVV=mjWInDYOOH!t`u8N*QigDW=6N?(Q!=h^8Ki;?yv1t)#P8cF*30d0> zDbAf;Sa|DEVInqaD$l`9a^^?C&u2U-@_{>7=#Iwb6$Tl+e%QLiL`4bRnzl6wG$9RY zeI`WS?F%KV8WVHVwZO>(E`q&d5C=21OGycgF-@#LX^Q-8>mCtf?$xUrK~cMX=z<#G=^Gpmx_tM> zkDBNj+v61P%}&eqV2iq)^5P4FvA6H=ixLm+1?MLvHobhn@@G+IUkZTx#8$@c%g}@7 z&@0;VzYR)r>X`Ry>i1AJFzU7M+d#G1SB=;5IAOa+|{3-Tg=GgO{J$ zc!E|>-#%gC%+cMQn4^W5$R%g*{W3|i2-(MD9@U)>mU=ujxM~h1a=GGigZ)xxYt~<; z_P*3S6DYUnTfaa4SXJTY5+HFMW*)6vH<;t<93-4fK`FVdIIi;7FNac__VZg~p4}3O z?jb-C7*_z;cKf_wnLjXSfDe+1pnHNg<1(+fTFw|Xo-6u;QP4m0eecbx`YQ1WT_q1O z)av%t*?6P83h%LVR?i=#dJC+47D(4fJBHj>{(SFEpJ$cd#eZ!8p7@~ue#uC5t786K zBO&GbHBc90qilV>3vr!<{o;sCM%BUziU5fM;y_G7MnX`l0V8MG>)UC=4#w(Gg^5Gn zEKx^z6yX-bJN59>cE&RB>*x%nzU01VOt+9e$vCZ#q7pV_wm3Zi*v{>npp0#R; zYFxRBb{UoQ?6}p5usU5MOrjY#knBT&1M3xp#v9=;*{;;Dq&19Y}E6 zbdT<6UB#TJ3UXdx+h97XO0!9j&Ai%0t-h@p1sRnE$#of75zI57!XyS0hhPV)Vb7}Lsg!6d&k4F6nFyfmX6sfb}jBC zmXHbFEG8wE6ud^8eL0acRLfcS{_7*>k2j(&H)S4aUeWE()XnZaSSB=3Qn{9+0k3KI z0nhyi?QVcgtagq0udN7(Z`IB3rlOMr*Tw=@=M{ovJy);J58Gwt=4vES@BRiMvCap- zAM+%b!04%b@BMrCPO-cBftk?ND}|d4wB4n zT@p#YsMK%=8p(3MA<3+T#1_=P{4y?!f^u^%F0>COG953S?~uM!d_ z%fDd?6>AS_ZOT6}JB{bj$@n)%vy|QaeCA&EhZMIv-{a$H%lXH}B*#kd2NynBBu-G0 z>#vaW@$DhS9My^w)mTMK;p3I)L$OsY{$iX9Jv`ApeD!kpga z%9BUMBz@1OhAK5kL#OtM!Fbp-M2 zZrAkmUd$28g;8Bnv;QuuPxXc-zbUp(x_e`9K2v+yoiA=^=uV254QUjURCH4eoA$Rn z?YzCMc>#;jG-bNJBt&u5X23Y$*=>yzg#yM!IRgd8G-0;}+g!$d&omcJ1Z`=xb#(Nf ztts+}2{th^V~%2!2=`eXbFdr>JZcjt?y*TSS9?g3J286Uwi$O;(pI8u5l#2xWvW== z7$nm*JgCyiM@~8G?K{1ZjStWjJDOinZ=v{<>oWKZ@Y-l*g6iu^BIgw&QAIjm%Lp^kKC_7(Xepp>oV`%*I?^3a`S9nwCvEiOy zwhmbOrG}WbuK7K3RFJxr-}obO1^cBpi5aT%x{FK&-&OOR61OqWTJF53*J z6L`8MuK2rOpY}rjif-%G=VWAL>89G`?p|^is8@h0kO1X^WRK+TTlRMYYaok!d z{jlF4&-^X)Xp4=D9u9vs-U>$X-p{}NB6LWiIX{&I8qq1;`?J1N>*CSLD252NlrOz2FNa`I`MNS1q(0W!vB#m~5wY$ow7-DWSt(Hm+5)7dc! zedYucZlX@#1v`CSy?T}6?MQc6FS|Uv4=6&ugkjqdL)VYEau2xE#8XlfRnPw6?1xs$ ze-pb;#d}`Yz?z!w{Oy4Hjgoy`A^>CN5YdT%Bq=q|Sj44QsqYvW&6ZhEIOxVB{__Bl zv2p%xoO}&`S3r5Xd(#EI$3sIjximsw>j>e)t|NHy8y=b?m~EC6w1`! zc4fXbqnJt3`GbgF*D$5eQXz_**SCUQR8(gx@xcQ~rwoiR@MXxsMstNm{onXtqM$|z z0RfFT#rcW>nhAsFNlAIbg7Wg_rCq(@m>Ss^iw4tAlT{#ta-)E8mX$jTd zKzSF3B6tvfput3He?P*{6vF};rHSyTX%HLV|2RR<5hAUrK51xMp$9HK3c$%jlpN`L zm-($rV&KF@=MuX?r~bFPSA3T5gq^t{>HN|aX16)P&BfK*ku1)_I7%EvO+zCwU{9?U zS=uENkr8_2|99#k#R7n#Y!FNxuY3!cX^91tm*O{HskxVzS5GTXre>OcMQ!c4(b#^K z_mWZd)B~~w@a?itsp5(E^d~p6UeYx)(-g~Q3)7hUFdP*XmE!2->+AGA`PHX^HH1i| z$4w7%_5&m zhy;}9ynAI)Ksx(Rq-QQQq!wBGUo;tyuk?H9;3Drmv@angW$b$|0OO`U;KRR$(^Y76 zc0VSgm!W-PTuLBf+>s#1air{Z3E%b-^QZ1fi11BSHc&%ZHQ(aYxCx?^xNPwL`2^J+t2d*<>+E&wbq}O#yqRD^tmqb z^G4l96VpsIWkp5E)yNefGUuA*tyYUh5Cz41Z0EbJKZxob3#&OBN>091Vf%0ogQviv zZrm$g0V%qCv8m28&QsoMwjD38VM;FIsD+AIt#k0YJ#O~}2Z`41ZsW+B*$zVyYZz-Dmjuhg}XqF1<<`lVs__}#`Dlcdzzck3H1$y&OMdR(1nn?={6=epV+5R%bqYbS5` zc4x?U`8d7Z!A$R~1V6tz39{pA9MPpx+s@a;b$`RH{X>bqDZ*3ftFtmZI#InxAHh)cv^_48VLFBtOz#XJ?xm z#mdK{Tt?q8+{HxWKPg)RK2_ba3_bMzIaUI)%g-!4la_`nyZn8Ib@J`r--qA&o_Vfr z`9kXxo4W7Rwst=5zh$QciVoz>E1WOb*x7xOw#pLV5Q?(Z^1%x8@oag_WxNdaEJ8t|KIBq1{|zvow6Uf)CG zqwA%tbnH`35P5ld1ATpEX401wA@4~^NOaHYI9%ks5D1gu2QXb0@Y}Jiv(s>M(IfQM zty|$?9%^c5qFCie?ylM0X!go6_w+2ihn_}fJNFb86>)#J8>@70Ydmx7#wTl8vBV3b zm48{*xs1CC&>#uG&&T(wuTO9G?fdrzpE$O5zx#Bg%O(vW$Tln?^a`7J-r;Kzi3f8J zA{Ku7uadMEN}Or?N5{8G);(LeVid0zvaF;;i72j|tvJChErC>H zgg)WobnIsqm}7glIl{=8{Bzj+haQ&>$*HYRQ6gU~@;lVL+Sup|01l*>R+Cm6XK&u)nxy-EOk5$ z^@C!cg`A{q7eproNY4SGr(Fab`u30O_bsV!YR5CdO;Ddd`H|tDhEbxD$cnDp?_3;T z@L!c{-AhaDBiH?JjH1YX=@rrMW34|JT|O6mewf99g?F zgG3ku#hWuY;XL7RQ)>2Ox^P}CpnmB|@>yCi+yySYX%`XFG=_8~04lj(@9!4(4wC#m z4U1WkdPHdNClNuJ|1q>l%A49RZq)G?7vyGKx)GGktVVb`?FFu^d4E}jVCcCEVmT=J z+Zda@25i>zJx!V1ZO{TvDF@#$JeATkO;5Wo6BQ#kU#zz&llTgVz)phe2x^85UTw>U zC!^ujG7(*@-ypdE@T6Q=Tz!My)XOiK4vTq?jc8v&t zaN!Lu0H~Kq8vL9(ypcvg^>|u+kH24$(9;M^Z5ro2b2@8L2=W2xo~=ObWZMGja!f6i)27pwNn)Lo zvv}Kb?ePa!BKy;!u}@uwFx!5_1gytz<7pk_|C@=N2I3sbfrO;*S~&k~{d`~Qe~Kn* z4+$hC%?alG`bF=aS7g~a%u|uKz=GT_A}i}-e;&Jt*f{?xPGaE|1J0iWX(N;cJuFz0 zt#il|4715`Un@0`LANj7r8S5Tyhe5wm?U;!S*ljrck7H2^b{QWd9j0v)BvxaE;@Hb zpM&zk`P=CGH@P3nykfA~ef{6NN}OaR@M)SsOm9>ov0KMmY6 zINU0$Qx;QFbJMk16Zo(z;*MF7v2vs>fJ-!-jQ``lF#${XDczZeZhE7=K3%_VZ;zJK z% zdN)4BNvZ+5L))UvedEbvDn(0Wj9p-s!LumU(y5TXI{#HUVNK}aU#17GSaC2UE4iBF z8_3sA`v2<0q$aRgCpRV})vZ=q2KG=F14K`&>vUA~>Etdz0y0oEH=z{8j1N?DnXYH+$&k0ud1D2F8rWQnlk zqCp7y07BqUDLU20S&!Bh@DdF+#g7`EB4fU(2Bq}S^4p+lE= z?#QXMan6fIqQn&bX}u+73!hLgi=@+#+qmTas5T2E`R5*tdpOr`CvoE%pl2r<;rF0{ zBFw|PMzs!Rt^$7jgd14TQWVOc-;{PTRL-Xf{PPA9#hv$@xib0QX7NUi*kYb9m zZ|tJN=Bo+QPwisV2LH47bxJ@p87{@?KV=i+!2NkVNE7?>gbZQBnT6hLFXGg^+{k}U zX_d2~zE;*t4xVQl$?b!90Ot2H38>ecxorLSpk>iL0&8Ld zz*&@@ai6Xd9V9S<)T%4sUKn5JhxxJzxwy11C+3|86KEg_>`k*Ez_-7jlh z7$MJLmcMGS8(vO9PWt+z!)8zGz7_iS=o%3>IhNeYK?2Fha@5@wGA=8B%W?T9e)g$g zNWiN&jdulab89>I!|$r#mVaFw;*y?TCUE@R>$`XpNd3}M0zKx$=yB`yYn@0aH^?l0 z=qY-%wd&>LJDM;>`%h%X(4F&bm+rZ#KYBi8Wtku5{!iDg5HI*ou*wI-@6gdXj79zD zh^sHP`DN-x5atvnfy$H!xC{(2D9KpNmwyh1KURN}yzN zPJ~*9#0#|x;|ZDCH@xJ64XRbC{-GKZ$#RP<=ZcX0{@jlff?|!;Bmy|`u!7{0*|PJ^^+oUpp9Pt3$;$ry2*O&fcrrPv^M87k}9rY=!HKq zH2H?1Jt>5MM3YV4*6MCO_0G-@zvKVt)*8*!!It?agg~4_^56RTlZ;r##SfrOFaRX1 z3~DlEy!ppH+J)YEtMOhbT~(Xn`G)UoM}|wl5WBk{Ksxxv2LMsV8U4V%Pt7o&+Db&%Z3ql3 zl}D5P^#w5OvLw^^hD@n}%J7zpb9OEcCZTn&-^z%qW1@P#u!#RE`>&#)M^$OD2;0{@ zk&e%udjcu6LC@zQm++}5HSY;D&%xZnm&@_7u?#Ere+eH7_KmoXX!5)x_Z#YfZ@RiX zc`#T1Rc4ltyz=5QvYJ?GYDP%bo1&Nnz?%O#JhcML+(c_;hL(&2gZNt#Q?*Y4j+_cP z`aj$F=zc(6hSF?pQ$@sC45sAV|CJPPVb!>~ex^zr=HPL)tnE)d#}I0pb=DVe$-=U z{99Dm<>>z9owDokeW1<6(Ar*PeFlrqz0CU0fk%eR{MyR*teOa#pK`G7-T3(=kp%`iu235Pcg&Ypo?xjAF_ypf+$=qd z>&GQq@*!`)fe;7}KZ|6H%;bvf2Yblj=V#2BbjOS7IxinWYahmi`@PnNTfAvH0)HHhbu%qbcRIK#LG}}8@O4FKVJ-lKqTw9{{p-a+4exQrTzDq4H%1n{yJm(rEqWg;tF+`6@49#_WP?5~~fht>$X%0B+P{rbf{mp<7D z(Z*TQq<5Dxs$xJW&)Bcvd{bz!r+?17kdUQtSwyWma)7j zH)~)>tb%zKCN3`7+DFe`t!;MZSR6?e_)_$>eW=~(A z$H(KVwYWDOH|bOt6%|EBMha0=Q?t0=I09DzwtM^2N4V$6fvvVjpH*ZWPq2e)oFipX z{_gfG*vb8wyiBHuDv-SKh{nAB~%CRV!^6qKc^UmSBiJiU}}bn`VPwGYy*N0cP?H44w9avIA3Yxd^@4a!LjQ$1ptS zc4}!{hqX2`X~G{R89v`}U|o+*k(&1pHbOsyp-Dud1p#1uXBF-*a~{ z0BPqtTapl@Ff{Bk*8^17X!G3^n@qin8EceXY8!#+rNx-CmeUt4x8I839~H1@JL>9^ z(y3MY6FaS=;d3i3ecf!u9D#);q-Zq?^6jFObpfHv6Tg}GT8Q2Edu@~Pci1-6ss z$)cqMTv(=$znDZYs+R6*gw34*U#rOS;Jv0q1H!Rc%p+b-jpK%MUf6>&=#R3?aL{K$ zjTfsyMnEzZ%JO{JH$?lO<(@amX*14kI~ve`ebIC~@|*pSU3$x{l%!O)Op5D#c`ue9 z=~}#3{s;iHQy^AA^Ekti*&ft$yWEJe1wg*hU`Y$F>i{RiLF0+E^XZ5vb^mu+^JF!3 z)>}fl6PCI8d5!wJtDRqfOtw=X&(d^sp>fFQeg#|K(I?H;KUv8ppLFchFXqy(GWV{^jP5X!wNF<-W(;+C zc%>Q%jX7&3f?W>RY9`fX*PqLLp2wXoxQUP)TGtIGq)n`M8UnWDeK9{^V01eLUiCD2?f@!z7n55* zbiZ~F=zKCwbEI3HpB0ebDK|knm=Km^qoM}^>Y3oDkAGuN6!?u0kdAMb_V>t*L=rmP zpl*qW%k{Fl96a%#Z$dr6n0C}jVOggfLM4e9Xc=sd<#%wXL8p&C@j_ObZjV69?~7r` z0kjr!HTS^vjO-0i)B1BDmU88S!{DG5PrJxVwup({3zLHDe9CE%kM&=xbpxmh1QR1 zYY7XvX!=|}GC-hMj`zBf6V3l<`t>yG(JK)aAtz#Pd-~!?=DYP{D5ZqAooE~{JwFTi zWJ6hDq}WtaY+PM%ChsJw>s3C1W`zNCo}jn4(V~yIaZNfRcmput4OF#~>OT7cpdI8a zKvd;Nu2bl(?MVg2K*fc@S320q3EPk<0?f`y-di_Qf9(~4$xOom+*?Hmh4M<&^Iret z2_kQK0nL+s2H8efF-+h2m)z~<#g=SU@F`}A+jXO)$eDJU!F z{JaH94|zGc5zo8Y)Qk-8WS4zT=i?qqmz{+oZUXs|+SO;8?Q}NX-QB6HtGc>cYUNgI zBJ~^5>i%0<6vEl1JpUq- z3Dj;V=x_+|X?b`VM(XC(n^bewHo5AKOJv)D6r>8EWDtc}W04|SWq;4> ziqy{T-gDc46Y0A*ZCkgYy!s12kSg04UFKy?CeO(l$zzb}p?`!lvaOky`gdRwzCgZH z2u*dLy4`scwbeS@`hr|aCf1Q+-InJE@|+XnmwTtt{#xb^6$_x!?6 zEu+mbpd}DnS!Krq*r1LSOt+KCfZgvo)uz%L`71~7d5pDAb@L1+qxPHV@EVFi6y_-V zCu%NwW`YeTA66Q(*t-C5$4C!7SW|#&^M!<67}wTMCXIA+|t9yC5U!{syszh&l+{4eVg#vssOA@ z6-}0DX?=y&5sJr!xA8sU@y*@!fimI-pbOJ<%63wC+>TF90zxJLXH>lHb|VlKasULX z1hNNvRYa+t7>~X68*H|uMQC>O4&XOTJ^TnKDK6Tp;iMKoPcf!+!g+Huht z?p6J7B(KL9lsotqGuPN%>gPo$1iF$0+j*7YVO(6V24@nmS3(wk;PQaVm5@w!X;H$J zun@oYnnlY;&p>a#G?qgmEF^>FsAr2X8*fz;M!g(G45_AY;Invl-_HfbvOz;&EgMZl z^h2NgHj*Ja1|%R7EmspI3JIkS#qmcxHuC6XbQ$g_6CI+dZKknox*>+Ss@RQL!@^ow8$t8G; z*UI5DllHB!iByh&aAZY-`tacc$mq|hSHABxtRM7&9U~3M{jycGTi_F%Q$A0Ad)!TC zJEI>X=X>bu>*+}e`eb#UwY@7hmk*7<*GVrvED_Sz=4FJ$3#|tmdIs8p#oc9xiebKSZ`S)M;mHh}-GEWFZu2 zjmK%eE9yCh?%HJH0y$c8A68fhMhdS4zGK0uK!9&-by<&;!Av6wn{j-Sh<$vL)>myY?b#n-TpOeGi{7HP>ZB;vqCU?%T$`D% zCLWf_nm*&(-o0ixH{1o3CzP#@1wY_^Z^KSs%Sukic3!Ght%H0n->5&a-QM0l5%^TI z+M(pE#Sf@x<&SbBk(lka6$Xq|{dETt2n!wWF1Mme$BYMN$RbsH*MY@G7P0aA{NVX; zFw(|T6ynUIwIAnsujgjZisW#OW>j2uD>RS3K4wKdWP!fZIzAs#eZ=VJ6$+1Zz`EH5w-&EEtr*;3zmbkK zW#s|~my0%SYuR$Tz_wIjIcrOP-#B|q1!R|{Pe8v=R2CZBX$6xgs8{TN-g4|$6#ot| z-#<$oTi@ZJ#DDdIb^Kv$KDe63{TKbuX#CHr0oLnL_A)cctUx(>>W zFI#B*nEj1>;)$Jl0=l?p9>x|@8)IKWC&>$)$1uAq1EDCc9Hil^2jtR79V6+cqJ@mx zG+Xu?iAy-E+OSGuizahoiR?DjdzdKzxxgm(yuVVn{c!7~cGCGMv){1~_%{~$?+*%X zZtyUO&`{*{NgI!t1ox~!_Mn@nuD)Q!fuQoOZ{aMdif(3&H%e2O-oBX&t}`9p@D^lz z8Zf|*>IRb_+`X<8GM`-CfW|x{0+UjVsxzcE7?)46*T(H9NcI-&wZKn5LESI4yU;7b zuPttB`s!$<@Nmh9kn0>6?X1rx`D+*$7;-?ZkazalVP9jlv^lgJXya3j8a2W76#)L8 znVdv>Vr43nIF+4etlhbx7?28shwd0)pYjy1F>vBFkNWM=Tov18&;(x+FCV z2hS*IC4`J?u>iLKrD+$`l3_BZ7U}Tx=FT$=f1{$VCb%6N!bz>estF+*@FEAXd}Lm^ zX-cG^;wAlt-Wi@sC}+PF-aSV9$grGi@uHIaI#ppOI8p&WhF4?u2?P_~qy%5@atihJ zPf?xC=~vG%8v5rt&28u|4W^`9m=}+Wr`p>X6SUO@AGfK<$toA&n$;0LkbIV(J-;c} zG)CFSJm057#N-T{mSiU*31D8uh>~I3_Jv&?%GDD4A=5llmEkesx$_t!2ayV(;~Ww? zK&E&KoPQEcwjhc)DPQV{zkBzRjVl=>SqY9=qp@sGu0-Y8vb3VqR8L4IZAUzqs|n*! zHrLWhR_;1@>mSWdSuYO}fVz^HOi zU>i^VF3};of+!(D$$g-mrE$V>ljH}9)QK#vu;l(?PjAvyC8r%-Zo{xm6f@>MVL9-X z6>TsStNaCBiC_2623HM8`0G8&M~dQWEr*?LXu#Y;ux}myW_d)+Lee&vN_SUsl0u;-wfSBTfwL@KC-~ zanZnWKa|2Dt_$Q>X4mBPY85Hv%1V|?vK zX8A|Vu}|L0{UNB+o9X$^u}$RQ<>O7p55q)BU*G9PbcDwWwEO%hZrQceYVlyO8dQ%< zWVNd=#J76^Br3p3Cw$cCw(RxPKg4rTLRA%y2Y6;GZGXLAZBBMvg%8_$VrFP4EAYZb zY^&W*#_^21pn-kXrSGS*y87G?$3C0aYqMMx6jE*6WrFBxS_Z+21fr9R>N~mFIK4MP zP(b{IXA#$hj~4f+OZ~u|VYZ$tU3s@0bPaPk-e-g+*&u8AJHVOt4Wuz2v6sJ-#$!(gbk76!sDlRR9<`4o zOd-v%2A@JQ#Ly%Ph1Yq<(9imt&gO;HCVXFQ5tTbtDMl9H8i(PT94+g2XMKJT{-?W` zr$f)IJ$FgpkP@7wd-_N~4~qz)X+Kd!q`*j8=td9=K^rUgZ$XL_pb?mfo+!O4-`Z1D z))WT7*4v-Jt!&xzVs>&;MCR=+i=u+UFetwH8=>1V-YY0%`j{DHl)XkLCYA>Ne>sbb zPoMUQ-l?AV`MBpIBrO5zV(-?<{t-;_H`VcT zb1Z(>N~Vd;`;8|+fV6_|1^OJCPiuv6SQVx*x>6%;b zxSDj1{5oOE9yxZRsI#I)PECy7tqR05>P8w6Xe(QGr+X`Iw5h0*G@29$U952#g|ch! z1RXt)jaTT)r8d_z5>f3+5v2Ak6(kOYjOzyEC|N7mhl6H@Mrme#p-wO6HfQcnGFQl` z=}6K%n2PyZ^#l>+kVb5 z`}H>maIcFGS@zk){r`@QF|S8Y8_gS~BqT-?wIn2-h2Sy?R4m3l8=T1}H&9WhThQOI zva-sIdK!DMcF+^cr1w$tG%v!fkD|;Cr?H{i%0zGGgD`Ap~0!^4!PSc z%4>wCnGUM1Ec24+y!IUJx?1#J!s)Ic#piR%g`nNeFhr#F-{?YQ?|TlrK8EtLzQeWV z+#WAG;RyF&FZYnw*g?(3zE4`Rluh=f61U{lzY*u+Ny7&zoCb2xdnI&l)C&2Cv|*No|hLLL>xG}po&n&A=1x+n1cHp zAG(QB67zjDk7dEoXM1*n#Y z$z&l3kHx66w%Y_**W7=$5bd+mrrq*B%%o34$x^0JOrKv$9=}gH^DS^nd>hsF{9r0EM9HT9;J@Gu_FDxZ+mW=yi8q5?wTw8%7S zTxfK9O{Y@YnpW)8-Vd@m%bau~hg=0R5J;NC$P52mukEJFw=kOAb>7?8v>m&|LBHod zb6D|cNpvW$ZI))5EdO3zn--lR~_`P@*3|-8rr~H;gvt6zTd+ zU){Py*}Iv%vFUU&G+lXpiP5DtWGPYt*8hc$0gNW842i||iZ?mO3t1m!>8yaf_Kt9o zxFd02=c71v=h)-kjk-5Zf{Fqx_Nob&E=I5mHKsEHb2S#qiuyg@$Rnepv(3}NooKtwB>>wZMnk!fms1wS z+XiW9v|J8YjORP|O;d3KuZxsL=kW0GVs&UDFp`QTlr^ihC4VOrP?m{i&@P_p$a0R` zFPvKK4EhC@>iMuk7$e&a5I|dTbeuwJN?=049Uc-y;SmbsIa9Hy8=0(DX2OyFXj;d; zDAZaPKk*UxS56lkmN?hlu16&q^6b(gtsHl{#n$7;V11s2p>o*tvg92_Kh?Z!SCS;K zQ0L~(D=qeYww?dwdv*MAC#A)*?*sAO4R5&)&0GDPe%q}3*B6ma%J(7y-^#t>-pUJ2 z>xRRGB}FCfH_f8(0kwVLdD4`6XcVm!&g^H)60Y_>jroO$Zjg%iK}iJheT#! z*ro$zkiS4SA)s6=QJ|$6K0Xf*blC{$2X(ekMme6{jNnxNlZD-O^2!=99<{KL_2CQH zcvhur)$U1TNP{{_3VNxkNsJ0P63fN@nbT36J_X}YEGxqb#kT9SPoL<=P00!jvD0E4 zqOob~o;e_Rn*~jq{ii*)K+82YN@w*$rh4i}N0{;^GCKHtx$l0j&69o z?v;QD{6&g$HXr2WF_P>iRc!}Yr-n;EwEMFkm6=azdo)=O0V5LY0UbR(1sjJ=?I+e1 z%N_(A`phm7GeIa9g1(Q8^-bkt)n0~Y1djKIK0&7EpqQfu-7*9u>KW@#v1-HZ zY8!e540stWV2B~4l&(^bDqp(YcVKZjGli&LJagalud|+}z{f)u z&_kCGecsdi$%pE7kn6=Qm!={^{Oh6dC&ml#=|H;WN#jMb8Yau}BtM-$O%ZOKm3?d| z`6}XAK*~fq&+6#3i|CvqQLwZ!Ex9@dn>tzMX4%(6MD3#-@}Hx!ED6(y;N;7=Iw~Cb0>B#*x+S$`+s3 zm16<{r4&pS8#XCv647=&`m*U(vtUnf`@6)0NOp+RGz5FSR0TvkI-c7G!eD% znDCVI%S4{mznc8b7mTkcq?uMjrML|GrNTmTppG1(Mtn}@?^^EfUjO!&N>xLrxN~>k zO1`FMex*?W-wP_{R}!F#38H_MJrUE|ZqX!7fU6@czZx3cpWOAEE_)M)Vo2fiAiDa>qlm0XIBRpk&cL z2cjKq#e*U2I=?v?%CXxyb;#wOWgu{E0i>-K%8$GAtjtW=C`K_3(%X<4_C?2xRsLVy zO~sW)CME>Sm&Qv&I<{Nx9F+_%I1PFkq<-P<08;x2?97$QW#{uaCUe}i2$bIXbBl}8 zIZ5{TWjW;y>wwUkINh|L!u|Oa_7raZZ1k<6hgEpxjLE$ndlu!{@79>b<(1z2O^V)P znw;J3XIxagGkw?(xQh-E%m@DOEL=>g0qdy-|0>?Dy!gm|W;4VsxJK0H`&lYnXIUMGHq$IyWsT}r z#Ow$2)iMJtN8$a|XN?UwG?7SZ(#m6MGG;9AVp=Eb7_*t&pnXwNJ7`$gd2!E^ENhNM z;UMghtKU0M7g+ryw5p7wvR9&?E;Dumi)@M}eo&JS=fMteyJ^;^H2Eb6W8$5?P>PtH zRU`jcp+*#gB;wF~4+co4f{CLs#m!g7x`Jf|ZN&8!Ew^k|yvlS(owLTjR>dxLX3DpI z121yq&3A3L-%(_O#?|mxmxADo2;-Ww!l$(*z;c*{Cm-2oiK*KT)V)2-XE)F<*1yqX2O36)@uU9dQ62&-KNXVrwPwjK{rO8_l z(todi15i?YDg;AJhuH5&zI0ZC=Is!J(D!q@oFV+Z<^9f}OI~$F2piVM$mA-FG3KS| zKVh+sKFoJ$FP12zImH6Dvly%`Ic+6aB!Vp%vUE8aNx2N>SAO%63QG(}=E@njU zB{IUfr_WALhucx!7WzDL=;Y(o@5StFaM_=)!&ghkg>95f6*hHlUhVmjC;u&mn}jR= z&TY!AgZOgs>$4+bPwm-?vgnZN>DWD_ZQMr?!BW4V=Odah%&z%}>DUCWU}*_FiIwlk z0}0nXT?fCO_wj}ctipv8Qlo=Cyqp3H^Biq8MgB+OZ2%O3fx57Cspey+__59PArl1a zpV!B_y(LB2p~>b0)j*5&AS)?AT^7U8mqMyk%zUC^WywHfo}2OF!?09n;AB5SrSq2k zmBG0*_2;picUd-{`oBRmnza(1JV=l&IL6?9gb?WOp9+*1tECOVEXG$-zfI%3@&)H7 zgk0rd{|V4qK0+-{zV#4#Dk|CIhWrpLIz)7b#Tj-5hzw={uz?+Ri5|)ZscVEeImyoX z-#pGCCTr^v^R zmi7vbP1T{>S(AfArkC7yU_vS_r)9&JvVz=Pv7>4wf+3}@Ck%JVV8Y_-QxJ`4IU~&M z0PdiT0f&}GjX+DAmzu62AH{}-^f9;_jh9*2<0ROC@>`(uiuiT=iolV>0I~ClAJXZs zW}?cB%2sMp$K8Ad`IjF z4L(W1cQ^ZU9Z?+gdUVEDD>huYckw za3=S1n7*4tNA~Lf5P5QP0*Z;SeBu3{!NG|5?B z1db%~1jY%ze~%7?;SzB}KaNTgwoN;_?-Km#= zbk_*~f|FDAGWnH8?N?nry{8%U`;Vc+(5Islq|Te}@@<3T^vZwsm=woUTUD0Jn>G#> z=cn>)_!-Hl!eb$j(fwPGd122n=-iyoP7AN`S} z8J3*=9y|<%5D$Y%34_5@SgW8#vosKdA1KeBGn?1Kg`nY#)u1Z>t;?5x!1{{(7&VGh zt>h!zV4bkt8TAk|TIQGkpPn32{N!!K+@D}ZO-)E_UCw$eNc{>BA7xXEB77Ftlx`2N zN9QN7k4|9n&oj4-1g48%I2g~u#dDc*NTGj-PC_bk9b3AWaN5v+uXxaWYAsa$4IPLm zR8si9{?75o`wS)L1?PP;kmjv0o%LM0BrBV^OWoDzVJq;phVs?H72GTq7oFbio7raO zFBXg3o_?19>LQnfDDhW&-+Beh@onNU6JmduDO{d)kAR(WfQ zwuedbl{TKhOc4UZ{fP=C=(!jxmn{R_xrAP;>=ucKy-CTu-wY55#Pde1{Zi>>p4uVR z^pLjos>VduiG* zJDXQg6(T^|wvY*5|IiRXL7Sld#OB7&hzfJZ>iw#gof{G$Z_E9tL-zcHLNb=LbW zu^sK$QGD@%_VzgOxt!S7tAf1hVrg<^Is#J%1e7@?E2Mv)NQU+kZG?gY2KM=Z>5`BU zsWBFT^{1?`bS80X%oo88XU8d3Cc6xZLkvkC_>Ti*2O!e&dy3RMoN|xX-*=^?Q{V5l z-qL(}Th!M_d}#TD+mj4^EibQ8f~zljo9h`KfM7AvX&%=2CKO6EXE?&fpdX?oZ8FFO zB>qEaEoJlq5Q-Y36|-A~OU~vi0(JFY37XtL8{;Zm=Ee~$vY-2uo%wx(nWfb7srd2V zrtTL=eJ4~z!(E!tj4EoX!ZDyDxPOQkPt`_yz(A9^@c42xPsq=2?6P7@>;bBO3mg+W z&v$llD(1E2M6`+Li+p6cXyUAdToGB7mBvQMW{YMC67q5?f?xzclk=oK`xj08TG5V< zE32yc9ooo55S1HEOXXE?u<#$!xTj|pyav@MRd;aBo69h^!W?;ucbP?PDh+j1Ddob3 zANMHjB-h&3A9mv_k<_#%De<)*c{czK%*_)^-iprg7)g|;e)FOc^G@2%>!qq<=fO7j}B5x~%X$ zSLbl9m_)ajvOR7}1Z()zA^uZl!cCD#X5G^!mKJi?;Q zFMJc3=9z=KJ2*67?rPc!eM=FodGwz_CkS=9$vyEspN!)g2Eqa3oz6MiRf){K;QdTl z90x4vzYxXOdJm_;@~L<~GlA%WK(GemzYlnXVVFPez@_GTOo(fa)aWaV1ltISP!#%{{P5>GGtEf~OT{O8Td9FW14RnzvZQ82sJLR(Yp;#Opw2kO`2VWP0m}#8CRfeZ zdTy>Pn_wxKEdL7?ql9H}`LIpDqZ+eOJ-V9eEbjj-HQD5{WNd1H6hRYZ+t}EYmkfFH zS=7fr6e}TGclJ`i>hDn-CT%@Px#X#QM~dEYuw07K;*BBiH}Wgv$4KEy2GvVOF?-r= zuv7o`JB5nboeaV3m_QlCP`|BDt0&jVkTJ*oQHo0B4IdY}f$A*$-^0z97$!?sHe;YmXC>_%Sv+pHJ*}PH zRaB2kVn~9uh}{esYxKhZZ=Fe6wQ5+Mjx}mtHk!+K>C5U^9(o#y`fqJq;_3Y=ETfcj z-YsJheuSR}f&9KfaMJa1HNE#wY5IOHy){iwcfL)8Djuwr^6@HUppRH222r->MnLW6 ztH8e;u>_~Eg(atK(qKC~k|&NCmH5IeLkW)vU*v5ZbHpGaPcjVZh5!;;@4vr41fV$# zfoTg*WDy>95c}sNU&c709E!jeN_74IiR-gTWx+WTpr{eI3l;On(yGZDWz*oK=ZwM* zP2!^r>&|@f=Ize0z#~*fZougbG=Ih(Pg%Qasc?I3Mn`>ku{&K+LzCm_Ygl6@Q70VE*1z#h)#WkfqaSbdMG$20rsNS3ak%|Gm~7~+45|L}Ab4`ZeNi?nQ@@N%w8!uV>P82F_CYe73FKy6Q`btb`K9V_(G6`iXib zwpSWQ9Xuxe;+QzZX%mQ$$te^N!oCM*SYq2D2vt&>a9S!Quwyrg^hZ%mW>s}Ez65Pn z`w@zvwbGMDi>%1MaYu(0CUh6wAPig$WarmU#2W9;=7vnnw*Zad_V0sO>=2SDMTRkusMm{ z7%}OsJ2O(44KI4MS_(uV!-xzPHlzaEy{IQGUgBy=YA3g{Y;lJ$G|+EIBHUw3tY%&Z z@j$(V0qLK2kOW$^(VUZAX4`Saj@EjyUb0O%IfeEM^m#wi+QJ0+)xXS%iu=+F1dglC zVQ4kb3`|t^wm&yP+EAG#EwfTvzZIM%r3<8bNO6phphewJE}Yb8r-~|U65p^S(I+y7 zP-JFvy3o!7&n|8xKuDix;(EvxNIFh8x0n<-Ld^>vzmQBOAJtHll)^gJY(H%Ny^Ejf z@NkUv&@4N4*hopF*EhwF{8tGDF0bUJGdjAd!cN1a|KJbjQ5d73mzJ`R|3F1Lcrc-Ni^3FLa_{0@QgVL0%pMlT$Qe034Ub+42=Mf5K?hzr zZRJI%6Z-s;tSo}&w00yr-l@T8dcAyEySNzwiDgLyxy!=ukj>gGN=a zGIyrxu782!{&LX91kh*4M?Ro0 zHjHQ}ah9Fa%;`w-^W5U12|71pyH&B-^wd&W3H;aSi#2Mff6+i4F&L>R*6~-!aftAU zs9H5~n5ei^CYzI7g8?pejH{vO#C4F0&IaO&B;5H3PiLa8{?lSdNsd41=4pRSgkASF z8;Ae53g&=z^?!3?T8&}zQQ%AF*oaEzd{n}B*d})5!{Ct9mIk%OB09V&|Ci8tV9yjI zN9RbHmC9n?<5LslpafP5bA{KgF1zg{KNewlcpxp3;jsxRMb69B^SxZ#H)pl_L+Ta? zEj_$uryN(m#%C}Iu=n>?n1}%~8UYs64jnhXK_pdWy~gb-0ZU>L z@X`mvuS!g*P5oluZxX{s6aw9YpF7FrT(}sOD0x1fB+d75Xz*4bz6sBE5kx8bf0YTM zeD}Lc_5tgF0<1t16MVHnuF%_u2B22K{GK= zIE1+P%mNP+GoB-9oqzR2Y!si2GP@NEJkN*rA}uxOCf+x4L)>4qXD;Zlf*8O4XFm37 zhP5Z8PxouSiM7tBy;j0q??pS919kW6CyRfdg&P8uGchLh_GM(K{W4EQ$yS%qGY*L# z5hl&4S8Q%Fx_<;Kt=o`_?&{gKj=RNO>34Jn*>=4QYAP0?m{hWc0$XnWf7|j9%_}?5 zcy*zT86FNfT`U}9F?W>OdgSf>g7Nwtum%wp(&YFF$A8iAHosv;5d8@c z<4w*ebi>c!q0T;yX4#^RwyfE7V`+%BlK9x}OIH1uc()i%8Uv}_T-rZZK5hV?vO2fo zbDB{_YBruI(p5dXxy$=}L|Ij4t6_fLwoT?1oTC2L7+M*gB!wJx045#sf6ME|r)d>uu60HN z4IRl$9|7|jCM307o(mR!p@e6HB|e2|GVgjAjBjb>f&Dx`hU5$lP3q2fGgXZgEr5ni zBuo;%GCng(k%Z&2L=h6gs4?o4H~QDa4%$Q{{VNho&S%1o8o|^mZb=O(($ABGWk{N38 z7W75L8&_FuyxiMM_~})_aJu;w7XhpU6zpV?x+u7wBRZc#K1VtWBDbqT{_;YH942JX zOd`1+6hYYuN7%yzb9wmx2CD4fr> z7n0Qd+5I$m_dbeR@NjMVmcpTdi#csX*96}Gspr^UW?l+J|@b2l@alNv_F&077S4W{ zC)dw$+x`v}mt>j{F`V&s|vNWwb4{{4EX_{D4I z`B@o`H}vrwn%>DQoTboGB`D+n{)!gXxxPMERmf{NqBNH3SxDfpi9!CAew*iYr`JOE zC|>Vp&p3#Wc5~wO*Zn0NV&<=G`^w~u9R8#c=(gOOnys2qY6zKWCN<;Y>-`I{V9>zNE}Rx^!ON;EhHp*zu?XH)@lO^x ziYP(PK0Y#@K85)EUomD6gOsIcd)h7M)UtZsUr3O)fkAdP6gp6u#i3n-w+)tdo)7F% zcD~bdddHZ9?=((^WVDQd5`j5TA37!?0G_k@)bPLa8t(ZmL3GQ(SWE{>IOiq4oSUER zHrFVb0Hyl91%{ta>cf-@064jpJ3Iy5b9q1Ej6C&AW(^14khW{=p@sTbh`SI{kB30| z@y^XOg9YM@Zx~dJFI8%iR0$f(jp(A-#uF6Xej>qSlYlc5Zj-qnMX7fSDByP zhz(r2ru+VPyJ-2AnR?Qjx5Wtx0_^`D7t^cSmtq@pk;WOF&U!G8$Ir#w?#m*J9LnG7 zG@Ifp*>R&sRL9|>-LUA85ZT^Qm$Ja^#8ye`a6rvwhFIYCh*6Z~%q@D<` z7Mk0-sq_Pq-L)4_Yil3W#0{i$axd*pPS#b8xZkZ@A$)_p2UDC&I_UMERv`nGBmIr3 z(yx3BOl%1>__F%F;KkHJcm~^$W|3i(hefc1(Oh+k;L$F`J)u^)p+hHro`H}fzE1dN zG-S(MQ}*v1Au#F+QYDW~6wS%(mMY+*GYZut%@ANt0pT6#)l*2mw1GXV-WX03RCxbBy+7i|W}S$zu$-&w8}gSgW!2R12f)-k z0YSk9&O*`aZM3dza7`GEjc(}1#>V*Qr~nwi*z4*jt*WZpTP2ERRc52@c79dR034mO zEgo-yPgWjS3p5{ieE+)vg%XFoSz&N-nBw$_0}WtBe31yU=ih>vcaT|tO(v0CrueWb!H*cN@ znn2P+DKE6&bG9!rl43sY$(*lIPy8a04OE^w{OWc>(stUwat6j!BVfwvNM2X_78@%T zgiV(&O6=s*6-rbv(Vu3`_Ui8Tz#|7tLJf1jSb1vXF3h&L2nM2@f*UuKm~uH;_*z6v zGaP$FBWg_RAoS0akK5tA(Tx0andUc@!T%x zEjZE}TLVe_DC?9)hQg7<-R=2O&fELTp^h$fV7fmCSJu3~PUqeFf6er>39?Rlt&oA` zKoLMTkjPmf^SqW3+iP-ATCLh5r!3OCg3XJczk~Bp0*!Gd&xwtLkZa+^HTk<)ZxhEt@<~r&pye@eN$(zg?4;t( z*21iy9^!UinrZGaHAazGB5SLtrpe)(@(ws5d>KYW-lku>bgd_g(K_%DG~5L}@5c*x z6@(V{j>Yfw;AiV3Qy>ma4Muf^XScF$Vd{pA@&RmWbKr+B>x#+ zTCC6qj9tILFjKMx_v<}O+@Ck+S5;G58Q}5+z9V!+BEF2TJR}x%8`$;W%26y7)er-> zz!{g=5ZmXjhtOpX`LxXD^Bh;;QVIXJRUMip=@TCJ*2zT8kn~_r-*4Fdua}gl)Cj$VC8ex!NT$cmX5evZb`b619e!C3p2P z$7^868wG=XbGVhS-7l9AJUk%cv1bV0{Qb!xa)UFL*MN8uk^7q^HKIpbw(KG7l#tOCf-UIw;b=05E^Jx(7EV`DIHdZC9>DZw)*tfhedrFn5!ATnI(Jca0@jG^-xn zeALUS7#$wwgWQUzYq@U2V`OTo@a78`vaT6}RP*ME-*D3FPsl|Yb?+_2O2*mQ*~h>* z;)C*)=%{tZ&RxYu;8HF5X-d$cyu>qiah7s(%K>Z(tVlIyW_v?3CO7OYmm%d(oJEYU zuP+ksk=}sW?!7HZ^<1~8UbhE7sMFyk;JT%Rq@+kU zN{4hycQ;a^v~)-#p>#JQozh5mNP~1Y$f3I%?>_4LJ@=QtJa9I9ub4G!M*U}pj=eMl z6H%0sTWcEo_Y8Dh=}(TFhX?R_unWpAvXE6kXw)ao7%cmT>&sw`simBQ=(eEQSDKs37N^`hGNAyilZ2)=&P zut>0HUL{~R9RZ7V-)6g`QP78*T#w(qVEIn9KibU?kFxuoF?F{_sOw1kcwt}op-nRS z_CmVWG8*+JGWbtq1i^8o6`6=fui`4MA2Vy_OuaSYp}d~hi5x*t@R4!QcXmPaA>g zo>D7-u#m0v>RMdOHwfsvalVhUX)G;ez5)C&1uONqJ}B)AQX5~r+E7==L&$FW^)sljeQz{XwG9Oq z?x}z$9Wjb4Fyu|9*8^Ee7F}T)M z30dw~It^>TD_x!$I>%|A7Baev*4-X?EjA{1_*7`&E z#N=G*yq>YD-*B^4o>Y&jsa4a#Os7j)Aisj??42R_GNvo1d-wlvjJHPs2 z@9p{O*ngMW=*SBGbJO|W?iy$F*I1zi%bT682ubQLwI~=2hP$n`p8t37T(G_MHqURK zzkHsQ8s&LtGM=6;m*QnxpY;HB`+4&@*X=9V=av;`Mxu@SLazQcC=dP`KIRN8F|!Nh zE?+>^-2M0VYfrZVfoGC@u#N^l!Wio3=UHrKK)>AA*H@NRe2LkhNt?nc5R_uhg%wIn z?$|2%Ix+qD!*3o=@CI*GRaLhD@pVMtBM^-qG%DqJ#HKtzzFTZ^eSNJN>|ypa2%KWo zQTc+9P* z3m8NBgraHjktxwC!uEq)<+ao-F1V?P<@AP1hZg}N&F*3Ym{Nxu{9R9N0W zYLE1O?Z07c+*)xW^54D2e-7GU>dlSorMAvo%EX^JaV)R3m0u3B;BU)`ofLidIrBIj zs?9F7p{BBS+cGf0g2f!NJdeqA+Hlg{6FKTzX^gMlkQ}J}f zHuJ|~K*w;(jiyLn=s)RGAueH{{`-mKNIY=9uj-Axeo+g&N)V+g+2nJQ{GFn|`9o!m z)(=^L^-%m#z5c3ChAHS2oB&C!kYuM8W3)VqFrc_1ITH=i#wkIX1q^$=#CwYhO?z`u zkc|Yt^1Cp@8l@G`bPDusBQz=jo%#jF;dc^KBAxm-gCXX!YsrcB(_C3m zB0)v7z$a6Jyb-I9fP_Ra11T$F+57iB%x!9OhyG<+w7np(Z%)1pxxS9d)~p0f?sad` z{iN$xJVGl5tx#iH7qa<)Fm~g{u7yxxa(;~4BL5yqH4m@sQVs`81rq-riCaA#B=6<} zHX9HWG2!+6guOBjw#agZ{7kMpw1N&U}lv=z(9FA7a&K){wtSVKZWfo#P;A6PW_ z%}d7?GDOHbwA-^f3m>S1(zyffs`dt46+f{&BMI@N{CW~Uf{ zTuDa|d?mj8>i3U;H{6-A5`3fXMKji1I#5FF6J!HLsWo}2xVrmCZ_6g6fyTZ&;*S!2HfiY zyhjob_}tL?)MiiQNs)obz3ebyrfAmclN^^)Vt@@p)5nnze8j+u-N;3}XT@hy*3IH^ z=|bt+Qc@8pu%E<)DQyu0zoz8+wqejHWbZ%A1%w8&<7^fG=>v46CmAz8ISLu*t6v3V z*FVKRtwd{f(kYV%{}J3{=ymo&S~02oJ~1Qy0`dSCv9Wjd;;!`Tt3*-Q_2a3pW!rfQ zUpX3E4clP{jS>)r)kXNO0X?)zAjeaZ-*Mw7ud_UZEZ*ZGsLTEO85gyDQifJ%-Wqu_ z)_0Gi6%pV(kGH>29~;GR35>?LCf9a;KsN}02Fnb+z3Q7cWsuhrr5gCX)=^<~%u(IW z90~6Db-PRByLgmU;XF**je)hc%cT(0Da!#T;?Z58eV-TEGc}dd68S6!EuSh1mqf0y zuqwANbqR)>8xsdxUZp^$Vd(whewO?ChTMxmROpvn(LkzFpH#(=TH1O2oPp&g%P)SZ z?mS}cS{$04i9>yU+VJ8kwU(h7v`r85d?D4p4xa2IeDj!|gWeCo%*ZkEQ%_i8K(~yO zIwl0$F~{rhzpP#!@W|oRsk&O1A&VF_GBgu_Vv7z-nk#!MYZk_`4YeP1Z!CR%k|E^) zuvWKbwOnVK2h7CjV#0a}{kKtIU3CrgoN*>VU@Up8qjW1$&>A?Ie2wycb}}hhQd6Tz zF)vcL44h14#n~twqqAq?#wWp|DxH~|QquI>eFHWYv$-10B^~~})4}P@RF|}CsX=S; zrS}BHkIH8(d-I{iP1H*~M>5#Q!F^R(cD1*BD=arCHQj{~jh8Qj$cVriZI~wV+aMetwaVB5x|E`eXy1a8CSlQG$uFu{nuD??ZRDBve8z-m@;M z_#OZSN?@C^U>duYS{XRUrqjVQpUk6oqm#Xo2ZozkY7CACY15IRzMLCGQe>E`_D zTH!P#c3Hiz%j53nbB;}=r9)}s>w1XfFR7WvVX9qSo<%7{49HKC!K;_`)8CDK*CQjQ z1DYJrZ}12{;_O0ItbkP_w62(guicOk&s)?*+=0-y?b%cCID05T#xy#Pn~R_3oqcDW2`wEx@shV!AAmv|NINx-TJqXJsFxnHstd)8w9VYwMJ@&Qi z#H`;(iFLu#%CKq<2ReD0$icN9Zr_k&DYF-%7Q>*m!*b8B# zw$Lg;ul7PiLvxyXZAQ1I%5O7=M?@sI)*YU`+7UP&ghgczIKCWU#c!_Jqc(pq*8s}ROwYow^-UQopAO`#}fLTy)pF1KW$Bj@0iHpZuILUC`Djj8(Wh1 zvHm&AJ4_hqqgrkq7YICK)N-U^!i3BFT4^BZqtHj1a+ox;kW)&M7`CG{6rJ}Sn zIYdu`-7LCRE|EDGV3mcO1yOxq`yLA_OUwZ_0y;($3*+5JhQ1s1VB?0NVx^EfYLyu% zKs6Ifw7-jL5nh9=N@EsP3r&B_XiNEF$=8C`ZbJ+8j?i_<+6AWcW_;fQ0hwgN=3x4- zUz=QUE?h{);=dQutGbj=xSK>Nna0O--dU~RPP)J4rT_a5S1^#y%GZ>rq!xDwb8aff zUo&6BFJ?VxubizK(Z((3cY?3pc8rAYP2}V`ubNFpj@iqiCX~vXzOjLE`DyfmjMCuw zTE*Dy;l$Nv0RhS9d>o32i{O1jZ|%pRM<2Nf2PR~g<#>)fa4@&{BfVzrk)1B-%%0{@ zb$O!tvKn=pqnks!-X2B_AN++Xi`ex8qjafAQZFkaA|RKse>r;xM=2gY2RyZjL?iU7 zO#o_qU--J-!K8I~gN)R51?EMe^Uc|Ia;LSf?$>KNgIctA4tH0_@D?VJD~TaZ$U}J8 z#6xR9)mf%kU<#Y2%c4gF;ghV&>L<_ZlWZw`)L{}8Q#N!z5kk!63$@(yU1}>m9jYN# zQynr9f72WWlCV8u!CsV~^l+A1EG-YCdRfvN(w^HZ0;=wt^%;{9KOY9nqtw>|S{ds- z)@MoUk1L-c$p2*g!WB3~!ah2s*4*MLEsigdyEX9Mb2-%0|869MVtVsjxcAYYYXx#; zh|jjkX^`*oCM{$m_$+lnAX|-ni#eh$95oW3Bkw|B`bmQd?zP{@>;5B9ymYcQGN=eeS4Z)P!OD_z^_<;viIW?rVKQAt-PCYsz|6gUlMKSUtn(AtCq;kd1NKxGV+_nE$ zji+b1OsR&Yhl!FcUB7ltE#Ucmx$w$Q_Tj1-$Gf!%68M?23fZ zR8TxkFt%DwGgpKqYcN3VBsZ8OGNcLw$H zuq>({?VjOG1&dZ2A)j~>eK0TMOe z?}u}gIbHp?T2f|5c(=An`trSMn=Apy7`+<5{8fjKj(h@<{)SEHHAID3HyQV-#Beyd z>K`yg;{T}6?ZvFmYC=u%;*;RTSYDZh+7~Z$i;F6jO&ux2u^bt=tapRM!lc?I0^jo- zf^@t)npZ&132~2mU$_wrRye(g%O%y(i@W!de*bvflExBfx?yaG}N>A?p~_x5-^>?ChM879@|3muF_a zebAmgEcFE?6+S@G-Ojz=Rig=DU~q#2kM;-M%zSks5bAS>z+t z;v((DR%G4o3rPO#$wo>I4W55-7ddz14bV3oIGS1sEPBlGa(D!BQ5ZWZ zmwDIvYT_QS>ge^?Bl~(BHfvR`hlHRcbK9ng>H#54Vg|(5dYDeojYxW4OZqFQJ6E3k zmv=Waw)eHI_u*ODlO#_>5=1reMoWP}^!<=nC)3Vnj*?`5uo#df5qd+CEa2t<4Vhp7 z*Gt%ky#3bj5uGX1uKijir1*6u5_a$V3lK&yPhfS~qr%S5&kH56!9w$%_IZ;|aPVA# zEGQ-|9`3HsrduE4$*rM!o8tf_5fzS)t(OGAjZVXYm*3JBXJ=&qq2L8X?O+R^ z_GulE+FXC7;bb)-CM3rPmGRRyXY?|!#q#sBmDZI zkr4Zt2y*=w%E8RgIGjJYVw*Q+8J7$aqIa!RopQPxQcyKNX&ro(|Q>NnYcPr{Q7 z1|rBEQ;NYsHHU<5!MG$T~ z8lZkECnM&F6dSYjUb?K@c#fFx%e&u>2uME?T}3@4Ru7EqklZ|;pyb=kh~ZGO*$t)e zay`|WLV)VaSi~6Tokr0cU+m2mR(#K6t0_)o)-RNKN6E4Wmg|k=K5)fb+uG7ZwQZOV zbF!96*)~>J#}D!CGK$svo$s)bAg(rgUOR+@hO+MymMhB0WbV8-a`>hLDJeP>l!~o= zWCBF8WploBA_l{(SnT<9UIJ%CQ2>?{NFvYf(;=<63$=bkD|y>v>CK%zEdPsR|Ij5Q z>nyha-6W>% zxib%SJ8C*_JVy?`gNYk`bknpWbK};?!&7-@t%JAE;O#1IudLF7xT4~amJCywN=Uh2 zw_6Z{|z*>FN$ge@o5r8t$nuz|ZWxh{@g+wx`iUcvlT6Pm8|@NgdTv9pA~G`-4# zAC{a%=>AMiqHyM-@-`idkWiMT>Bx`SKC}{ZHKcd=g@s2h0u*Ht5mg1L9_(k*qvXrWSv4_)pmsJ5jH`rDdTS@Qsyx6?FJ*_vAloB z2iDr&RtLBb1=@5T-D1BptQeElFJ)$Dq>DcgS`%n+`o$r!+Oc#&G&jGMxlT5AgMCXf zxVImgBE$n9wT#27dI)UHMPil7rL;=*WPmXY*-+urCW^qZ(i`uBwoDIRHrAf)MX#%$si1X4b&dS>8>L1IKD z8Zke=$r2Z&Yh))wC(@P)da_+*p8Lv*+FD!9EFH|ncV!X*pT{)R%tpP;Qsk&&#`j8w zjD{oS!}3SAq!%B&hS>(rU#y=A%3b}30$h>7UOkktY%Ie(2#!>an17swEaTa-)7srZ za@@|ostxBA*TDxmwSFrN_;@+n=oB^mZYliHb6qe!7olfld=BCFpD0ONB^LLjJUId& zx7x}0PHMLE9l*h9Ff+{YJ{PA^%$)&YX%&Z0zlb5olk5;;31*e32zSWy-k>k+X7ry~ zjO|5AS1S{m0fep-s8=Jz$Cpfw+5`+28h~y}{3{|0;!c*VbO7iQ`R8+rxwRB!TI3+uhZg5m~GJ=~kp^4Xsl+ z9AQj4`G({y;G_ncT0T%|aB&xKZcpAu91XlZe-LxZ(M>;#2#SmkL@H~U8*}@Ux)B0} z#Vj6Ib8r1dKtMpWPn*{(i;+V|MbV`EG(^mrAX+R=GZ(UOa^JWm^mQ5wN3=@bVd*`^ z8p$p28kKzW@%jajtfReFh@OAeNkgd}h(V@^ipxkYKFl{3g^CwlYB^n++utv1A_8IT z)f$ia`qh$iOtVY~ls+i0-2g)BKT=XqklEBEU?Pu?HwPoKH#pTXlm-NZGJy3aJqwFg zLSZ4j1Uf-ylf@TSBLv^J>S_0Dx5k>91agY-$D?@UlUT1qYwXahSzQ)9YR;6(th~Gg zfy{f@#o=<<-QlCBJk%>oUCjyU=}OsBkyAyI5si<0TZ^y#6#E%6EM~>_-dnp&zP*{( z`XlmH>fUs1%+vmXqk0P+221Ohm!M%i=Wmbf=i@>?GCJ`yVgv!FcxIT6F8y@Dy}&QN zvb5WE$QqT#5?*K4s0ih9By9(!;bV8-g*w$H3hc*VpQzf8ZnM^8Rahsu)uM2}duWDo z>d}8zjt|6|6rt~_K)w+1pEc)Nk)_@a`3R`(w_|4=p5H#gco|jq?q7TU;45sY562m= zwT+SRhU;k{L>bVGY`E|HQ{GSpIfgLf^Mwo%u#aZeAIvv4Zuy_TA&im1J(;?mey#Iw z`id_A8(H=cR-+X+TBh_WRx)>o0bSd&pru(ur605&79~l;q@Tu}(D)TD3GI?4b8L*W z)>1~jaMcO^1_;s!VI?!XNgP}vRW|XR%--jRYUK`7FIKi6^LZ~A^O>EA`<+Az*5no^ zJ)ErLaj@44_v(81#5vyM!bB@v%*!37__`d2j-KvLj|jEAmMPIsMYnhkb`lQBfC<1y z9t*y%&+a{ts_6DW@y|pF4(}nzo_$?L@EUYff54ARs*$cSMTJ0+ECtl%Io#pw=Px~8 z#gMKopAOIr+tJ!LM$+UQUPDO4I!i8>N!Vcj6{X(78G)%uzneB3=hQUbPTQ?@P;H|R zX}ByU$v7+hn484nP9<(b->{$UMmowvaocnrH;FO*({9+|!}h3R93Os5qjR(IuMFj)nYKJnK8KR$pE+J+c~R!u^6NwP;OpQ;~FMgxHs#m$iVUT=i*T)Yj0jI3LL4d?p5G~bYoa{ZGuMS(Y-=+wo_;WE1J8V_ zW}*u!h`+JK6`s$%VH>09Rpy(Yyy4w>Cg=^Vb{V2LgFFeso~DcR>GE>MXv91UQu)+a z4rDrUqdc;)-pLj_{j4@7B#t-@Y)aenVwPd+WCPY#-d7B^wD;yG zGC1vYl6%$RFC@+`Z8b3`IG`wk;Sm!2LxlBF{=k<9zem`iw)FhQ<97URq93e}VG7?{ z$ZR+dsfT;stvhKEYr1QsDPTRLs z%M7EntM;QGw>kHJ4NwmLDHK;6;8$Mg>8f{+eOzp?M)dbrh21o*)asY6%@n~!!a&bs zpV{8Yuf!cWOYsbBf=_mNJX+(eeNWe(J+0aCoXhy@e~ z`JF*3$v}Ow(VpYXkK5C}$5cov&HmMsX*^E(=X6Qqa!%DQm?@BAg0wC`OqN-yFUfq~ z^Y_r@;dKgkzm?R8)H=EbD&ogTh)}dQ0BcTRPgnSRrwvcPO3pFVS<5%i_Vc}q^<|`f zQrbBF!rtmE7Vs!aIDHJ$D%QqomAu?e4OjEnx9nQZV}4Z?T59;=MP^ff`E-S7@-PzJ zKd@;^oq!eGSk|4?nZtDNA~E~a6`MCLHzf>Ww6%e!C{5dA66})1P#RwK7r|%1GN&!c zNJGD`f}?>y0Ztsc5BEnL9i2x69X+(I!>jEV9{V~GWx18HOEV48niVL4NH1S!QKkQD zyFIaBqDqwPhU(;a!!akm?XmJ`tS#QHQR2AtAiBJx|GN z)S3U5Tc{a4b)G>Xt@Y<@FyV{IeKPR%{r$5ydrb3GU-FCHR<61nbD)Raz%V%S>@?ql zB5ey0-|q=D2^z^)lmWfr18s`x7hKrERFbp3hRKkH<7XgrE?kM1WN{s0tdM=B+iU*x zc&Q`(a4{1|u5(!gfR=sT1beL49WcVhgX%(Jnx42T_u2mm22BXQrfJW?zBKE^@>Q`@ z#_8QDE+Go7Yn7 z;pa|Waj)Se=sg&@Uj-lV?h`u)%!%1nsKp80)gU;~=ka_D0K>*ffAmq^peeLaXRCh1 z#em%ZQpg~ZFm=7J$zixQd_IZj4TU#XQ{x-^UovrP8Q0gPOQh~Li-bBovJ}wBzrq7O z1L6eLSQz*VN8`O_unnGX*SY6rM5E?pp5$8@hDRqYpXV35L6_F)iY%cX!M)yv78;!O zEl#{^K+ASw@Yy8%4{k<2)v|B@8>89*WIP$o>(U5+v>+H~hhq7jqS%jBFyBXJUS%&S z9>NiaYFV@uuVBf`*W3D>G!@~SI^#e5xf-eXofNe+z1Xjp5xMC5tURJDD=w$oykcAW z5ee7tmKFWvD_MwCaV1Xqxl=aIiZgTkVgfeGJ5tu-#n0vv*8U~4ZcLsrkI$G$aI+qQ zsOT8kNoU5*?_PNbuu_IFf8o)Pv4ngRnX0K7S#HtU;Fcwg_@S_a1s?{*%dp~d!o;wI zAwThMov}EM@WPWbq!|Lr5sRtNSiWzDr;2Zer3o)PFgLB=x{O|gu$4t=g~=)^KW7gY zf74#X$SO2{nEQ&@?ND0(&#zKP2m^UjwAm*@g^Nq1pwbtiR`zl%Pv?vLP<`OUBHmujo8|4n!P#!E~arCMoKZ!?TSA3 zEt~`;Pq37i9+?shaj_Bb{!r1G#|ej}?ENiex!#f1M@{p;cGndbHp$3_R!}~uX|mEM zqpf8^Y_ASti`%Ct9V(@iqjv4Ggym+fa77Wm>R4PgJC_^Tx=s8V3U}MX6AvU2*lNk5 zz)mbJmOzSiyO)|rrqc(Xw!t{d3cdajazkylPLC?!J4L+EGhk4-YOTZxAr4yC6A<+8 zYQP}#Vu>}0(~c60^1+6V@t_qbGrW6agPUr3Sg&v#s>4aAy$z`gAhVSJ%H-$v=0Q`C zwSUi@%s02aNU}VJ;@om_jzpdhOBeqKeJ4k{VuY1eIGgEL+5*$xYp)w|2l0Nti7osW zSuU%+_9(%P=XM23zgz@1E))fi)#*-4v#~VSZb+J*>|{CD#e#Z~|uNr$2; zNz|((on@InocnXeH9H@lKCnv4)^@r0n>GPo`vA%Z$1`g*p-=+*EHQVgUZ&80(1dP6`% z#LV&-T-po_K-F7UeEHdjp;uQ;YAmT!(Cnu;A6=k~9!+d?ssL+KH1a|j#|}n;jbz$g8HYMzR?X28)4y55iUD*wUx0g>i|}qt zc+Ris95ECLT}66+JTTIp$6~&t0RD*a=oN$UY+=L;NQuyb7n`IRIVL1bHdj%kjTD|f znP~e1mkfOa`VM(*NssT|q=GTMl?fk{~1I^R2XRXx!KpaVMoM6)%ieMs%KJ+h|zVVxM- zzIDjN7+#8KW@Qh1{7YhPodTKvYBW1i*d%p{w5#x338%p--ng~4RC{C6Ov@EPjbZ}e<}>ipq2HZ;5z zSM1P!I2*Jhp_su=k7}xGi4B*%F;c54$-X5&M}~a}@)2LUS<@ivOwuQCvz%?7DYsik zGzWzR4#BVF#OIKn#Wt4a-7i0xiYI9lnQr&ChLa$+MV2qst)D2JNKtnZ&@=e|I{>{% zVYv@X{PPop?|Inhd{Z4_p2U;LQ()FaiKQj5e-3zFH*}rf^xgN>Qe}7x^PEzWm2~ z?Xx4Ke#}{mL=zI-%EV%CQVBPQed7G+d4<@&dioP)d14bM+KYhG1${pIQAUm=HSyd+?}-S`KuG4K~&UN9#Z z4kwfojEhTGXkUG6z$rU=R)-DiyHB)rRdUJ zXhzCm8T9*?X)Sfzd%(I`?yxll(U?F+3OXc5ld9y$Z4!&4V=>%5a>bKmkvvy$_6UC!>rlM$}*A@M{Na_;ui|cqnKcUS{j}< z58UyP>tH$#UUn%8>+z(f54j@f*F|Wh* zWt1!4N6$+usoh40(h5;@4C+F;5^(AzG&(pmJ*J|Ry7_CPi9G@d0fE zi$UM!iK+ejn2^$LrMZ838qA^nAF09!*6qPLL;7<=spT`n@O$!S38G^pwUcRFXj|`# zt}a`%W@nx^W_MfI3Q`cO5Di!a5f0FQD0**eo+q4;0~mx8Obn0zy~8~*u(A2jOaeWB0&$uz05#I=NpkQ$iLbnDebQ3l z#?%bpra?om-}QUG92|X|!CPXmpOBvZ(hf3JL6&-IEvP5E{2d3wM_@_^m9rx`;c@8DX ze5NIp(&gY6U?Ju-czvO%HTIs`emaquoYvm;6Ew!-$BlumtMOU+H}kUKMM3E8roc3tSET& z3BFX?y*JBo0KdgnpwCOb`;{mR9VDLdN~t0R-HSB|Od&Symu}&SqhH*_^a(BEND_s$ zagRga^Go3a1VM|sl+{gta?EPLt~kLz&eYSHZ;~izMRbW26CrBp3DT1)oibvazlEEf z0Brx|Oa8I8Ov!2OWqjU&aC&NShpL{f33D3K{cl=-g$rxPX3*X`+iQY&-Bl^2WQP)rIzc`jwho}HjPTq* zr0t}mwBo~|4TGOR-U4~ZMIYgWo-0n1c4&+Ot5l2Eo<(-7;m?~KlFBHGe^z|uF%?8X z=4KV(=+H)JvAo3Q+L}J(D-(rJnhd$K_o=!!+;kC0Z0WHaCz%PYPhmlF%d(fLmc z05;}yYP>8e@C?Jv0E*>cY=?ihfv4nV9uO5#bt9IdsHcMMn;0)lpr&>hh{v1MzpK~@ z3f`Y;KzL7I#>V;fLpUNnq^(-YZ~Q&nf4GL>BUmXOlHn;OW6f1-@-2Y<+M^p5xm@VjC?X>4jG0beZM_Cd8&W%@2LqW~-N7H5 zKlcCaojo7>QiV8qeR=x}7mGBZlPM+Y1UrVefh$!qH;1A1P1K%CKE?8FZ3^ki*DTSS z13Tn%QdUfry&-hVs{Qa`PC~Kh|Ap+}ik37z zfl_0y7Z7kEosXj%Tdp3(b8Zjd=oU?n_6YZOJN9SI{dfl(Z8>O19HJ$YQqemf1ZTMx zWAwL`L-jS#)E1}bp+eC_QV$*;JW4)%!=oS~h^kUU<1nIv|8%(YC8X-!b(l+nX+U>* zP2V>bpd+r>u-I<9GFly$ED{xK%}?KDsk(w zEiAx1_3!f@k!XvLu6>_MFx;w ziyR)&eoB?5F@IT72g z{AG>d<(@b8zg)gMKCljXT2nMvWhF&v%Y+p@?B~M@a0t$T*tmD_Huo!>fg3jG@XF&x z@8Zc6?_5poTXd9OJ`W6>K3>|9eL;`6J6D4{Seof67z>|@aB`dx6ZBcTYzQV3`7`mK zuhRyHsoWCFOXFtdSd_Vxlbbt+u`#IDJLefufrjj0Yg|$=e$?3|M@p?6om~oSH=KHD zrV{0oWKnsg{&&LUE5S&0Uz<4J#^9xkn}y+LH?cR0_K!~3m^ePQR#mS;S9x3VU0o#4 zG4*>uWz}d(XeOZfrpHpCj}O&hs^){xbR_`mnqJD_N9FW2^Lw|0kL>6839$e`$xa1BN^B|H! zeKwv@@ouhtm~P{+ePcbPc~S4R5v>W<4Hnig`uLYprD=%OlbpZlGbQ-ZQvNFK_^A1w z7a#qL=q$J6=W;JBrC{Xa8Di}TiHduUdCuGivkFGm2hJu&lE9{K09D`dA_l(<~9 zH{xIf@TRFyk#O84W$i$zNXSujKTU%vC!VGIM~L zauLg#x~)5{6)f4M%Hn80?~{J)+XeFJjqPvH?%B;CxICZhB$!tKc`0#j{?>y7@#NkP zI4G$Fp`PMzN9z2o=)pn`3RLvPeJe8%L7f{caoqGNePlCiW!Zn^TOUl)QvMt?ybE`$ z93H(dMRTGVo@~FMuK2`>i|?>zV?yJ|ld)e^@cdg4&%jF&jI4|ZWMfGCB#^yZMn^)) zfi4thIq3a~aII30mcK{_>q2V72kp`uM69&yVp?%kdyU2`X!{k=XO;7=03WEx_<$mZ z2oqWYFGI7Q*I_c>2XR~5dHKV=nO_rJ0`(42c^$E;@p_NfzKjfpoD2M_%xOrral4>< z&#~oC9GfN@9eg6X-FKOKrT~7f01(2f=Cu#zcuo-=g~%M4YX0i(hQWPzmj&$u8`CRa z($)t_*{}#!^(8EScWqN|7A5m{@1DvLC^_8}z5R9mT>?+VLA`1b6Kb691h)GzzHI_y z+#KK=;nntWmIA2`)fYJX=rlMLygvoIpukm`rGPXx63AD7$I zr+t=Cl0mY%Nl$BV?nJ1@5Imrmf24hz&TqA!ANOZzrcD>qyvQ=A`$Sf35e)@3x07NP zFnjs@JAq^;VM0pb-Bh^A53aA9e#i5h=%28umbOJ^L@CCfW?N0#eHkvmD^2Fp-H+OX zo;U-Piv5_Fmjlz6dTQYJIbs=>L0G5s@ymexXSltR^=#KooOhS;iK8XMLJGipcq)X1 zAGhb)^#?CY1vh#k+GNM9X*Rw6#VRq+y-suiX&>0IJmnSn}0-!qrl*dA@ zth&bo3#5=m0Yd2M8E8d^1b-6g0n?fhAu|Yh@#%zj)0f|I-A2&W=fHE@ZHMbfG#D|q zXjmY$f{pDPFhWXBpiX#)`=@nmo5pIpbm5DwAh(EQImRKg@rv1hGw>>3OVjf$QkwZt zty!wR{Mb7O2SgUiuN+4eE8N#$=}uWT?p|dWL+I}1BGA=`BLr*; zbYQsV{`bm11=(;hxTR{p2+7HQ=qe?JxUiTQpRVfTy}PJ!so({3k$!Vmq1KWWU9kge ztjt2}2?{OK+Swbhlx1^xbEP0^pv`Ix|HIGE zLR>ak6cX{4fX+plv23w9^x$^56{&*I9k&gGq4td{-C+95V{H|29R_h`IjaPtx7&0e z#yxkLqzV_%tP-zy4c(;X3YW=kvH*LrON8lB#ljo z=xuY9@qXhT7L@YfDmZ*=nd;>)i!Su4`I)7%w*3qFLLm<>1sFrn)6!;icl)x80M{$# zR3QBEDm3%SPk!hn;V=7oHg}38ymY$!`{5Ff&%K{6e+2IhzT*L*-N`5kkt{lPPOMkg z$?Ilg9mQ7)gG1TRhvIjUH7Cb9*#@3fC8p;e$!m1QLTYr(+C>gJVq`U zHcYttn`PRTv4jtf<|sYAcf5VmFzrN_FU?$�)>3Iva?x#a0z?#02XoQCPC_d=s}~ zWXd`xe%n}<*H5u_{#n-Ms|)p88$Bp$XC(%XeEukAObTnJ-OU)fY_{?i4wjXX;xQ1y zrO}cswmkV+I{Q7G)bm`(sgvB-WA}052;Ebf*CYheixu6kD5u4@5>7V*&~KX(OStF} z|G+9QgC0>bj`K9IeR3Q%?(DpXu5L>0b~7cRa5+0U8QxxB@1{2C(4O1QWJnyDX`(>z zC^L8xp?1q_%S27!qD7YXWA43J1n*Au_41N>3mGZTu4ka$Z=kc~PwIVl3_~7)&*}%Z zBiC#0q~Q3T^TYdQgMi3T>qovq3=gz$Xt}u)G5swqfR$?g^(i9D_;Q;{W~=LEYr^DB zrx|9wT4f+@D3!wqKT4eB#~#q#)HcMV*B5RniuP<6R@uSuasph1ge^NUjH!=5s{6@q$?DT@C&fS0W`npPwa$ zycmp8d4(&F4F*XB7vcH`(Ek*ldm`j%xj|}Qq7=eIG^VzRMM8sgde6RW=2qvU4LH8J z`=xd>ENc|~(v`v!6t(7}AAL``&c?L=2U zYGZRp54;!95SpBx#`v}aR2s6Oe0;bI@I%`cp)lqF1Nr_f8!U9I5-@39+b~&x8jkx~ zI5)yxtx1nQS!)-1C7iddRYAx2z=pk|i1?b~P`^gnsz5>OVskURN$h>KhI+3t^OTMo z@Z}=>_I*M7v$~;O6t)O1K4t7onp1Nw3rN~4gzz|h*PmXTd6l(@lDp$1rYf#}?|w#< zQ4`-6G2h#05S5fw%&PRr!dQvzXQB-=f9GdwEPhj>+xw%OhxswQsEt)uG~`_GuD;_ZTQ z^1!c?*MDlP$G$V=M@H~WlxEK=65h06$`I_lvz^_1bsuD)TiHJnkG#>cc$v+B_O$kj zxl@ivz#8jZWoIq6IzZS-nL?m!g?kd~Vmm**5WDu92QVs*Fc;nVk7_~*U)4K*x$K<| zTqS5JDBg}_e@SEie7bbQ<)*9tMoimaxk1IP1}>PNmpyQqMh!Wd@1V*M*hcLwI*Y$i zI#NW%#ycfj{p~&o!)@eFjKgAx3u1PbMaxFc7^xMXYnL$%}zC!^{?Z{Opon$h7JB3RG z8hj7}!Sn+iott-;7S`3R{_ymYLs0qgw|1&WWiQ3o(D`Kpei%K;@fW;kzo|a!^spr= zgx&vr(uBJ8^XdLD#NB@pQ1y>*{i;g`haTo=pFn@T?{dS zgbXp$*Ppy2>G9SNdvs%bO5S%c&DWEJV$xLJ2Zzj!etF^e1v)L+jk@-pYzlWG zumtFW?;@lOhoV-U_Ro0Hi>^)vSn{F7bhw>?{{F0(m>8H_lR)wnWF%=w360+4?N4Qt zc7QwII<9zCR8()@g%OdIl2Cdmkrt5d?naRA z?(VLkyGto)B!({OknZm8jv?N||5@+i6JMBFi~F4W7kgiOL=*;{=8WuZ97;?zDbfHo z4!(tQo~`c_CGYh(G-OoW;{@GoeQG~sh`R~oa6QA)h@$k-$cChGF_+`LKwU{iYZAzu z@^x~V9VS2_zC{%zt|TOaO(`ZulYhjNn>?Vd+1|CpG}!KYN8EZjtQL|k*0#L*eAeeQ zQ8X^)E_3kxk6JldV_$Y%?6R;*c^kO(*G_&c4xHF;ld(!UzQBOHbdJu2g@rGal))OB zng}Yl2|15J*&YYQ|Np-L-atrdf)8s$h>`fL;V!2uyn5hQhK%#!3{YDRMN_rw$Q?$Y zFF1scCW*o^Bo@C;*6$*%2Ez!vzSY$HRa7Zk@rr7@bE)}N7|mqC8aEoPbeb>f&}voN71b zr)A;r&DGH7CoDJ~+&AN9=X27VyhKAgDZgKxq{QmyOwKpp$8wfREtFk>ZebYQ1X3Yu z@P<8nC5u}QB{N>`@wn6Tzxk=5pgooxFSC-)gNM=6HqopjywJ-f{eI0bgl}}@$`jw4 z$2u`BAs~mOf*z~pq-2A7Kc1k9b1m z8Mz6)Zdr&{(y5J+|#Y{<*Yqh?}?6N@I4WHBAd2O)^4lfGP}vsxyy z`LUePiulFgv4oiXOI@F8(FqO?R?*NX(Tl9}cT`cq?&<5J{ZQ2qGajlyK=fYr4o}+` zdR$)%_}CVjT^W0Ndp|ftZus21yPSO?(&CQ&d5GEiL$%P0ootk&trTz%2?3TJVt4nC z^cwk6D+Yw0(4TJ+EO(=`i?+JnQtws5i#PD{yor8yF?ha?`7Jt3i!M)@nP(6aqusHc z>qW?7-|RP}*>zv1Iy+iUROlNiqcWnFcI|(Cx1!{N?L)<-h){?R>EBG~c59KIJw)AoU359M#=~F6xmv3W3I$ z;1JEKp4KvK<2s*~%G>@UEM_M8=V+?l#9+sf-gN8R<3sKp>iFJ~4kVF&a@#g3+Ik<; zA`h7v%54)R1(l#*7fSu^h<*)E(@QLfK-!x*DCCuz_vSl10s>Zl)1R-_ebkvjdwps; z?!nK{)fbwe6~FL*E*F%pd_{DScoU>Psg@buXd`P;f(S@?|_z80Qj*FB#|lgvlC=+sS>tdj`Bh2 zc!7xNTEI#buo;}}FY?NB*>`P+5}^`tsg(X`e*zg`YL}b1F4-e!dtXlv{^Iq#83VpV z9>x<9(Sgz)6BElUy@A*?O!)IC_=tV?rqg~Fg!`OjU zs+!;O1%orVLQ}~_>39LM$gcvJP;x)5HIR)bR}?kt2nm0jP5ar0-`pAtZwUgDzrM5v zZpV^_Y5f~ncs|rICK>?bAin@*@E;wC|2QAtUvD4e7{Xc{|G><{ZFE|GMKH7GXwn66 z|6tmK<3s*oJM0^H9^Y7B#l%EmHKhE@5|@_aA6h5UZuO>>CE|=)$=|D5e*ePH$5Wb$FJ&17YEP>~UsRk8!7YpiT=f6O{%`oNDeLpRjlXmm_6Ts;B# zs1bU(Y)j;yTO-$+^iPTLGcv^2(V|Zd3?Tjp&$qT3&Eot9Li<#k1UU(E`s3M*n?`VI zwhV8JjYO7098?`TJE@D(dG`kR5B#Du3#!dQ{jq~3t zs2A|*NCO1Ii4q-WUk=~dWq^D08a&$BP(F`9;w2FAkx`ZYB9#A8aQ{~KYk20!UYP;@m!>oUkc#NH0ve4^a7>G$_Vf*XN!DlCua=0r7g=og6?d z*u`uN>UHgl;J2r($szW*C(fS>ATIm9`ut~+dnH62yqpBQL-|%&Q*+|~@Zd3g%D}<& z{sy`VhnVYea;v0KB8AhE%D8-=91EMhH<_oUQS{3KvwK#|p?#-wG z&e7u<(O-dnhB@<+(rV4OpqD$%pRqTK4w+uhsN8x3llPbdzAk;tvZ=k z`4WSMPuYUK!4?>{C=q%kNQ1`%`>OF8sO7}Y7w$$a433ZgAloPxD+yYkDk~@u4dW3> z%;v?##1sV3p~P7Z;cc++xwkg0*Q(79XRi_Eq`xuZ0QyF8^q#s1--019169uV6Wlq# zBm0VJe4nxU*!xw#M0FRKkM0A&Xi)5#Mxi(#>7WWNvs$U#5du@po)NjyAK*M!2-&S? zSNE(r`FZ^VcztaL<}>}8B8kEu2rvHK;2SjwQoIJvmweyX_gZ$IrN*2Dj$PEj{T$K- zuET_yxyUS;0j|Tg*jAIEmvFTDunv%y*>o{TJB>EqGCKR>3Im2%zB=8;0e}zAE{x`1 zVL09s>Y*YVkJatWYhi zOtW`g7`2LYdbe9s+gvMBS>&*k+3}tGjR>EFmNZbvR{n1`S|I{cgCZF>o`~61{KLTg z@57!$&?>5p=Q`ek?_4U@e)l7+$6C1Mm!S5VWov~^TB86L`S0(J4LFTrA-Q9w(C_!# zOOs{>V$sJ`gLpuD38+{|ZJ~cjNdTFM4o&u*Lt1=FucEUQ?}+eENS%K=ni`(X+La{L z1S4lkOkPW>??pYo@>^+<*2hxrFLQ5+XWqYM>JK0yoSqfm3phGfGOGH6e(~e@v9UF@ zKFN7?myK`cZ(ri(+|H)KN*D~8URSEbyEOa}rP6hiMQgjedU2@vBCjH<6%YA0y=jU=bu4Mi__b32y5N5ETrs# zLU};%R!_zUBfu5*xJ@+*{y zLL|u??_yVo*=d!}kU2`@T94l$*#~gVVdEA1`U>NIWFCnAsuxnEZ+Wjon(&_aLr@|4 z`-$0C50h*Ln=FLOatq6-od5aGF!-Aaue#}OrSFp^tiSQ_D{kl1QOI`$0wUWJAJ z0=F@6lwf*>qVRO z42ffi@933#?d)B(9k)66SB(nN4dILE(*diYD#}=XgiF)1;hQ|d771QbgcNoi5(wXm zsj)7=7L5qe}yrP!5$qn}X-;plu3-$>i>KT+n`in~04SK`)hPI=Z5@HP9}0N@tdE>Rax(PH7?_>_ zj5K3<>2KkKpq@Tmzq6H%Mt8Y8-jMZHsOX9sea2V)05Y|xmcQUcF@lCr`*yPdE>{OQ z{F4-bQ1+u#WBLg!Cl*wI`8N7)Rp_c9{qKN4Ow0EnYwHpeXG?8bWxPB~U*IImX>`-K z-s)Hg!?tOqJRlcLGA74f=hBH@{0MD^slS@&{?)dy5+(rSi*6vt zn3iXzoSU1Le@t#MnVR`d&{F!Fm7^y?DJqXRba3ds{~OaJ1kR0_@6=Og-mVSBae3yn zX@AO9$*hL(4=GK{f0Y`SVfe^1rhT3Mp-)W-{N<+CDGY;%Qil%Q3=Rp%4XOUpC_dGc zd`xv!B+3{kgd6Vz-iSM~IzwXKo5{7{K6LlNbdocf#r{(BVV5NANycm+CF=ONScy@_ zCrB5WM>&MHFOn1{w?0&Id4+tHW!_ojTp*~{iqZDAy!}+uP`CX_{Gp#|r!RUKjVMQ}`^(UZs{nq1p0>lC%Q7i))>V+=p7t1KH5BUjlO1x<5^XHp= zmLKTM5E_L)GH6M}Qi-9_V$_3GdVDY#HEzbmNygPpvsl;8fG}+FgwL`QnO11}5Lgi* zVFP@3opFR%f!OEZvKmO{?-;&^A~H@OiyP7}cv&55|H@`aDw37BH1GnGJx}I5=g;eA zzTCFIWZATwi(QsPTW_XsELh9~Sjfcl5db~+N|Ub*>{pCk>v*$O$*LUDB5GsGfn$bg zx<~s^X7c^}_iv^ngt7zrON#)sj=-)&F%luutnO`GJ^$l6dQw&CQ_RmVCxT1pplgWL z#nwQr@euBG*IqMZ&viOM#pPk>1uewpY5^b?B>23BFasoyGLex^{daN{cf=4}=cvbN zkpJ-ZAN!Z)2V#MsM^yHupAHboxcLynK=Lwws@_xRBQGUY3Hd<+7d)AtR#d7N17HI3 z(urkfPt58$f;8=z1b5JvvWm6E{2Nrw9T#}!8ajK1AJ{&aG5PG6QJhXbTUAa+KRnf6 z?BD&`&OQQ_3k4rsf(Q?!*Mu6+KNFxW5Yh~&J2V(`MsK5mwL71pEcPuMT2_tt=m;lR4_8kMqn9PN=2KJN} zqxo#F3;m(lvb~UspM$~EZC8{({~qfzzPZNWtF9>rG}a{t$12lJ!)zxr_I!FdT6A@0 zFP<0H)0c3hFX^l}Y|R2;i1cZm0l0~;Rzl0(L{^kG2cQiiL%_pu@;p~yvrI7H<} zvIbyu)93b1y?2+uFvS1$i)%VqR}zEC~`c(kRZ#iuVSR=$A&)H$6S?Eh9mn_hV86ks0lu6@l_ z7aU<@V?*NzA+p1VNlQ&iD)1o0ENwW^xMh%2BDeKtv{PkC3^jZsVNd^79TnjnD=n}j z1bR7i4kI|jNlDN>ND(C-M}#kTK#(IaRy10dPrpn?q zYd!Zz@ObR!4XM=LFJlrV6nUu#PLel|qX5y5*60LdBn{YlN3$4W!6$LoC;LKUZ@Rwt zYsO{^qU$=-_uxzDuf3=ExuoLxO{hLPrw6kbrTQaECPkIl|LWof^fzC8NFCUi$fVQ_ zxk)oW>d3 zqQWwjb+`ivGi42#GE4kKRk%0&f*owyU-{pNhU~k+#itcPhp8j>x|SDTjfY8yynXd# ztX#1EXLDvkD(c?$oLCJH?``D8siqyG`-58Okdg5?;zA$GL^H)dL=*_F6!!>~+~*AO z6+C8IZIfa(^8GjIzQyF|)Oooklr*@wIK6!4Ke!(q;RiD_c@sEay2?f2*`qjlEN~;> zyjz@~DDj&09*x@;ql01W1=-A-i~i$9)cddjaHh<)nz+=rBXvk0q$3`Mo~2ngaGwO6 za+3XdL~JzZ->$=7C=Fv6o0x;8-r|**fZJ-tmNqg07=k9L8<3B|`V+aIgS^SD8;toB zP1<>pq^7f9Kl=FgqQL^Q{tJQ~>c+j;5hA`_->NmogAW4~?t6j9oe|7WJa|jF=KnJAtSJ zo~y=NZnveI&KeaJ_oc=K4c`l;$vf_2jb`v7g9|=2yyB2KoWP6$_2!jCa&cVQHL!R_ z^Ox}H>vRmmsE_VavG@Qd^QyNI5}HxPG+Pg~-l~H1Vfa_MUBRU%#>tLWXp%R29KD$w z*bNb_Ng1D!LTnki?*`OPP3rF+J0H`dmT*wz4tY^OqTQ6tQWWw^PtJt!1;;V{1L^ILE6e% zI=-UOi}pzpwk69Mu%@#4)?>&L&oe)LdD^zwdAlX?`pJQ3$HJ+9-hgH#KVU{W(fSkn zDsjLtOlA*4;S;v`L5iCA*yWjsE4W7{#D4lEEy6Nr6dcY{FQS@P9o)gI@?El?=N6mS z()0oTwIwNxsF!w##<+IyjQJCb)=WJKl zI$#3HNbYxvEJ7_5hUU;~f0SxYW_r$vYeY}d#RVSX>ZDO0e)n={cp1&F#+|$3k*v{v z9BhiXsw_qqs!aSBk1}NUCQ^fYh_K6j2LmC95KeLE(BZQMWCba5hIX3}xZ$@o(;6T* z1Z%0u`M81|{Fr+$*!wx3`ioFOXBUOGiCVN!55UN-Q-V)zMsg020m=H~RQ&dBZ&TG(y^UAZRx zw#f3rqPV>QbFA(ZVwZtI7%dii1AKxVOo$ywZ?PD?x9G&HX_zfyl@Ak?0$cETE{OE+ z+V=F!U8N6q`ZYDnAcfiKH1P4F8qRRRAzxNQ{r_If_b&gFeSGstyZy1ibg&2%}0Y`mtr2)rT1c^0I~i+b3BiJ0SKb zJn>H)*Xv3~a>eKQe)laffAY4#1B;Ryy(fe!uy6TK|BKhE5i`+op8g)PJ^2cS36F_U zv#t-O`V|XsHGeA#%Ng<3xL8B_HR&bIYYRjxO0G8-EGt<>HZn%u-ADJR28P3usZCS- zv@Rd2egp!RkT$Rg zsgPpJ!~hyl(g^n{52c4rInPCKMn>uB8_RYC1+fnA-vv{8E4G?-j;Vx?sM6R09l=56 zsh*fH`(_@l+Ri^?#(4dk6FbcVzZNgochg#Od|4D)ORnicLR{*rXw07AT$qi!$w?a8 z-{8D#SZV8b7M}sSLO2yNZX?6-8a6q5KZ#{N=6Xhp`mmgg=+n*PJQ2qHuHd6-VU=iZ zf;a!9N$1{CsD4YR_-ugdsK}H#A*>!6a#_`8?J6?ZR&zrezR|`R-xBA!=sdabmeZfSR3FEI?(}~H zS~F@uQFah6g|tzEl#zy@T*h7_bJM?}gRcZ2^8Fp*%Y; zO&u9|@78$o)+ll39}z7OF*|y#28DsYX2Z0E`P-$HZ6$P{-u6pjRDiNTFWmZ_aLd2U zMPoO*j~DAF0Z5$l;a41(Dwu%36H`V`xl|jNg8%o@t6$%lTkZ;h=e-uE_D{*Ta0gc-Qyx*g<9+s6=_bgw=lWZ-4-O2PYTFRgImN1TD zM;%dsHaC6%kq&UXL_4F6H>lG!^H>j3Xj~6-mDDKYajA}3<%M4^zFeqBcNoKdqyXnx znVyq+Yz}?Dx^x_IRY`K04xEzwp~NXP4|cZu#z!bBxJ9h#nr>~#@5@Hi*NlQ_fJF8+ zpu#r^2_PYzv`VPnhrK_J_`)ulEZp{Ee&?ysdXg(L=N<3-!oAOn(Ih;6TGH_!8`Y^A z17c}PL+kDfC8d4L=M!ek`cyT6t1?bS&EB}bsh+r(XmA&t2#<`r*@Yspp5ieDDF9dK zm7wja!(M;=(50=JL^4ZY^3QOT8G1!}?hqUkY8Bv0X%zQuB5NBGw>)KWzqA3E?B%edl0qKwUD+fK+MbCKAv+eVH7o zUnx1zR~l77crF9CmQ>+ZRzuZ1pt-#Dccnf=+>@J?hQgYgGIi6nZ&U9=<>kyhgFYO z^apI@7 zY!4wBrMqV<(GM8KZqHn>FD@|NuX0AGmfsfMyN;b}Jk(9~YpSih^?z&*;MprLFHhE1 z=G%lW<89H?^g)NRP;kdl2-@;((Qr!Y7`b^-{N-E{Y0ApUw90>8Q6O*ZI(EQkBBSMx#ZK& zhUsP`RD~wnP=?l;6If!j9*YmHr8`$!Xi>G3>l17-Ds`pSQO*`}HdV}vbz6SHs;~>; zXxp(KnSM{kmUtRrr$fFi#Zl)THKnzFL^xKz>Qr{x5Vy-Y8~LxQ+^f$0EyUr9i;(pz zm7eXG5ViP1Kts8?#kZ+(Cl@(h%M(Hm%U#^yKlG;YGf8M`AzakKRziA!=9)f@D^!SX z0(MR%4Er@DMfkNoMf^-{6x{(39i2vh+x|yz2kh&Y!htKq*_mSP+)r8$g=YM%Rn`e) z?VNjqTOy(47}%YR!oZ!6cm2rkx%-Ah+|Dnwk7~;^wu< zqizW~S+n|J2!;viVBKC^NiqXD<|XK0t7@p;Dpvaow}MKT zqOtOer~IN@M&GKOE_0htsQ2vA)7D}0sbmZ4tt*J0ez?@_kQOEIca=s~C_<60x?8rr z@AIteA!`@hiH+H~0E{swd<-2E^N+N~SsN?sq($Fbu-9P|fziFuu-rzM`}peJEio?{ z1`(+cU)7EE{Zeqs-6DlaGP!UH?ci**p!4DC55#7SAZ=L22D11+(ka(+PyB~o+AHoR zoP4f_tB1s!?C*b}&Ko?m7-TcOOmSYGm-aRG-pTV{dG}DwUQAY2Tn1J9v2b?}`Ma4u zR$h*{J8Q~++(?t?^z%R1;}ZCAJ$+Yp34h?}#ps!0neng!A-r4{ndhd&Kg2slFwn_e ztcH_h-np*G$RBzLuMG@%I9as+m-h?RizHj4erAt`%Ig8J~NX0k|I*x)U_TKvQv=*1q^3!C| zTNi?BXJ!asZBFcP1g^+h+npaIox}GuMOrz>?Yg$Y_aGDeOLh%#+DmkWpKm;T3bStD zoNtkXU5nQ7kWUq69j*x>4={65FAX+(NT6IC_UqQ{rPh@tH(!(N6%|M^75juc#Q8A4 zviyf`kA_-PrQ78ZUokqLhNZXXP3L@9f?LEFc*OcbW`s<>M{{@1c-E=rP077>%Jcfr zjWWXf)MVq-?@A_zYp!ZUDXV3gSc%F%LQ#OejE_)&c9UYtevm4cz*~QfH(^5e9B127 zj)v8*#%EC{+%N8XZY5l4%!QL8FJ!n6#bb<|`u+Z5tLCd{%1szDA(D3yQZzYns5F1{ zVynXO(CvLkwbnE%hD4@|Hd2wyb!Gb!^L(RrKe}#d)L@b=5WfD43&&#luTXha#6w__ zDB}6Qlw%$(dy{j=!__2$lq|;L1F%eu9E{vTs$Hc#`J3MuZnn2_}sX+ zUhgSe`(kYMMSj7HL%HcDPz|x(om}dNPd||qc6;Ay;v#OgZE1ot_NLyfB*+Tx%Ss5U z^U*Ql22P6yI|+3P?LFqGaF6n#3n5D4yTjUz_9966T>{%BcARoaYDD$Y1NNnG)4bCcE2KF0T#98#&~ zRQd5rozZ;O^g5aE7!MWQ{B}yc*AoX%n;Np9eJX!bI3;93PZzbAYICyxQtRCqWgE7aj|TrsWr;~hR3SKWsokdGFsDaqk0JA>8yJO zed`_lt>2eGy?<&=nl3g=`=e*X3$&-1Nb$kKkxhQ&y)IJ#KuE&pRx>Ep>qj zm&F=jL-!t9V5pXVA`2@=oVb(I0a60tm35(iZ#@pCikX27EC!nlk&bU?LS z`@IGA^E4d*)nr#tv{(IU7qt(-*Ks0oR_Sf26EN{S#zj9ivLQVrwP&>yao<~JdEOA7Ks|;;o&n@;@EqQ6>X5)e#4YR7hy83bR zX8-ueRz|J=Sk9Z}`QA8R4bct3)KbIAH^uK{fLjiT{I?>xyH3iJ8RAO9>1Dz`tiiS2 zc)OBBkj%i(5@Rdj2bY@xLw2xiz4$B8;kc9Xq-LZ=9KMpq`JvdtJl-OzEcoqMSRs6# z$?f={6fZjws}xSwd`Dk~Ak7k49?Xbk?H-k1yjV_FP3rb$?u&v0oT7x^hBY?vx8Kh0 z;@eD(p3_(t`|2phVPYc-#zmSA(_47e4o4K^a9ac5|7QVwxGm7u#0#_hxCB}dgtNBU z|7y;oL0#_LM!v4q?0;)F+~$B|7ZHWul)75L%Kq-_j$rn`2>%c3w z+d^)x40xpCxJdUOCN2CK%*C$DE2AgoDX~5S;4wAp-lXv2+iR->$`(g+_SXIEhXQAz z4XW?@4-DMA@+sF>W(P%%6H<-CEECdMaxBg}Pt^M}D&LnM(spEE-9W)#e0*p4rQTfI zAI4;!(LOZQXJ~7EF@HFV!1aLbs;=H1lw>g?7188PnKQ?^hFmPVu41$-=(lxi`YJkQ z-|zzgBY&@sZGAOIHF;r~ZUIJqqG^uzjKtyr{ln<=44}q?@@s8IEKW;@>{uyieDQ>AM8HyjxU+Xc;jY1`yq~p^EN~2JdTdp zjr?`;un(N^C#EJ|k6lW~U#vdk$+k8$9und?TevlWpTP;Ww(>uVO!Qhbf36zgc0zT;y@*-c&n)Xf% zp9Gc(*@8f!E(|^9&@IV;ndO4u6c`aDkraytIXBmTt#N(?{>kYc%EWEp3I2DOI-~L& zjDOF&%23C+WE;C8C^%V`k3MTL3c!*(TyBL+P8!P%APU1F|G|sNAf>!#S!5#^uXCAs zxTIW3k7~J|)6YUu)L*DI?+v;dEMj`ddAZLC;W(rZKr!e8qEhn9%Gi6pk&(3G1+6RK z1%0sHI0i&Z#`_Zmg)ZIz=D+{7e)H721tL-pCr1D^YioEtpw7ysQ{Ui>|LMDl!N+d1 zH0OJhHVSN3xi0yXY(Y0aUH(wnqvUP^BGJjvs7fE3xj%nrHyo48)oXS#-7E0;@BX4k zB09{P%2&22^?7LUPhHxy-J8U0(BD7-9iFhl$7#)+W0|xgd@5gVh7SfJ=6ODKl~-4$ zv$*p>atySO@6=m#C3KT}t(326x=2$i(3Z!mA43QFp4kt*CVa?9>sjIcaOe8s#-I01 zZHH{9EX>G6Agkqf?H;CKmuUlwTUn}-C`2uzSSCUfS*z)Mh$Hs`lk?bulvs{c`HX1oKD>N#+7KBRlJGEmm9@P!4?n`GXKU@LJ`D7JqdU6#||Pcfm>= zsAk}EFNw#s`{exb`_R3vJ}CRUVc~$r2bG3`XLO1 zTrK>Mjsdlf+D0DMUL1IxEfHOep?#v-JZ-xR(qeHh)XFW6!Lp7l6b%1S+o*JSSEd&C zX+Mz`|3HWvRCki#M_N+ihhnSC0OB1x@TbJck9EY=xxPa+fJFz$K6>i5_!zGUF^znq2gyWyAgxq|^1 zj2~6k?PW#t$wh6^D39)VoVV)xNrZeSod&yKWt#OTnYBuPZNl7=6xtC(o4cu1Kkgjjls%e;Z+^pQm zRF)MLNmEO7=Z#~vcY-?Dw$_4!VjGb#J&PRNYwMpFcr2cu$D!4# zhlgGkGq7n^lSr%6u85rx*S%O+#4~OA&i4FV8kkP9K<>8}L=1*n3`?a$6d#BDV7Pk+ zHQix;@Ey?vi|Kp3rQaqMVaxwL1z)Jflomm%2h%eooD94VV}oVug*eVFc=NW&sgP?0 zWlzQVPmK~Rrin(o8jEvV3vPPnz8R|t*V|B88+~t9#2oOy%PT!%16Yf%XhaL_t^BzH zn39JB3Lnpb!oB3TsOb)AI&NoN{hq?@Rfd~@Cm=Krqw+_4f{w@dOi>MAq-D9e)tHTw z#lX(C-mcKT+}OU%n4%ojyztB-19pzs`W26%$_DVB;*|Kc5fG=e&S@@2YLN%Lo)!T$ zdJCXHF3h#{g)XjoSaha_s!U=;XX&YF*$20r_Yx2{2jG{VV9pV%O{7Wq9!b_-)$z48 zue?{>FoE+2G@~n;SeO|xh#K;QT-@9}n!duA&L}Xz&Gv-9c6%4h==Jy16{T+=gXOJ8 zCcm%GY6#RD`P#K3;cZkY3P#M^s=j)wM8l|Shr<@iy8J&`UyV{9*Cr0Pa9VwREDk@L zpWbAWvBb7H5I~XEyiiE%6f1xKOT-XF2VKpSm?daxR?(|;+j*DFNicD4Xf0D)w^yh<9n+ys;9$;zdkA?aB}9Th9Ve0m}oo8<0b~&{uqx&g(dm)J~Z9$ z<5e5{6H({0XaOxyVZ)0vB`OM}8|6!*K}ca=qoA)F2*8FSY46zA@wwK_oBSl2G#~6&Q~%T-Y`uJ=9m4P=l#rcfiH~@2c_G9 zosrKJ&*W$H7mNF70``$jhy3E!>lsa6hAtVzlqYK-07^`yj9m~5s@3WN8LcwNl;^!x zc`1wb0aDNV@1qR%dpJZ`JbJ& zzkw@k+$eR5dX9NQDt1jLW7QGn$-hfjzU48n33pjY9@Ki~o-Y1S1&8tr% zmfrfaT;BCO(0xEe!ff$1C(H4qS-}^fWjahuek~HKsC+t7xu5O&x`)=t%*5sAaOON( z^qAY%X0z^3?pj0@t5Z-@+womfwv7xhrXNN?+4p{I!#XWYi|rZ0qCIB?6)H>H9od$i zhX(jtsud5*`WG*)-`_2)Wo4e}w53gqKbNLvOwjI_>_cU|W7LG#SXU02al9s3bJ@rH zR_T198x9}-B*3)of@7@@#w2^@dZTp|>m+rV1oMI4?|Vy4B9PY|r86hj;f{y7G+Sl` z`s%=StcPC%yz_UA@QkOwEP0ThJl;nPiVdMo?&9yCA0Nu51VThx1QJ36CCns%mt%`z!e&>HwRQuHKSf$dt;#zj=LkQ z6U)y-?T-=pdx7$dTkQ;y!+|MhP?j%Yv3B#@NnSfWZCl1({-hlMJ)8$PodkrvD zx?9|rMofEq*7~>69*FldJuvIEUk8c}rnaH%k>3@nZ9u|H^NNQzD)91Di#q3Y+;KZj zNqYy@x?ho&AKKkzOmgoMPoM8TF0~XmoZnGj%!Lv}LY>ACQYT(Hy!fwV#>(ph%c zgV%ef)v|5GlJ99n5*CN-%zIg!yy-b#Cx&wGv~St2)r#^vcbzDlZ8BG?LW46E9`C8* z#9ji+j+A=Eu4k)&7_4~6%lutMOO>x0D8rEesNa0x;P(dR9piF*XfF)CumpTOz<>VY z^H5i2eiLT+#66xoCExboSX3dlGR2Q;iGG4otd>#9*S%Ry9EDp_%JDhNqsFw}*ka3C z?5|#PhOsAw04y50pL9aWG6KXV3X|skC@hk}-LoWVYB+$b?HK zMZOL~@L*dU@_{S+l7R;nV#RA>QQx&~^D&#h+svwh@AY~p$q7I7pQ%Wk5M3sd zVMt*diQzXnIQQ#=E`}M)1uNt8uM5KT`>|!1;IEXp-v*LeyHXLoJbe@^1cg) zG~kK1RJtLd5(!lDz_?STW2M_pjR36kj@vbE9&*&Un~h}~KuLB0@`KhaH@gxzv-6gR zdV756sA7OCt*&N{iH13`_GcpNCXMzv`-{+f``5d%{+I1~WZAyw5xQ>WDTfPh+LZ0S?Af|}Jy|t*8Ey0FdzyVUP;IAMR_yaH z&WxU)Ex!4x+c$%yz|jWc!lRBq&$zm6fK{x2&2Ic%IFj~p@D%2NrX9F;c8{=FKtj-x zoE6_Z;#Rb#>#CO%{>3}5mA!K@)l%%I;YIW2(rA>4Z>MEf-ad7XIaWhl{s84zSc|E7 z8wSSl20;tkq5`ua2oVj__-uDDU?!920Y^CR- zO04zt;kL^r38CP?dwYYnn%G+)-0*hqAv=Nhf$+4bVApLQUn`Ea z7DOn#!DsdPtA(Av7Qc0!wX?PMGOe0v%_^Wo3S~8v@YO8C<8Q0gbAOqCl!2vK3x(;? z zPXUY;?F+^jUb;Zn!h%`wv?$88emTm&YRC_*JUnHY*nJ*-d5rP8NXWTrq{+oj20CS? z?dnm~RyvXVV#aZT4ASf4i2`ZautFUd$B`q`Z1>($fkSC8WQbct^z+qkejE{51Pcro z(sU20(qqY%r;C(!D-lM7CVt;Vh&g@zGv=Dh*PD9Xd~)$b58tY$7Z-3t57!#5R7B`1 z+TA@@-5qhNtnu;w50DheF0awYuZF)K(AI3m}37f_=z3$f#oD6M*-7S-r0D}y2^hH!_3wq`QgHF=VfLSzoB7F!usGqF+*%; zy0i(I*1I^v+F;0$CHUr1y!42{5SHlTso_2#q zhAv-I9r=L)W!{Or0-L654UALy1Iw5;mqVTwe(3#?!@sTvu*&g-sq>fMfb@#qQl`om zQ|;ZWiyEP0^*GgLVMlU5k2+Q;?b42$PeyhMxq4T~Clt(JP0uqh$Y8aM=}#vewQ0Fp zat76`Y#Jp?xU}{0rwUD-rN$mR*e=_p1jUwXk>E>-4*oYyp#bQURY>4xMfrEf)ykw@ zS1%)x4F)_;HX||yhV+jX&ptAca{6kyL-^Lb##_aN6kL4R2I_tWnnV$Ag*msD!MZF^ zNj;=O(e9Td`I`@5Sy(r{F{SU8hieK{bZUo-p9`*|xc)d(*^hl9{=i7gG$S#eTDW_w zu=t!#<8HC|Sv=oCq-i(|i^?O?Vj1XO#IYsxBUHLmv>d`fvUYz$R0bqS4y4{474sqv zytXu3fE*4N(a0a4%;*I;L7$v0pCJCBpb&a>K7 zuZzi1!wqv%X4s!Qbx9u8zzHSfa_Y~si||3nV4k0xQ{=u1EzM!EnAIso*T=j!O(6{? z`)t(gW&$H`HVAxWh5T8aGD4r~_{3|a2YS>;lJ0lUiP(EH)am4bRd!c2`rZy{e za(0$esSS-uRs+1FgIHKmaAcA{3wD;k;_^@a{V2i%H4A~EteZ*q)wuHq?*p6$YWGOL za@m5l%HOC=MuaS`>~BV)UO-W82Tn?`>gtE#CH>&8vg0r$K=E)2l_%qUNI)4(^_veCH_e3K6NJHWdMgHrJ1ooD7J!Ws*~R4(WlxZ zY3X*5B^tv9F8}szAKwj=`{gkE zJH;mdANu;gNMTm(x=;(*|Bth`0IDO~+J>#sURHK$H@?_Rz4TF-ixY|*ay6OU^yL0MtF%rAmjq+P;U z%)VA;Jt%M?m#z29C$BS3AjiLKLK!np4c2l>BHd_$Ngv=4^erlBuuLNv~aZ|*CB7D(Nz?`mjwEk%P@=}neD zDzli1p#uB9LQ8ZM7aqyF!N}?^B(-}{V)!+tV=)hGmtYabg@xkRxMQKi`}gDTYQ1}Q zS^vUVtLKwaYg=>8LV(+4Z@!YZ9{GdXb6C|@KZJN=Lvx|XX>mpS#J+yM#fArs2>bv_ z)&{m!g=uDbCf&YRElbr@H|dSVWdj=0IEKYAibD_Qq+d+(lbT2NQHOagCo)g9ebiF( zKHTft5%#4+;b1htZOw#XUGVb-&GDJ5Q`Yj`$8*;{3L5V*r%WbY}noE^*=)RhSp6XL*y^P8a=l~l3qzhP-! z^D?{3VP-8?#rg8pAl9He{sXeQO)8S>q$lrFw>2hfbuH!#347DT;-B}UxH!}k7l4=l|U&^MpA1`b?RA!4N=xNJBi zhro8Qk^v-narZ#TzdcXI<*0f#5y-bY;9v8j_it!I(R;+shyu7=&CHSyH3BADG|=U{ zUN8jV0F=kR7kaJj>1Pgvn9RykgYZl%R$_l$rZn{Nq(o&~<6+Qv>nM zm6pK{(>_V&$|a3?S1K}%jLEdJgq+dPhCkC< zP9q;&jt=BY=Zg<(#+_fCU6NzKb1uP>=qz3y zJ?%<{EA8|S$?f;Iui41K(aOB|B|>(CDOVCj3bqtf4`zZ9$LyB92Ls8;X*!4Y)hWM2 zg=^wd2U^p40$l|{%y1kW4Gn?tlHqjJ+13nlFf-8Jd-Mvx6(eZc!;6*gZ$@Ci8%gJ$gRWQ&;Cpe>XnLZy7;AmM*=Z>yCg5#CqK{Dt3N{`1 z-ZQD~AQ4jAN{D_k9$729M7}IunYMw>i$EW-fmSqKY52y!=Cd}t3aDlS5ZvRg>)leZ zeKrb%1JM2?vxLILR8dh8XD^&sq%P+b{F}ECjj0P|oTVit(u$>;w7~w&F~Ej-0t>33 zLSQdhMx8ELY!;Jf$wcOUlSTjGP`w0Jjytn+!)K(X9H4m_v0aRV)@4x z(*kVDZmnA+Mi>Ip7m5Iw5ow?U%GaqJyG62ht3WXdj@&Ye@+rQFxjBoFLe_hcEW*%y z4L^)NvCQCxlsUqT;nQYt8A%`wkFoP zD>~eNb2Z8mZW235N2e8HY-Cx#rcEwdU20X+oGwy92{n$O1justTe*-7`0>gha7ewr zn}|qlMkbN1fHD{|6i+A=Zfl)ck4c>(%#Vwe9CF($)#;%pMiN zG;eWcmPub5d74+7eMWWMTo<%y$|o+m535NL$P>21ZF;m?e`_*0Z8t=%k{$vDfBANf z7nm$s5CFdE`r?^;g67JXZD3dc;+m7s`rgcAC1MH$i1Z!A6Nbj^K*ICu#pIzsf4sz} zrd=Jy6hvL;ys{|C2TM!utF|9Ws!#Zy&-v0S@u9>(87=$csg4xf;SI`E~5QxAdgVk1#Z^B4Hfz+M@- z^rR9N$sAe+Z`5RTPfR*jY3@M5*riAF>wRX}Pj`f-gQtfX>wN*?`gtEYvWn7Y6!v;? z3`6~)l0W?ru3TXuD6VRDh4V)HQb&slB|DZZu5IR#9)YcU{0o)Xc4_~!X!V!Int{qh z>dN!1+q=HwE~=G=$u(CrdYLwLIqLHx>*2TL;fz#_o_W^j6$0G~lBCSTr>}G=2fj~? zM`A#i%E0!Dq5JM_PVNqj_qfcvWvR37N$scZf11(YdDIZqK+KXiDK>|tiP|Yp4XkBw zX<0AZh~hVtP*e6%8#6aGVx-7&ANU4JN<_mNLuc`w$m$?Tko#=HJtmBQW3@<7YH`)ZHk=-U#y8$27O)^4ZV>be6B8arzLUUmKGEw!dnGr$8o)L{f4UH zxNPrXs^D+1WvI6N#a{i`bsIY&S0A+`g>$ejj;C>|#qrf$5!lGctW1iz&}K_I$+4(Q zDFScRFSI_q$lPrKXz^x|^?mR%^WDCpZ8R%$(`ZBh4$gf9%l7}}HX_~WZf#*4!D)t2 zRjE>gNe0fYPfN%$Lgk3D|APXG?ooG8q<5(mCLlB^iht?_u2lPfXu|h5+QTqC+K-NN zv}DS$dRn(3^c1lPlmjC$CY04R@HCHkUGE=~G$m+19WlnZ+=w-CtzmxTXS7%nVH&|I zv?|{eav`_&Bu?Z6!CvZ9BpnxUY;O`$hLZxFosA>+iaKHZ0OPw@Jj1P^-a^?rqq}lI zw8>1M*u3Vj{AtgVMun#7hTTQb=aA27>i38{4Liq;Cs#LRPcHybw>Ol*X7p7;DH)0Y z1WJhPB1RN;60qg{di+-djS_TxsE$*7{+?G;UdNVV*`B*8j#p{;<)bIYHSl^Bs-Ia^ zOA;6rAndiz*uFsx|6)*+V9+iwHI1-`x$a-YDYwl^A6d;&w;=VJN`5asmRGcJMp`q| zDin})(S(Qnqc;Hv8m!?#+?X#IQ!zvrM?8Rj<_f0*ESlf@3Byqdk>Hq)p*Fnm;$WTo zfJ~9Bqn5;XG)dMPz+yx}1w>0yRzCCI=I17I?PQbTx>WFiw+MUk*zCfd(~OGHwe^H#IYm`)6_ekDigE5!-qhL5xKyvEp0#@ zn3zX32#N-)*5w5RRSk?=O$HvmRq&7Pi~icpXx&FI9wM6$i6=N!p^+FJP zz4m~aH=M#X3(9i=ic0R7y{d{{sNQV=vH`8u2mwTrEVBj_jyif!&Uc2R!ESN=0)U(A z;UrLjssEBQ)TWu&*W;S^F($ZG`~W*9r+RgCiVoLXX80 zb8x+>0;zB)aWbqULQyB_o8f#x@S|B`Q|E(IdFrtJ@!EMYje(Y zV4DMcJM}7NN`(;@A1;Qa$&~D1oyONfyC_N9aF zXu?S}cAnPtj2DlH(BZ>Ggw1x>9&vG;zjfU94(?#S{L+gfAz24$p84KxVuxLP?>{c|J!&pgJO+#;oJH(lwZjQgvvGF_ z4R>OOz>x3-HpRnN|J@Jr$vXR=8EPAwHKZ@+lw^AMy#(BmVQIbPNo?JUui_4(z~C;( z$K;QK^A?48{MWb{saS+_SFWYS;bC%r!0lUe3A_@0(4grbIf7?^NRwsMZd`<%yRvZ2 z;Ot<)0QREC36=tS{k&v^IA4s5-ZOhQBzit<5^mQlS{(d2(M5?!Nd?zHr^A*Wlg=r+ zVI3cd;!oN}J+7F-+#gc#|^thKI>(;s`7r3T-tvwl?_3zd5EgW08iKqgKHoH^Qg1=>2 z9Y);~fuq3&w}CYq)!y2CC=0cs*+n-xHj1<4Irp5HrW@$G<-rno#D5U*0mBEXeV^v) zx|UKB@0=a8rKc%_s|KHHY$t4zKU#Uu(fo9{Xeo#<_+&ZP5RlZHyxuM>=UZI4gbEkl zh7GZN2bO1Y?ZuSVdD(+vg6K8l{E_zUR!&a;-1iR6cE#=m9T-scPppw^S7t~Ulev~$ zg-lH>p-gx4@z)8t<09><&d&A00A%OnyfesdYf9mAWPC@i+_sbo^_}R~X_%)v&>oN0 zOHkO*&~gSB*1o&DTV7tCWLDiaWc^!24CdX!R{;c43RZWZE@%s=cf4Saov$<;ygz9@ zu~sQ#61>IFD=LZxSOeRUXTZCt02&yIOCycsV|ZFxIv=)r3u`;yM|huaZ&)knl{|$= zBIs&NT427Gbuijo_daWnILm7!tU=%u!nHHG4cjT;+%qyk8Ygf`@~zZ;imwPi_6`B2 z-VG%fWk?SC^pjH6SHy?o$unDW78->E&B`|x^kPxy==yhn@e)1gl%c~RU~PZn-edbX zdwmqCIsEKp9zOdJ(zjgv{g?pLmL@4_$YMYK^6qs*vs{D5(cRj6pYIm! z?CyQxTSdBi@sJ&%;ok(|nDNFBA?*><4^mc?DE1+zn1IyMg6UD|#&sb-=&4x3-0!p| z3mAYN&5f5R^HsiY;r$$P#d!=wxzEP7*;e6q4tt6$?lu^SL8O#VSaHjD?Z=bosd{;pDDpz1=d+=rCB z!gUmBc6MN}u@-y0CgOFUYW_SGACqgkYwb-clvo=SAZ`jG>zGO?6V=4+(A{TZ4IX#X zrW6g*h80Adcp?QqOk4sPP~R}@gaju>rlpx>R5MGA$bbo`9>%orQiS&o$Y3l1`f`!j zaj_iPu#woAGpchNU|}vy;Uobgg=GO>GEyYjpZSoBB-PDE#gQK-3%@>N{Lws+6Cq>* z=S=s^zb~FU<$^`t_^}yZSE!AsnvWpaV(POW;+-Rv;^Fyu>_COwrj9;49t<4dABFP= zqt!jm_hzY{G?N;GT7=(;!q)3f^}k1R$V!)HHD*5E$=vRFw7Rk364t5OxejnW0Ji|@zV*KuKfj4b1U|aVOFs zRUA;C{q$HjjXXV!p38%u(S>;jk<0U|t}xsP{&+Yb))F0?c-MAS(0VOE{cg$xBZj8n zO@26x&#hmd*6w&3T1WxDKB($d4^4(yr~BqkX`1rN%rv!HDE>H1mj4B~v9RP!DsBne z*t0`_b?BDm&Rc7IfVKkw*A4NnvBWPGUlicDy8U($Vli<78q3V8vXDLniq$4EFK@=` z`frn_{90HtP9wDvg2rf*wtikJVEG^ybe3g+0m|AA`@mAtNjeX6oCLKI!+-f?vu9#2 z0*A|P`{RR!je(mX!KY7RQ;El&ReGydQvT{Bw|f!&IrbU(aw|)%5-W49i@=8I7HUqZ z@5fAGUmPPA2n@^O(~X5sKY&)t$10aXpL5QuZNMbf$g@9w$cgZV$9pWpLS9Cmp3aL` zBM0^a+~$|67BUUY*A%!m^C`g;z|!aX-?^ z6Q)Uf7LgRA`>F=(8X43)HTUuHV2Y_&q~`26mI#W+7`slCP#LwtusqPT zjoz-nmqw4j%bEm`&&#K64+`jen-U#PWXpnc-88;gz0^4d`YPnI4O#ugZ*x9&(?8^1 zdnoA3!8mNe%z_Ir|6ZM>Xj7|TN2;lA`IM9)lc6X<{R%E}(W>9@tf$tJq%CMdgWcY}Z#p2R`uLtcX49`_YH_^y^FMDj0z1_RA;b0*quE4zPG4ek#t1lD^7cz5_6j3Ijg!N_G0eC3Wa$hh<}m{SDY4{$S|`AZ8He+ zv>(@K3SjNUT9><2oU6;|W{Ki6dPE~k`M+ic=Eg3X5vLsO@R#achhM?6L7)ZA@;7W= z(!qJO^652_z6+@D#y)0sW8{lP+X}d(0=1c92^Yi}ip&jH@lR49tVX+z&$nB_&Zms( zp;3QGjA#IksNo{lvT5t59EO2GeN{0{IESdanp~MBXj3hWV{5Fy@fph%DF%So9T^`VY30AEQKrsrL>K(x} zLFEYg?3K;o%(+(>SS8k)w6z9Oy(aZkf2EqGu~a}EaVm>G&$Kc8%kl0wMhVvB`sF$= zA0ixBqQD(onJ3|e#!-7^d!1)WliPH1(%yl?NJdfi%ITV#J_k%|;;rMc{F377fH_HE z-`EeI;M=`!hqAK$nSDd@j|A)GO)P4*Fhp|*Y^Zv>YsA~~p`;z`CANxTF zs3VEzkUFxk%XO9~swCTPhz0d4Ue}=8fAs4SQ`5K2EaIw?S>Y&ClKy#bRCdmy)=O&} zCnTseFN$V`2h77zF0%1S;w*BPo8h>SM5ggc(|gxMdhH*2`8VMykvR$lSqM0K&(a^~ zHmIBT4`k(PzbMdk>dre&Uyt8>Bg{Y{u~|PAeeqQ&yrSL_keFAXlbR@?!N6IJ$wx&7 zD>cb5>>G8OQ*t7VSBK!6PmHZlT$-EjC!>4n6yHaD*A5ECPVI7lJx@!kopQHtp#+rM z%cmFRX8De7n>HjbfC879UckU;KHXOzU_Lz~J`vBSVUt)#@C?N>!^3qg8?g$lJ5)D* zLc9!YKym7{NMteWtBZh;^M6T!bAixgQufuJrTVf(pn{9sYAaYY9SMeBl)9hb#8q?+*3n*DrOoDa|o{_x9jTx2} zfE@p0l)&FoC@b5iqyxT9dk(A@EB9jP$l!iF3}0({a7JptnylV%aVhvN-DYR%RTvc6 z$U_Gg>`-JzcL8Ct3bpPfWh(M<+lpN?y%LeVj3wDHkv-pqiE~?WQzL;qAkR;n=ol_} z!bYc08#A(DmP#v20HvgFA*(97j!HB(Q-FQb4`ZP2AzihyM)ni)WGs4@s@pv7mooR~ zU&QmW)@^yZLGKa9jX;VXK+0rB`N1I*9Q${$6w;8qQc_&KY5!9l%2bqp2ha7*5 z{a;V;@4xJzHc4!PL&#EBy(86{l`q2KSXl0eO30C_M3L2?`-Q)Hz6XHBo*eq55wFGcvp zCn>+Kb37g8Ivp}a=2|>&ejF<7rnimuJgS907=pIpMS}y}MCTyUKsvzW%)D7EKLE0n z{`XTZuD!Jep;Cm*i`tGuGqbXsU44)A7G0iNW{T5Etdu)5@AKKjXkc&zo@IteL}(AB zXNtPt&!tDda5eIK)55K9BD*zq4s~xthfQ_lBeg#b?}e~?TLrn)yF+Od=bCn_795cC z&?%t|zpVYTG=5Rr!c#7mMaC|bI3U}_CriM9Eo9&_d^lb5U@~-~HK8Z#qxsjI$_x_? zZP$zBpjPsvP`pSfQJO6=!L^L0LaE%A6=vHO-3eK)7}R@Vh$mRtrb|)KAmEe#sZIZW zKm-A0kyy`PgU9tg1pf$i&eh0qzF00vEtHTU=OqrAt!jxva(t1oF47gPV1oP|BE!~@ zUvH*gKQ9Fy6-(!XJS|~CA*v9zXcQuU@nE!`E?PfpL#WZh2TUu#+H%%P?FcFyXSPSvWmpfESOEKJBH$B(7=EsurH@m*XIjw-13f&u~1~_7W>W#l2 z@QcCfkx~AUwdcR;MoxCbl|O!Vr`moobTu4VLQR*8L=(H$={Jbs4u)_#WxP|6!c}Uw zl~5uu<_mHyD^VJvEiH_>yP9Dd%v=wVLq4xUiS}sdL@^@XXn=13kku#x47J9jP30>c z`WHaL3jqo~F*4zp)IK?3>IN0zXHkbo$t&ov?&BL@N!Y?js7R;&K;q6Y4f>oH;&3j^Q;%u8>>zBm8y-d>C%85By0%e+tiS;q)1 zvGHm^2nqRVqyH5y`kjB?lDMyM53>kqarnYqKx|B;4rl}=Rbysu<%2)2N%lwIrOreW z^vUuZzdV=tFtL`cU^`u#kOMuXxYrH%6ZxnOC=L+|Eaa2q*9|ipG=Plhp@4MnkcMe- z6qD(E5;@Xs-EnaeK3c)RYLRytCK4BJeB7>OTe-PuOf$!+QQ!NG{mvgk(R=lhMCL5` zteGh)_2a*W24ia_wo1Ob0-s3NM_Z!(+Q#4ebmVYCxf=B?3Kn7P?#=1u-8$JB!&IEB zA`kU1s+Vkqe_Mho6fxXrvex=1rtt}lt3LtN2;7CyyG0l+PXUK7DIi-kJRmsruT%J| zP}wanoa*h5BaIGN?OqUT+7yC^Mlz#+jWQy>_ zL%siT0U5xNnLhFX^YE{+0tftu0R4q~-y2U#_8$ZHAGP^!llPx*KoBoFXpM*?J;?HS zmHbG<9X^CBUF#6?_v`-s9)COTdJVvjavL&1?$BXC5x^9tMJ2}WpqvB@|MQyo+o>eRQ2pDCA(A9KpEBb)~;pqI{*_~OMVypHxd0K?kcN~(o2n*lnzrH5{+}f80|J|*@P~b$ z^fw^$_i+|NamqNTE&IF>lz*QgT`0iq80_ddF#i7Qe;<4OBp;R3_ApudUzrzHVkf1m?|C80{K$kWXe*M$997J5S$ z?_Y!a*MW~0zs}pMc8&k~`mcYX_}8kNx%54yVwUL5Ex^=l(f=)-f2}AlBht3MMI$bq zL%QF`_Af#6tD~~#U0Qlx&Cpo07$E&V(2W?NLdl8e43F&}Pn|;fx&L*4E006@km|P| zKnVa+NBLDp{oA*H4f%f$=785!tU)%9^nX3!|L+}in^Jth9}2o|hO_uZ1fX(^GNp`h=Pz_$} zl(ONesJl??ZofSm$Pyypl*cV}<((IJV$0rS<(odp!nE23>e)Y(w*nMnl!D&Ynw##s zk%5@tEj@9GPq7zSWa6L1qH9W|QyPM272UPwm&kzXavSB`OesX0irn5QkGkL07JEcn zSrHzGveIT(Nx95_z0I#gz|2Gl*EPr5eC;>zN|$0_znSYJZCiZO!*gHrl)xDO$(kEfP7ju0AUbB)W6yAGJ5z48-!3p#rzpf}+5DV;t>P zQTipTrds7{`tz;2Km4E5i6|9L^27Vk-F4f;-{gmK{jxwGm5AuU`UAYT4uX{W02m+w z7%!L^*jivRfz%)XMt!RX`#hV7lt~XEd&Hl9llndsKjedJteuz`?ZxOYp&P(RFmqim)VL&G z%H*=qF{IBG4?3+Z=7Jm^Q&Y49LlOQf%Gvp9YTp;80e3wF0m=Z^+2#P|Y-&czKxx;7 z48E3hQwqwnI@s5ea|m@i=_u558quD8|Kj*UiT|keyUZd|Ti^NXVjm!=zS!;g;&QOB#s^l~kdtz#Ok8Mk@KB3KqM>C=Db&{5+hdoCZEF6PuAqEV zZYBBWuu-fbMqCpKtjP}lWn#W*Kmi$HqvLx4lnB@qaYmdCSdJFurhR7qP#b|@1p7-Z z4GX~xz zsXJ7i=ZoAxauC_73RmCDy=L2d{7?J>z4zxAsQ|dv(`^_y^VA`1a?)**7%Y#xF5i5L z(8J4WUt^uAvOIw&$+Q#;wGj$7LFTD@`ifCfA@ko<{x_(PqHD4BLHez)$v{r*ydnZ7PU+GX*qKed_MeF%&<-JtDAy2KqFk3*ZkI}7GI92Dt zjTZk|nM_GK@q<(LUw*Jp7}!K|23da^YYF_SbUk2gyz(esbfcDL9rxkvaUEr=Y4`To z6A-$-zqC|oXbm%2JLHC&=G}H+0s$Lf> zheLzq0p38m;SkRYsN!EjjW4yYFA`~g3Dx<}G*OvrXn>!7U%MYmjN3|$QR*8TGLtZG zgT&VCXe^XgUI?X$;x9t`w(AB2U*rZIlnp{9$8p6ZeG^p5IrTz={!}1%GTJs@%Qx@) zkN};=?hWRX=BeUQzM5E*en@Mw|7NebOVF~2@e}c@%F_ByQ4{Fzm^n$`Y_Am5*b+3; zXd?Nu;C@KX#bLe2nrcAyB_6&N#_5N*zKO86goPk}QSu`c0DhbA^=}5qG(b3H^zTMp zE)LRGv~bI;4!%!KjavvEKOOigeFBWTX-&S;5`ugai*F!JHFIw-)$4Pk@_V@bK^?@C14-A* zhXp+@l!-Xz=XZt1<}-=)YVVH-X^_Q{H695EQ`VthVtLsd-XVGR+*|2ZxL%NL8o$A4 z{%|N1Yu84RB~F1ld&F0v(}0DP1e%GcSKfSrAhSy3kL8ph9d=^(QNy}`3h8LpAJI|C zBJcV`Y~+C0>U(?Sc<#?(Ot(|KUGb-Lo%Rt3GDy)1*twK?wvZ#fCZf%VV{Isg3V#oM zWv@hp7CbUpuBR{I@Tq{vU6lE4T^a|{f}^_LdWQc@T$^?fn|{o2h&wLq8TyD4rK`*P zY8@8&S@}&Q6^_^u@|ExoU3;Gy<%)g&wc_Td^Ko7R(Aoow(4;x?C$XQculfGV;SJ}X zgF^tK@qbzwx^a@!Y&YX}#;+D>WuCNkf-vZ9=RRE1^J0IepJ>h&ntU=NYRil*r}HN5yVa!-3*fB% zF=HLe>0MW(aX|U}^AaogJi|0KW)?7O3E6CnOo4v5=D4901S8-#%d^$hpX_a;ngF`o zERUV)$VK|cU)twSUyHvEc8T3qQ%H!Yax)2NBB9Rs)r3ExV(8fWAcCJ6weITz9w|SW zygNbO=#SPcdxsOZb;2H|5V1?|d%(c{k_230!w<;^zHOC@e9<<=a$XSo<2bi_`bMZ} zJS9J5Kj8CCqi`um27bIDg=YYtR9=2s$ge~IKh4y|JnIMz%gRXqK(hIg;;|coV;M}| z|Gn2R^)-3CtSbgI!}XE3>lTnu_!(sSdQVyY^N&XDL!pG+`wxS|=kMrU51^oUC4YM6 zTeJcp{&Evt*QD`hHsnLaU4LlHq~0aB!iXt-9BMeZQU!eB!-~!QDQ)2<-=8p)0R(L2 zj}-xvTMN{cW_xH09b1FPF6-7D1sk69I~TSAm^px*=x=Sk`Ywvs_t9!QYwulAtc!YR z0(+b)2MqS&&y$-B>nMHAJ+Vn--e1=!=ulhOE>DflBP8>7S?V+ROO#2MZf0>rC-y&H z=<&aQP-dalrd-vfAYN~~CE>xgXqu!1tPbJ`w++;%U;*N3wn+I9k zFd)JL?hJpyr89B})J6hW>au#?C=g9_vap#PdUaMAD+=H)+K119Ei~y09aKu)P1RsJmSo?jNVug- z0KD%@BTmU#BV{hcj>p?f%03nagR6iQL_I=r9DSZvyJ!aDpf=B9PlZsn?yz$aqMkrFUa;zIwjh4Qkpt5!%zjqfwEkBMSQesy zj*aDCYu&Fj2kLOp@{G-6317NkGPqmdg|4I$cY2@N+Q#*{HLoH160 zypEp(GT{FOr2mA|wy^wF*80yC6NQkfJA1vQrDtjdJz5{N?m#b^e04SByAOB0)MeWsZWM4^^W5D(b6 zB{9d%mZ8vVz@4!<8K4Ql^_7LKJ$4t;JIpntW8P;V`&swBoQB!`C}md*hjo2sT1jZb zL%07^0`JgDo`BN5w6pM|Od%^7sRTjY1t+O~@pRRo;x#7CLGMI;*D3>cPRF|umQEJ@ z_!J;>{Uq5@T+ByJiSLwt8|ZX>x!$8Mvd=K<_Tr=cCMf!2-HEKTz%5d?vA z%_O0VS$8ZgTCrl2xR{--{?-Cmjp!6zrKp?*uehDsSPhJDN^A$K@*MFNfy^E`CU!LF zuV)7mia&kF!^IAH-`_7eaF<>5is?S)tH5vVDy`FlX*IQtmx8vMN8 zIEF8wm(Nd2!Cm62^IYR{2oiCl^x0FV7a(^IBMr<%RLz~WvG&gAVLxRyOH^3Z)8V6c z5rNy|t3myKl<>~gY`z9{X^3_5wyOU1?cwHb4)6!-r<%afJaZH{L6>5%1i9~ith z(eILmr%rrQuCiYZhTN?=)EMW`f{wMOZD&1yhF~@6rs(lao*QrQA9$v{W-)nE8~k85 zAv=flCuamwg>a(3Z;udU%+(W`3iZMVxHDKfuXf{^PXfdrk%2zC^wD2jt#{v$PKL)^ z`DGcZs-zr50$Qq0`;bgRmt#Se?)hXmuA1UfTw;-@Vcq<;oqSfj!gtrkw)$cPNPkpW zKvtpvu8Hr}mRjIn!e^EMr4tM0<8S%Cg-EBP82-Kdo}0qt)kkIoW)mFG8)9B%slm;C zZ;eu*NL@c$^Pm{@(0U%Oi{_mzI8_X@d%P{bXsLRgOJs+O8LJ!CDA;xI9-dj!Gy6G^zv zOPMn+ULc?dT%uaDD#c;Et{)ZELPIyc7+KeHoO3K5VbUBSSs`<3>V-R|6wIh zrPugYG-(NwSVc>cnR!;^g!CRjydQV4qMO9=nZ;Buc+2ES5!5)jNG8&>9Rh6x6I@0U zVVd&iRvm90hicR!bOB!SIdJ}OoaD+((TSf;)!;ARO8e7 za$N`p&*^gx17kW8!%u(@z+>pBx&TeH`ScV(i%ZL)h@tPxR++IaU=3s;FugD~hu>`L zB8a~Qv4T>Z_hvriX*72ai)*L_5#wz5p|VAIPfi5G$MgI2Z2fO!IgEnaSc`~b6Q5eFFbh9|*H3Ww` z?SKxaJ;Gd2OGh!HA&CU{rth}1LdJtbj*e`)1|8v7$!Q2iLhc>~RH?OQjzmGB)wTh2 zZu%Y zp>RPxosQqzRF!L5Ors(9EeOIRf-m`pP96yAIegcpJ)S4VacLO!BQ=g%2~#qe+L+lC z1-g=V`swB?ROAzNV)f8h=1t{MzHW!?@UJzU#%ZPmCLTG=6vvzE&p=}niMsQSHU|5G z3PP5;w!CUs0Xj;SFE&*tE61cx=bY)@CR(Z+sSZ!N zqS3_Wtip34Ue}}e_^Y6J01X4h@9?Fi@wUoMQBI4MD^l4W^({?iPQvk9wQ^HM-A)fj z(MIv*dglOzj1YT1olh{@Wz%s9-~{V#+v5ri^o#8{ZKs@9fZ zh`~#Opv1DOEbFIOD3n)9*;rrbjuzfo63n?vchOk{Tc0{!MkmBhdan!gurw^wvAoKs z>l8ll#W6SR4&4h`1B{p962{#EQpf4nOM_0tht~J4bMggi8%63`QlgiPpi~))$F@8= zyaKN^@b0dTuRUZ6-8U9T3gC`$yiRjAhoDn>lK7x#AmDmjwQ-oh~Xh&mQd3eVDFD z5ChG0JokX#()w0|mDB5*?#<-&0@{{z#@yKM;Sp&=WQT%9Z+Xz+S_;#48z{sJQ^ z0|if;I~=GKAb(L^e#L+99E^u&|F3 z;d8!Zpqv3vL*(gg84QCo7*Z7qf zo03#wddDx#Ni5d7{04zZM->*Q(T`~VKa%qiY(jYvD_!aaS(&wH_*>g${GuV zBr8z)61#3oFrm(9^P8@*LdsE$S=;Y`e`J^yRO5}l@XZO#@N+b81=kt+(dNP``J6!4 z<6zEZ?D6inNR_I)S`w#Ie3+!~9K5IsSLL@y`*RnSYEy|SlB%f_WzMi09H&GB-1|V@ z)ea{7#uu*$2lQP#`04sc@%oj&iC$-?&u&g zQ<_}JufA#>lrG5AneOsR9Wijo(waMmla9)mtK^yw+ytI?J%eXQ`dIT{#WDAm@FFDJtNV`MxfcD>Uc{WsNY0 zm!8OryBSD5GGINgA91qZX*M5s-nTQSZ7v-ZaepuL9P#@8`lOjWDNON3nThpH`cUlj zPq!?+=k@(vcg}XTblZ=lmJHn$-4o7TtJWXCbsZ(DjF>SOwO%d7$>1?cS|n5|`s>tHS$3CkIoW(!$QSE;Bo>ViQ!R(B_`J^=2P9uzX`oA0E2qgK6bAy= zJ02-b6>p1^##^PPmb~JMFa-z5wG6f);Etg7y#f=dQO1jvshUEI5Ba7JHoi{EtSFN! z`kFW1f62OZ1o~*zyFF(VLfOk?_Ujy}qaoan99sxrfXS=#q#M_}XXkc0mP%yWe8nn^ zYn@M=bsB@tg#`L&no3%s@h@;!Lc@5gkgrV2B$)dj{KYkO@p@fi-s^gX6Te*t{KNX| z4jMsJ77qj5E3 z@P0Hb`yAA|=~KbaKI-$aPPppGaio!q3>dIFs7p9ymP<*U0%a*hPtk;J5%bZchWT3% zy8PHnFeMqt4=H8}g>*&Dgfb}1eYd|JUyX!G?$LN0^5b>q^WI~%yNTb;A{+NIMPgoW zkSc-Vge%d;**cpRt9-)Ufp_k9>B*uob@w5;Z{x5T%e(t7)AM9Kw<~PCN-oIO$^I(e4k!LW)s(f(`DoV;rwdpD0>@)b_GD9qi%o_69XmWc^8xv z%f$LND+t7SX_6)SRPyB_xe4(%KE&4_49RleCIk}Iy!fv9IegrzADl|i51HOwKMHN$ z1GXf7{Tc3Y*kJ#<-9qpU^Arqnt$@>WL2BcZAS4`-q{u=f`j99D+t2f5ZN!M)PGQq|KK!^exgo%d0RkyqKvi~k>EZvjwdMT0Q}L0XW7c9iBQk|l0Rgl^{_Bc{QoO_%^NhzwLp#^&LA#U>V_O=$c2&Qm-n}R2jD9}e2do6U zJTp}o=FIEA&aVf;R~K0g(Br4F+T+&p2%k>|`Fw0dS*3eVrF3&wo?Z7o&9%k`NL)2R z<2iw2;dq_M8Cz`SWE1X(`uL>7UzrS%3-vCBDd7Dwc0t0LQ>N6iV^>uTDiR!O}VC<92 z&+3vbcUDX2m5wWwVF9EDrXf?>{(O?pDYw60@L>?ftpfWM*1U&JBJ2rMm6h9-YWY^- z_fNAfXxHpTXY+Tj`B~Di$eeYL8ozz~E@Y>yV57E*Q~gR)XN$A5sc8uJ2@@11?n(?Q zTLUhVYTm`r<>LmWiI&RlH+n(Od`3Wr=8u5Lz2kwRt%SyHNt~Tsn4R7FoIabB#9=e! zfm3cXbcPd>FA_?W`z}(L6O&SV{tkhdio!1>`gFxf9NNe`@jJ;uf9(*ek-`ergFx~) zBuc<8i|LMwA;V{jfpS6N_F9v+m%63+XYl!9L=lP;BMxM`pE3khg9Hpsk%@n(He4Q3okzTj>GpIhRykjn*;MxkRZT9J0IGzx8%cr&n z2!FB~v{le@MrgH3(L!itH?74fUy5$&V|{FL*#ii(Xxb%c5s54@;pG5TIbiaR&yto$ zYDm)6&}^>pORB>mV7!x)I+%UqvM1edXkI`4R>D%* zUgH;mW)ZnFPo+cs!33QY8Q|x>r(v2z;_J9ohvj;@Gz z<7_XDWXBZ2Uwr8?yHuj&Hh2Sx;fbfl14!Es&Ay(9G%{Z4(h+X4Wt?_B@t#HSp8?Bl_Yih~3RG-he!i^c znwA*C&vnxt4dB1s4x#NAz}KN8dD+h*#9Z(=Utt_7N4lD;q2zzW=`yC`UMDYS+pqYg zVFGcKMGOU*0%)R4{&_|hZl#pzl6$?QvHE>a9PWyfdtyd><-x|n%C*Q6mGTE5}~FdDQqZZq%^us%N#R2EOxo#!!?c`u?xRAOdBHoW0++`lWcHPP! z+#iV5GP-!*=%Fu`{Eaf()N| zmDB*gP96V6jeczzqX;}teM}$VcU^36YM=i}ahK`o?(DF!0|9-#${VJ7{QXShpixpJ zJL(lF-Zz$7(;lpERb^8DI=YXky|6i%NIwbT6*SflCr5QJIm*|;A*2$xatSEATeKT{ zIqdn!8p3$6Y@1{VDyLz1kS))>L!!=`^qeuJhfUjOn{MFxWeI+2=vIke2kj~LE-l>r zGHcg!99{TeKIW+rD%!6nL*}0%Uu-z;y=Ne!<(QP8S`4mRwP|XT(p=%V8e}c*(KKnA z6N{gEHwUv*6BC}KM`t*ZaLa+RVS05(a!`e8&%?VQ0=t*JFdVUIW@8qrzaMtamx?%6 zsP48mcM&d}hlMWi159_^6o3)X+o+Y`f6bk5cGrDJQps&`H4{=BW7W#nMQ0D*+jz*z zWr9h>l0ZBpeo8kQR07ag!*khoby0w!>Y4()*swh#BnmT*ni>gT+?%RlBB2K2{qoP- z5u|T5K4qcOAI3AvZ|=3=)dtSLgj)0CZ>YdpY0+*BhQT?yCQL|A{h@sm8ZiD6ySa_7 z+6b>A`_eSl(k9p!9JajnV9S7T7`rgtLr5sq`-yOd@pYmzr4{X)b0F&CO}qEIBwG|3 z5`O2l<@--z^@XD7C-@g3t|eI;Z1qduWhC9aP68JokH9F$O_f$H54)8HG6eJ${ZDQW zuXp5*Sfj1))ynT>P?T0$4XVkI&-uuWn2J8UlR6u*|5}FyznzYfh$3u^r^p*zy&P7$ z|D=8vb>CT5uV)}u+nZ@=r#V3R*h^Qf47tcqWie&t>XUyVlJ~xOL8NWPHZ!{$vGXCy zUn`*dQ7iL_sIQTriNq=B;H8k3&!|^)DPst<273v(uh(=-jgc`Ot!uz3Nv_$UM>_Jsf{ z$A-;fNmDB{DtzLb&WXM04D;$sKen5IExJex?sFa|Il3cs9>2CiRoisgVZVO$n9f&i z32vvkDXLSaa0EmCcGk%BF$&>m`KZ-sm*c>8NU%Jjn$6vjk|K=O-QFO@+UNRR*Y7C} zKOl?2N;UXcPHNryXt_op(q;xR>(N-NE&7GBgkh?Om;ED(bO1G$2(I#%0RK6LV~&Qb zrpilfnnWY5gkI;1QXgUM;79@Z$VdU~_3=v8_#w6N+d1|;@sypv!}CHXwp$G z$fp{`qj%w0D<3Qu4JT?yyJ+nDbwWLttGyZTt_kn*HynWWIf+jJLxrPzQ@(gtuf4W0 z;Fl!dIh$?#)DeYXdn~>211h4K{0jq%WhsRM#k?kS5;`lITIWotNh(bqk&1DAt#4+$ z^~-))OLTOoASa$IcVKcu;w4h(q2jD}JuEkL5?nfePlqMbyUa_6BO)gJ<^y{NMz-S_ zZ+`e}2W`PzI9$=#x`lXgs}`4hp9K~B*(`AC!y;SS$M~(^VVGk|FQZO<;e|QpQ^y$$ z2d%F)BJEcc!#NAyIsAf73Gt3+s||33dxgiT_TUi@-azL1;l!xU$x9&=Un%;D9Hvx( zM0dePkAoNz6nQf?HM<kq=(0m7zNX^D79ZeoJkTW?5LB0p#= z*$#s?A51^oKF8?8kHIEm3PSiv)X37bMQ zu{Xuo{-%B*`Lmg=3D~l$#uDlE-tRRQ<%2~3DA`AlwC|8!P{?5_H)u;pai-VqnrT7A zW+Ipad&Dk_YkcMjhW%6wW1&cbsk=^o=m1sB*KKOkAmq>d8-4*4Ci{+w?e3BuIdkl%O z?UIM<%k!zpp?J)OVB}e~+_W_<4b4Ct`5y_S^NswPnqx%O#}ug5E#fv$vWBLL}+7leXaGy5HM;N44K*wX)&b~)gWA~TH9Av*? zJzD=)Yik)j4}bTD4V0TMpCcxMiLIS>o;QOH)^3tdkH^(VlP=$)i+3hXR@A@Q3!Yrw zK-Eu@wb%YAjpl(3(>C^oKzKF*!0n2XSs}E<(R@sa)TZszNoo1shIt3tOa7C#P&MR5 zF9$Z+{zY3bV2sPMocYd@c^REUTZ9dU#P$I<;4Bx?W zLbN9wS*X#0P#S(fbz4gUwcF~Dkx8uK=9-RGZ_ypMCX^N4j#*kq38nr_X7}6q1xN0- zK%2FCpamI_yLIpuJG?o^YZO0ga7+bIIVMsvaqQsfw>9@I|1AO9+diA$WhvJ_eD*5w zESk1FPKRiS291pPKEF0MgNNEL{EdI+&b$D8_NR*+Z!aFi+6Bq>!3e6rF=0G1q_DCWo(l+L05D7%fMHj~k=ZJNj6zou@o$bLbnp`)gearmb29hqU6EgPdUj_Z zPc43EzsJf#Gx4dk@l#;{qwvbR6nzxf*in|@){tnAeB7F{I|W*{sm0o{*k*o~juc1Z z;QgxB*d^LGL7y_r2CIa2fN8k7lQGOkPg);82zs5IHolT`p=>PdWeiuSbq;?f)PKGB zXc4ryHeT~n#?bC36Gpa4d;({Rzo|#7E#7f_;}UThfA!j0H%-6LU z9%EG}m-jXB=8ZqUJ^X}UJNgkk#zaXAmYd12L34S0d+tpS zm&L|#cCQ!O#lnq>?eBaRz-_tH2f9{hZ4E}z{r+6$)y$tstzDmHS3wy{C@P6Dx^$AK z#TEs&tokYARmbcAtakyDJacOujo$EFTxMa|I&U@Hm%ZajCS&1P;PS+jQs|Wx29%!fn6$QX>mscP;}2t29wY=N@OwCZ*N-WJFDO<*|U8uqdtm~M4z5F z5r_GyCQ>nXny$8Fm}x9cvsf?-U8tVl3cXj8DNFUUui;QwXcBg>i>(>#l5)E)^fvqm z#2jKy)q<++YphBC<{Vx<7&&8NbB!0-ln#)jQ>L_awzV{lRrmzy2B;K^sF%t);?vH( z1D0HJ#G^AYKclD}B;%;hX!2lo?B`!XrjI@ff#BM_NR7-ZVzRFeUfzcej*j%%^yjfm zSm$QsTw|Npn(YfY84ueJ<4NBEwzxhQM-UdkDH0NOd z*w++(lMPSOvmevHYj<o()=GqMGrvYyQCG$UyU^Sf>oK~$?u`*8}Gh9Dao`Vm*K^dbxE z5^VV=d=C5@KC@lYO}y15O|BMdx)`}zAH}yRbd7syzCcE}z!@=QXf~*QQ{{a4jG+71 zEE=e@`~l^R#G*4+aaNgl#h{c!N~yRyD~4VvNeAoQI3nHmOL)Ps6rUEJn%3N%h6`YX zZPHuK{MatoG(5!6)g#)>81b9lcwLjBH~o}zJ&!K5{Rtsd8IU!IqqJ0tETLubfS;}g z-9Nc|%(vD;QUaB-(@9KLLpJ1Ze$hW=i(L{=Su`b_qA+$+qq>rrn%@%RBf3f72D4~9s3UHcQY~T8> zFxNNiceG%%nPlb1LGu^F@^YkNR8IMb?w6CF4O*TIC-z z(zG5I?xR3yHm#nhOp2yabTOafrWzdBJrXkMm$ZvBZDBpHyGR!Ekht=T-L@Kxd~Y!E zHC-?}Kl?IZf&iHCR>a0>UXmmmCH!^f;A5B1jajxaT>jLFVM)9#0Lp>TAz9+uW$qRo zGFOCRuhdP1zsXZue={05PD{ba?|$R7{vc7N>2eOKZ8@;&v(?`Xm&e>MxVm-JN)N9w zmaZlXQRBlvD+|W)r}`yT@y`9IQyQ}TFBXH3gU)4I9Um>so2N^EvB_zoFJ3sfRP~y& ze(zX96yN_7br-q{^St-Sr6pu;ts(ER+yC)?W=TKigYyj!W?rM9naN~xzGd*ccFSSh zp_6uQJR77hEsJa}sq<`3{%b|8;7i`B2J$#KZNNf+wpgI1>+ZI*;-Vz+{_M3wk0#fJ zxW5M8@+VV8aUo=Cbm1APR(Z9u zM-#*Y4ozHy4tBd_))MvYDOs+x5_YGU8 z>Uf|uZXGO+>Yo^$>-Az=+)X+4e0*3Z>}^+zDB*CwA$VBTzwM52=#!ahaLb;(fmkPB z!0V37;d%SwS5&j%$F_1dGmi#W2}R|tiNipc4I7U(A#21N%KNYLrm8jqsi9rh|CNsw zBkOnk0(v%{b|T2lS#9XU3V`0cYMyEOxmYFj;*=fp81DtMe7kO_p^Xj@C`(Rf^$^Qw z3*p?+1_W5{_uWVxtaH|M(RWWRH18KAyRUMxr_bQ*l}Y^A+}K-S9myUCPhB=NrjZx^kDC!5 z<#e>99)435bdAk5Bnxb+6=(aLR=-|{$qX}iD!;qv=#d)0kH#_D*en@NzSXjtWCthD zgafk#0apBd9JTGMYBFr~0%hiGZB?D;5hsB6|HdrPSa{5Nd|BHwZJ`yMtrxc!l{+

pXcI5`nKRH!rjapI9ryX4nkxspQ6K9yGIC`mO7{D%V zx{gg9X$035Yo7!&;;mU`(6)_R3E2(zAB4n>5~U>OSV(*{>kE9<0U4WuXP z3c{f3VbU4m_b~C%mkA^`1(~Dg5rGG2l(2ZZ-#2Fx1hcY1~Ju=kA}reACap1!9FWWibMd ztGpT&YMaJ6I?<3AM){j>D^(RpW-a^<8GBoq1v}SyHC+Q-9okJ|^!K&4ykluQ>g%bb z2u-UKc)-r59iA&cB@1&`h!I{~DT$9uZsD6kHTbUV7_rfsSgJ?%Jz=?qeQkn0Gh`*b zOlRKY$s;xSqIscQ-9EXf{+$lL|EB+rOsO;3$&By6!ZIOh#xGZ#pb#gCig~%y~07QuduyG{+ckfbc~9to#%p`SoYF$TAk>(UI;wseDkx{z|9v z2mLewR|o>Tu)N;|?-{0EXjH1i@Mh_d`hub%UmVOnfZ(_?4rH>`M(rs>$`?+j`;Q_S zik*iL`}MZsb{kx8CdNQ?47Q-Vt0gM+j}*;64A0{MQK#>D4UpaHK4hrv0dV?L7o5jh z;c#Qxw`ju9@M~rS1b5E*4;p+K%-<-qjPX3je{3)9X9Kf|z4Twf2H+J?1iiSuyTNq_ zX%PZ0s}!PR*#ljBWzu4rhUgCH`jqI`#r^WqSasN_r|N9XqYP!Gbs68sO|p?#@8{#+ zS^&t&3W9LsV%~Va*53Wae?6gf__PWLe&`gieg2+`$I@G#?fX?v2FUabzMr!s0x7wG zZ3lE^00JgqPim`~QC147{tR#(`^4P(RGy4?Oei03Q&gvW_qr25ril@trVMt_bqO>( z*Xrz#<(<8Xr2d2Js18g+4`Y1HQ@7tG-amRjaiY-_?rUtpQpIYg#t4jicMhkr=3msy zZaZML<8xHmT4c=jWIUw>?Cg*7LRR(vcxkc^JLVoWew3CR4)RC7b?M zZM-~>ju3fg@XOn>++|m~mkTA+q?!-?zD@n8!dilSSWX>Wn}F(9O^Zi6jy`;LUKhsL z!#v*n-tt(rHVZB6smZ9&^&~l5O9lSJ@tMrmiz`;Yc5Q?o?o7g z`R`*CSqNg74#|m)bjiy4*1yC( zSc5X7BS?;^w>9`fO{C#uVbgOY0tai{Ii6l)B*zHX6lCyy>rIhk0vDH9I{l(y=JflG z5U0uR>B1;V|7P5*t(8aMIhDR%R=fFTR_|nw83=;pov-P_EDI`6QtqS(2^rVDYtI{v zXyJ9S5quh~T5ONmb0Jjdk3DpoxolT@3R=KimdZsxOWZ6a$|IA@>(wH-;FlAup`t|} zVtQ+Qv|HA|g)s&VS**GehH*I;zVY3ECnUQKGOoy%&mP1D;7YUK7`wK>$)e zD;}X_<#LeWD$0_BotuPY3J91ZSW|-N7;E?{JS`Cm^gM1iZ(p?Q1<106|JE=#Mtczg z;wc5igD?!KK&AF&cBv3L5quy390mj2DI3cPRlYbO@W zoO22vB%7de&_~A_WPo7Xx7XtjD294BGh$WopW~CZekOKT5$vx@9VIokj5a`~)wr!) z#0qf6*UWG!07vlh3&?iv5!er2(3sv3Z7uhEb&3}+oR;}jSom5v_ka`_0+&n@4%ge3 zHl!6Z9r-U7KJ^K|Gb3z+6s$&vAU6U7^> zgUWa5gC;NnKX8$n+LxxjF|JRt(v`${oPB>t9L>5wt7r!3Ak+oZ7}%p=sn;3BgQy>} z9TwU6IfU}pOY1G9=$+-$I=xhi85$i3*se?ju>QJ|wMuFq%(J2(ly8^oPG4Uj5B9umKrpbBa8AtUG- zIg`0pMXhHc~{IOCPTGG@6s=z1vq zd>RL&uS#Qt+47i+bSEPpPIn`ry1Gvweg`Z%ULOP5aQ!G@G;RDi&)j8><}{VIQ19=& z?`4-ujb$#4RZ>#>%n9$Hro2SoH^1wx(9jG)EI&%kvbin2RnTVc>Me7yP7dDAk9n$fK)zTw%k*#rTV%Bra6J z3+`ZA9)gs1*es`NpVLJKot||J2J%Kx(`9j+C7t!Cc&R^Z-0t*Zk^xcRol7cT#a%#b z3M!~VDO4qa*F(!eqHaCgMT{1rO7k$PxX79R!}OHU!8!VEr0PL)+E)q8%ApgXzc38n zvrM1bSzeRJ5FG}$NP}Y2gKc4C>ido?>D^b&Mqxt`+1j_O&-~~cDjpiKDG}%HZ~~R| z*X{M9mRx_e?3}B)XNI)aC8h{=- z0V$#Mqa!{l>M>D$505JlD|hHmIDLQ~1&rW+i%N;y$x#8)%tS9NSAHQmAI-@bdL*hB zVH`%3(I$E)E*vE^jwoQG;LdIEttqSC3F~CLyphC`)x{N(z7G@T3(>EqZ1uJS0g3%Hvrq*_ivM_z5rp^m}#6J9i z3zj{T>8I*u0g5!Bx%s!6jyvya#hcHa3Bv3OROBKTQDcl=6XiC_ByF@+; zjF$C^Sj0A|M}SX-8%Im!U*A+SJkqaoFx5WXd#MYP_#N}zCv05xH5a&TypbM@phB2~ zPlZTjkn9`3+uv~fnD-qQ?J&||FVSnkdn7$Y!#?&0>Fg_li9+2s@B%hK`uEZRXdD{BDJVC4Wq zaZI?0hI#!NY3sC60D)SZtD;Xh@$SVd>?o`7HtzNez_IF#SyW%S*+0=MtFaDMQC(7w zZ}ynmadH%*Vr0SilX=*Hxn<&$O!hvzPV34agWSV?EPIC!!MRQgAk!Nqx%>VZKc0K5O5(B))KFKE5T+Lvz}=8I|U7#7*9*3+!?4JEFm9(F@;*8F&lx?2Ok`03vdk2#SNF#AQ#L zdDmlgm+BPUS(JV8I>|vbU-wDr2Z&0-YQ4v^*&^FxEz9j=Je4pKb7t3r7}74G!!O#ce2rf(RybOPPpl`(Tv_ofc;`!SOm}t=A!pV^@28 zcxg^WK5B2Z+%jrgG4;_EvY$%uQa$#AswtzQ)~l?dY5R@Y?shinw0=CC9VDzPf~tE4 zR;`uOH1hH8$!&kC!f+^{%NSH{W-hqz`l=D@oKno6H5F~O%A}He;3$NYhTsL2m9-a6 zU7i58GILlF3~H-8M_|SDn&g0$>xq@kogT|2M?6$bm=Jw&MJUSvja%QD_b=o@mI}}J z%gKI;{^t1pweW**&2;hnQ{0AI4N&}Mw6kBcbN&8ew@yA?l6+OFO50sg?((}f3vD2- z=%^0c%dfv$(Z!qc+`L9<0s2RX-lo-{Bsi}SP94!Lhs;P^EXe0#|1{K+YpztSzWWj{ zG67HE@#55Fy+1#>JP9Y4lR?u)dGTF3bJ8GlMn2zq_i2bmbnV6unIDSFH7@9#+SkXw z2I@56LVlf8GfF(Q2139dXRVYLD^4B3jH&Q1m#ZcYSOCnA_B16;t0BM&pX)n@6~e>A zJyJ8uEG%F&`=_BD?*Afwf+%?_rNM6zD+{+WR@!b7h-4y$9fO<~bv1p}KD?98PZgx? z-hx*g5_sn9_6W$bFVdPfwXT#7WBH&-G_Uf=rr%oUoL3v%f>3;%%MR2aM+03IF}&^k zoz0})(aJC4PYGHp7}xQrWhn_j3gx1@9s7#Voi4N3SM)dCGAk5CDDn9EQb3UtP|yoZ zt!HG5S6xCuEwn)K!g_2YwINSCB;xu`edJljqINy64ndapr~zv^f#eH?`&kpEQc)q8 z96R|+_zFGAU$y0z$%3b?ZpsqtZAXns=Nv0C`AHv(eO;VMNQEG98gy+jUtl4~6Ts$w zY8S&b+?W|^9Z&N|@19tz73Qui<>Nxqe@A577uek4$ta6!Qb^L}h(Z~cZ=sImtGhee z``Wddan$SecNiD;(DU)eS45~%;eOa00z0i`M9prESH&zS2r&Z9$K<*5Mo;!*j*2`atCG3Q9UN5e$Ybd!mi%NH z7yZlO4_je=Hrm)p?Ftg7c$`#|Hx?#M7&;?|+&$e8AAR7a-Mtg(JG->EQ@Ppp*8g&Z zCM^niBGJN)o9&p@QU#{)4A^EUa_vHtW^Ve`GsQIDi~oYFhr*XCW*HK#gayUSHW%bB zV9^$Z*)MJ^kFN#^wWJhmf^K*`kN4hE-SDojHE6sAC)5Xd4VclGa?d>qG~S^y8u=swH9AYT zf@9{*Uy{;q5?_8Pk}!?vOU;1eJNbA?NK@EQmvzt{_cSQmEs2I?;sKm>ZIBq7t-Mo5 zdW}On9c3*g+Ra&P!TLWn)Cl^=&*p(euo49_Ohg(LN^JOOd(HEu%5Z4KDyinW3na;a z-ra91`i~251Tx5pXJ!|n@fv5NAo(d};+mMmX9Xu?^LlODiNmm7XHStNN z>Nv&2RaIzKwq-+eE-o?G^}Pl<>$Vju-f?^>*Bq2jkR@~l5F!4!Us$@3rtIpr))C2)Y(QSbY||+n~SM&BN6Ls$6~2lZYWL4yYWO-#uY}^oeR! z2|_=9j?BH%67b}(ko;=3lmO4t-Q#_c? z9h)pItGOB-M1*9Hf!@mg-PX%Tj_MvWV07&<`~{4Wz;6Nq$(vS*Li#}sXo09)nzY2nf$W#gQlvTTrs@QA^U~Z=l;=6;hQ#b>0b5QBoDn?B7XCguej@s?$%cy zl${(?OgzwB14*AF28=8Xk$zimVY;aA>O;}gOQpfnk&0PNG3je0=cT7B zNfSLFX~|C)xX=&y2h!f9qB^30#%brxc1&#5L1jW-7rxf(o+|5?^f&O_5 zN1tYnwO(#x3DvesUReKu(UY+5t%Njzngz@+p9SR}+giEHvNlV&*!8pY)gs=vaXUf6 zLNO4}=L@sAKyw5)%9L%}?fnpMwT}SwuPu)b@e~&fbpOYys6vxTxXQOrxZh6IzoMdB zPz^ofH$ms#2jFS14Uh^;YYmCAem6K|H^JB?k!?jCDKTnroq*e6K_`mkh=a6+a5Ew1?gl9%+^~Bl&t;i+$a-jQ~=UwqMxE%#@nVR z_#nNXj0eT*A0`lL^QP&dHte|3VuI$sJhy`AcL18L#J#c_#m`%RFDd9WI9QQB-|$HR zrL;+!l0qUr$gaS(+BhN+nM@H?V_Nu<)|b_#+=!QJO;wTA}^~WZiWC z2moJuK{U`mBafR@n!kkkJ;UMo8XmT>cuU*7A16;;3VQeXu69|Ewzo7K-yg@A#KMz7 z?;&5zC(0;M-{9jzqU9D$*rG5e=pZ1VXL8DY^%OA+pa%wTlodaQ@Ufi7i$Vb4?J#4s zGb%cnes(V0loe9+;LLRIOC*jPf6mHn*qQf8T@ReAfFGJKOQ;@x(PFD?@TW{JDrrDf zmp0X@qH?H0NXYdHRB1*W%g|pdj$2rj>L#K}u%1>G+o=u-zs6}G=Owo%^CN*loy?EmSN<2LCIi`X^Oqg%EqGyt|V!p7D$VmY66;mmp99+!kv)_nLJ zJRTjLDb6uouSX^@Ive(R6`kuOkxM2NHzNAw5^uYFWb-`OM}F{_har-;Hj6b-a0@LoaKeH-_eK zL^xxUTf7ywMIif}MCz;h`7G}%VNII*3emQ7Ls8^7_DXX)F(j9U)2z{RTFw5J@S9u9 zh$O>~Xhr06tOyu-^sF5UEk6Ye)%&Lq+~3#q=t@9m)WlcEG2S?)CBHc(n7;!ue|jm; zX$0geXQ)}8|9+>UdXPO-HwSh!SokmQr2g$A5P;FzW^HQ5_TT@Tzb4Z_u+?lND3&y= z|Bn*|8Mt9q&rbFK`0~FU!vAplqGU49vJyeGd%pwzzuN}?#|0EnM}eYcrTE|fr2qS- zz?aV+fnu-RIokgue-%)!KKy40<9}TPrijh)hbdCo36&`(blLy2i}ZiJ3(#(UPIq?d zhqzK%kBw1yU?zM0w*~y?68v$E83%-opqsAWs51=pG0i63{jSygpC9$dBgO;N+D+hT zzc`RG=aGG>A9DZiaOF>P`28=R+_POFZeH_>0VNQ6Ss7g(&(n|o@8bR6uN=PxmXoBB zQNv`aM7v#2!|BI#eCOr=_f7uwT~CYWwHeF|aTSS?3lF~+bA^^u~Vfk%#Y-Uiya zTyMk79Xam(1Lpj_34ZU4LoB?aFU3)|&*e7%ym3*r=l5UDfp%{|Sp(L#L}cs@l+&d^ z;-pOH{(Dc7C+mNC)_mTWai2eb|HW-s{U2M9&vTWo87>UI!Qv9~UGI>UE$x3_26#y= zXDj7xw3;9PZn$LRz|)fq?B92%k`;N?pB=OK?wy#}q2dt1e|*sMUI&7oFZK1su%7ym3xc;9%R`at3i2W`$M5_MB>4E9p66m}A z8idaTY15ytCwAdDYq|QhV+f-d+b}artW-q;SARqFi$PTe6;-#1Vn%+p$dKml_cQg_ zTO2pHohY^4oiH#@=?bv6-vqU;OSGe+V-~AJ_SLhqkq*bNX-UL~D^12GtoVU?&dkR) zC!^XKa*Iw~5sDdud)Or=_n=oOmZ2iZhUXFRtA>+*J|S&PB_4b4=P3HleiNpFBv;~p ztlr}Dj;p5G1hDiDUA0(gP;@i6e!-_OSUs9v?GTd|KtzSYWFp7wqp#m*mI%GU!4cDN z7Tzb=8v6+;%v-8q&)5+!1$fcf{(B=oV0!y zFuw9!D!HQ?j#$d`?EfP&^)K6uP=fYv%zRy1Hvk;h5Cz^q&nC0XfCPf|&$5k0dzn=SiN}$JyqP0M-T9VxKf#BSfw=Ed9hMdTGNV8& zNJSRNu_yOOwz0e~5zlRUf4u<5G|yumJ-k~+tQkZCtC9uBRy=6eO8T9OZf z-29CBJ)MOzUM(=I6JaXd4y6?5>QgiJJno)cB6n7Dr3kAE)Km>`SsYfJf*f^9(}np( zjlLzmjWd_`A9vOg3|*DPNe&S5jn1r6$$zb?Rxki-RO4!}5axmXd4R%x-Gu^4C`}Yi zYN1Ti)7`zA2vNouv$>udk*`zaj1elUjZ4lKfD%zZeDkxx?v(%&o4Op@LKrGG!*VW| zd=TBXIdos#F2-9X36)}kPPj}Qo>~&_0{u}n@d zF%YYZ_zjxloiwqTA3v)3>YLZjR97u(P8<-AUI4R?`j7ZK;45;v*Bz#dqX?1;489S06-=wMWn~U7-BhE1nSnq+zTao0$V;IVC8Tf_f8MGmVw*Cmkod>*7_h#Sa%Ob9 z)1ROg$Q_}0skjwgcmEg`yJ8jI3rq;^^8bwklmMV~8q5c|3H%?=yoLb_jShRN2>Q?u zblo9`tro2Sdh4piaS6pFbucnNnLi=2Dz00m1gHTbr|C<8;_<0UoB1@j&D=#XwnDQh z1JW|qIqa&a-nmXPZ?owYIvcd5wC4-aD#Vgm{yif2a7zKjdqqFGD&mm$FTlVp|t z{3_{!kB$y8+ZZ_mgR#e?FuVT;O6Z~oqv#6TU;Wxt0R7_T1oc;H`FzWi(+a3hYQ zZNMnBgkw#wZHO87ItBNJ8x$`^OmS2AjGSq;pPM6vJ;amsIlqFZf9B?v8hs{8y6BrE zsYQRa+g6U-W>N1CBl7ha)uh(ctr=8gi_jL|h%NGH9S%&BK22*w5q-gzt}vXqxybR!Eox< z#=hTFQ)QF6MLBXry5M)iKaSEnXhc>-EitLE?*<#6#O%Z9o$B<&jb(stF0Lx7H1&xJ zmfpxLSdN@`kl{uI8ObKoTvL5BY*gY~DLf9{P(F``sMF`ll$dY*6!wb|RhQua-~V8V zy_SJ?EOR?txTm-EGumowwd3oXwiH;TI{jKFnF5WvcVd*7P<+tN-$n+}Ewu|En5x&_ zPr0oYAvVsqf8jmaZ=#4TQ7W!?(cE)EY{sa0YbEy@dggrnC4O#jrGgjhF-aE=x&fV) zf+igF9P*xBJXr-(Ff-(Jy_v%B>^SvATl(d?le)^Vn43twR@(PYLTL2lJ1iuNA3$_t z@hiIO-++aQ?0B`6`*bf!RmXDkno~b4s5Tl&TrUc-4k-5#uO(ncJL6}`D>^hc+Gc0C zI5@a1$lF6Y7_&@uko38^z6~p@e1&cS9q}nwbPJiuLMicwZXrL45Ox5;1_p> z09_1lVo4}6cEr9slTFb)@no&$>L|L81Jn5hTHoqGun;x)^Og6BeIG!E^%If%PAd zQ#22Ec3c+fyY2|9bTu#VI_zV5Fg-W4b_4ZjrcL6D#)U&_R@`o!s;LC(%Y&{tsj$Iw-9;Hl*&z73?nw01LKSn>cTIihr_!yD%w?UH8k559vDixG+NPm+J&BS zv0pUFdQG=qV1Gb`t84myOnrqzl;86;tRN*)5)w;`(kh)RskAgpgLDZJ0hWybY?+f!jYUoSUHgrP|e*q0U$3a|3P&`LO|8Fws7qHxKcF2s)7& z*(ZFa^ZGSdOu6_C#sc>`?QtxW8u!X;ZRO?#w#N+&;9seMon!w$Bz6vH=Hm>#jbO1k znCe(0d|G7qP%ND^R`>bxv1ERlCWmR4WMKATuYSmW^{cUNau9yj$*AA-SBpduhPZDUH@sU4lysohTly8ttT_fz4_Z`#9j_HB4*h1Dz; zNBE+;@#j&f99tL%Fk^>_MKYh4bNfjZVb=e-BrR|u`%&1GwEjX34Bf^i>rAS!9trpK zh~scJ-LkB;IeIi(pvUg9M1lo@c`5x5^!g&@>#;VtL`}IIz5mu1` zD(;8O7tBn;+Uwg7=m^G}`htsr#BlDJ_q(6Rq`Anori;{rKP{XwPz223oPiu6!WhiC zqzwD1oiJ$INM4`ue2PM-$@XI4ia@dL9SR$Lm;1qUi)!Dpj)JokcXA0vk`e1 zT{8%GTQ3CSyi1crcyEou7*uS;Vj9eSB_|Va)QZ*f_fI+7sxOY~a7Ocg`?ntfBqSEL zSf8_(v*GI@l-63b_5CRgp3mFi+t1JcWw8t>R0TJtBhV?wx^V`o`f95^;$R<8)4&GAkoHL|4c}>j&Qnke(DAm6z6ljT z7j+8;*YJ~x69U;ZWfD}=c}MT`#@*u! zhXiMsy9`fY*u|Rqcb~U``l5_)@!zZNwb%k7XMm@{IuWT@=fV$~Ze>UYZ84%YV79Usvh1+TZ#L=*L*A>kfAD2bGKzS%@v_zx_C%E62r-g3QB6<^Hm@ z+t?y*iEP~LQb?tG`L#!K#^*xap_@`QsY{ZXO>{Kr9*J}^YvtB^tYxj34Ses6 zzEqalSt%ym+1JOGbdi&QX*@eSt)9J7Sh8AJFcZ%_`Xb&_I7seeWnD0^g>$lx*o*9Q z;5>>UBP@G7j}K|omY&?jB8$$p@QeB_D_>neq@Ny_fzyur`JO4-%HUm6USf%@-fyUg z8guFsX@+bYELN%>V{@eNix{8sEdDJu%SSC%9Me~8`9!$S)bo$0Ms9YLmPzdcZgsaR&pgsI8A3lv1^y z#3xOp8t&ks_+S8Ac&4JGtQ(v|{lo^1rfv6cg6v6+6p@)d8oBB3>&6p)wjl0LOn@?5 zmZbHfZ%8!d?WNk%=#DwIjFQmM6#eoWWV!!|Ml=% zsgc++lC!rzSi7k3jA;p1Wo5p*wcqArnprswb6?Z|p+1eDXrJ-U(aZy}`OyLH(OkZ| znqs{D~v4(V`l0PpN*Q#pc=?FLtyGJ6o=k?^O(8?q~B8TQO|?AwP08Eyd-Vl^tB9mNu)P1dM|5X!T6w39Lu3ohQJs^>f?5!`43PxZciJr3{`lUhMXb0y*y)XP&d0i?Ghj z?@n2Nk+L5B&G$nZ$;_K-jmSc0_D2D^c3CFRYe_wxh;`DKb2{&)KJUShgF!*Y4THqW z27iRjZ_b8^iOB#DP_Zwf%|rBV zo1o0s6YMpSuK}B%9rQNgLQ9ud3j+Z?K|dG23N*;4jqUG!RPM%)MOm1d8?@Ega(FbJ zMjif=NHoVVsDfQl?~5=C1uUUofJ*2=&6-U56xpkkTR zv)fSq41h&u^e^Xtn3|E)`w!2|sf`QxJ(Yp6uxy-%>kqP56Yd`XWR}27K;w-7QM=<- zq18qe1@3RUg;~MIGhs1V-0F#W^o;&``&zD#!e6Spz%QXJeyDRO|mHH6N5;BKnuL1M(JSBKz-?|8zP}pL*o?xE{=!;aE0t4o~rWzmuMi=x7_si!& zX|{Z{p^+;`o+I{bXR6@ygYV5?bldf)XE_%FOOY@!h88a(qef^u9~)2-0oSTuz*a`Y0Gg0nHEA@OfdTyE zm5m*B?K0!dDzw#)4yA-zzE!*R}rg6BTyp>NX~ z%kp2+l$@^EGMNkrMrZx@RWhy>iVA7zc+^)Jk||Efp-RdB92nrkYGo>q1ZdPHt<0zX zV_^y7~%y!7PEzWvQbH$op1nyh;B=yc01{ zcw;bY3$SbOzaibO0Tb%o6w3@fCrf2Pawyih{H#q=u(1T0w+b^mTz}0BgqLGs5!K|i zinvb|j67W`nFLeW>)zhprUd#o5LvLNn`44@4^{OwJXnatB8z`3#ncZ_f!vZWa$rMzd^>lU&sx|Wk5PqNNjM;PML3- zopO;4C!s-Hn+k?7fl`YQatpKI4HtFZfAvb|ux5H$*kQ6m{wH!l4Pah7N&;?c!f2wu zN)oTBli4v4xDxNd?;m}xQ-2g?g|lY1#VJf5qk!7s;@s#PiG?%Q(8xI#K^kvn)gvVv zk~5@}<7i6;lv~-5Ne-&bdEEVeb%JoFt91=E(F9E{jB39$1G79Xz+Vf5vr$Z2e5r{$5Pz#$jn&%OUz}4# zBv1oDme_bQ4-GJ8Fe^m3{ZC`n$B)LFedpl1)&Y3lOi>;G5Tky1Jz)jl_7^A_OT(E1bJw^G}*|*h^H!45qae!1;dzXaOA+;ScjB5K&!4|_ejtPfyZ9Ra!-Mpv6q1mkZ07hW4}$!^TQXrM)d)UuNyfBj)_Ix=O2 zBHE;@ZDbu_lhkuvpt}TdhygPfv&7a;gINcn_$A^<$AY-__MA`bi)#{Dr!Ivw1)a(- z#{qe`OFmhulh*B6f0S#nG|BzK-Prn-)>Uc7UdL0|cx7;fI`c~ablEnAv^}@@uJ8>( z1e_|MwYezk!CeqTZmOk+=wtJ51udBNbB`qgQ=^)jfyCf(Z$K&vUeGsZG9GcC0DjNF zwKnad=vLa%42EG;9NGG9`QKU7q*Lk9t!gKNe-Qq@o7xBDe~;xP|s&RHz15?BafAZ2`$a+cW~Uv-bl>Gia- zn~aV?jij9Ub1@?PAR=TEh7Wn(S5-BM#$Ot6onEW#w3|H5t=pJ(E!_UZ$^6TfnGP2l zR3l7Nrl+^ryxkL9h}({< zv2Ikq13yg%ICOU+Q(?6_3>HpZFaRSd5_%W}d5yZ3 zllG8&oB?gmO0B(3)#4PsDL}vDEJ@I4Te5T0#fAJoe@=#II2JJvk~`yCNuM}WbX(tG zcOU`UAycNwU;opX^Qn0@IBEOl_Tn`5Ln)~S%&VV9ddDtb^x&ITUdNa^yIep}eeq(b z=2$rxpCl4V}fusCU8s6R~%*;h# z286|@@L}|@RUGayKryK(c4XH|N35AE%DWgyTTaJTL+|>jV%rGntGY+bYs*DYXMofp zq$XP)s6SeigdnqlXB<;_Uuddy`J1qcNc=aK;XRM>ZGGkU=irxR3bDgU8yHn3le03# z;3hBFr0}0wh7UTyWKL5i|G?<)`zaS~z5ofZA~%)@ive#TxP}R5e?(bK@+quawz+`G z5`^ncU7~OHQGu+Lqs&uR)-yUuMw&5aqa(n&KyYiL*dMyRR4fZ0zqs9c+GeuAm@%sRw^ z-BJMKgx33dsY@5fB!@Xn0f_D7z4ALVG-N07zWzW%nGc18iisTJ zYha;}tAZ+pAl3S*Lj_jrj>mo}j>J`k0Odc6di1WD&eATYTdK@%1!oM)DJDQVRN04N znyfRT%i@wh+}-KEK$VU`6=`1#bmsOB@NAl@era3!c3qvl5}9Y%6d)0pj|S4(Q`zDx zp{7QUxHWVzD-W?iWTDq(mQ&$f(=DH==6s{+spciloi>>sHu7o5KmjdEx^Hn~Cf4}) zcJ*f+G@DL~n*y78y_Z~6-E%dC>veiqqVo5Ni4TU%cw`r@)%tcE7!myT$#rt0yJ@jN zy;rPZv3-DS_^Is(kk+?Zz7qk76xJ z`4vG)BoG81F2UuOo6sQYLzi#=ukm{*MkUS=kVW|Y_fDDAo7+UK%Kk7%-tl3&{P3Mv z0F&{1ev>$>mi$NVpu|Xmebl)%+GV=$U*D^sC~xyj?}BJ^uAa%N(Qccm*7;71Qdt1_ zY%#NQR;x~l@#{76m>3J&DHEgTM!xeNCLG`(E#QIB?;2&4NSk}O6w{-IR7k9;$~vR# ztKR{|MM>8I<4A}BUfQ~qfwoO2Ox!b3*TNKq?a8Aju@TA70f{7X2l;f!-9VB}GzDpl zd2E2X(DY0ik)Jc&-_8u+05*7Cl<_ZKbM`Lolwrn0)fr!y$35BoJZMa2VZpqgG(8?; znkuw%kwx^qdws7)c*SJq?LTHzDIIRny4773ZQu={{2*28x*}0VCH$iGMX{h`kjnuL zMuNlN{t+D=a;Uz|$i+vo{?DDBbUoI64tFbi-0g*j{DL;LE|u!lsTg8j4+jP~JZ7rj zLXYA&@Q7THkc150iM>%a5Dj`3^jlp|)*W8+F2>37r&rBq_!oC?^|_ZuS>G9)bZ?#l zOyZwALzLC3eaOxKomMH8x57IDtl$ELFSu4c9I;jfQem!BMY+6miOMzNMK9RC5ASrz z>AB?=TmFqA5CekApF^x%sRFD$c??~&BPP;sBvi$`sXD<8wxv|V9emjNp>N4Zn*N{S zSscOh-rld7msi+__7#ocpLi#o$B!xme}qfjyF)2(hu}J!pkjrxzhU?sCER9ln~8QJ zi7G8270g^j%2sVY8T8iApLB;N)1n4*WwrgY-KBf}PNA?B%lXhYc7&nFue*O7EAMR- zJv~lXO)vCW{(FMf+s-_ySQZf`*n4|*xvl+lwVq!??K)PZ|Dy*2m7a|@Zs18?Y=&_4 zbRtTi=z{s0Hg?|=m=tUQuH~?LrLZJ!a$)nQ{GG3iK5r{%Q2+CEF>fRqIo1LaLyRf{ z31abuN8D<)O^cxdFq=Sf%Yvp0QEQ;17xWYv@T#Ym=lqAUszUih6=5khJPn~Sx#Gk4 z+6GYrY$OU1Yn;4!y$7^Gs5);Sm6FQ@{VNs9V1$y!>x-wvW$8y}ZG1+QGM^WUG9o)q zQZ?YW3cA)lqU4eRd#&7naHs*#{=<_wAVscjdltAw>4`K5S|zI$ zbyX=bFw+*_n6Er~6A$3xU*-dZ`N2-Dk&2!4m^x=SYb+k=dnQuUO{CPRCM6}1gkxi> zesqQ{r^{pEdnQw7K54oT%+5LW+Tbb+eQeLOvh5QjxTKrp8>>RGx^q`Y-6sd9p7bo& z$`5=N8n2U0tH*Kb>Kl&sLzlccC!;0y8b9Y%PTBev>4rLQ}VvgVlOjWO#iHQB?LE{qy zoE6#ohTHsWKo94)Ic%uY{@pvNuTz=d0V&?l=Ca>RBCK&DB!S*o)QA$-4H}tKacs7K z;JqLZ3c`P1w0Ik(>G^f7+-E|%pp6PGx{-A<&*V{qVZVdUqg$Rm9Z{0I4*ofowhW#Z z6i$(V=JLG(`os~OFVA1g6lv{#dB=Z1Znu8^!tSc(_t*FY(lIWr;kRIZPQPVGLi?!T zbU^4VZ+-puKPzmu0tNl#%q~z|`>v*87V(1+pLa0g^*S77h-1QH?uzN9U00n&tMbKQ zL%Yv2TVU(;fWdyv#IzbsHlMW+;uZS=;Z3rA1@zk2qr+w?P-0rGCtq9La`lF6jAO#e<3eOEsD~Ve#(VURMZ0LtLI3&)k zVsJ+88W!uO+h47ktK+2vvqdqd|DjAi%I(?|!_NzYzkoj(U=z*&1 zXVVn^(c1e8A0P)a5hDCH8JjY48>4>`dOY8~~@bS{Q`LSccLvTSbCGg-` z%07v(hIKAEgBj`3OCnU43&-4hYsJgl$|L3b5AK07;?Jn1ST^rchZ5 z;tzlppoWtD+kOiusQ8<(rhEO2B4O%Bi*9~AexTH1HFxq#+I zH=tYiWbzd^M@R4a*ocQfjR0X@OMk7Rp;HWPK9v=tf1mjnS$#Ohc)dIBJ?bmf*sb>& zs7~knr0)JDa~`nz+yV0#DZ(Xi zdu*}pbYM_pwW>6_0G+1Poa#{mBEJTEM!O7tOL$P`ueVL#Rt&i+xt)Hn{}n%(?VeToA8%-n*4qcaz=xoHoRgok53^W6 zT9iqijrCZ|c~wETw)y54X^YrWk&|tgAUklVk?WC z>xk*6c*_c0KLZ_mw$dYZOr+LHVv#+eFdC3A2k9%cu8)6Ujc$Pm`FdP-EGR6FI3;U) zr{Ys}o})4j&8s3|Pcg1Bis%rvhVAC)jHLVc6Mos{`<7}5Pz9Dfv4N1>-0h}V9rZRQ z>yP552GY#Xi0HP~nb$$pT6!3DMA%cd8=eEt4Jb~y%1x+4e;$gswFVs=UW6Iu1Gi0s zcSyYEc-F)^6}YCQ0Ux)GBBu3p&=V^p8D1gGgWjkwI+~q?iZPw{-NO0j@m@=Bh%NWs z{-rtj$u9tmpNo)e&!@yq@UA6pS%$LMydeJS0!Cx~?+-?c&3KGRSlbrJd-OTA9^B`` zesf1i5^a05TIysFMO z@}P^vz3-a+G76VrD9%O0De!suBB={_lS7=Ph-xK9LwGIYb@>t|TB}|v-~6*|ZFCH&%wU(3u0Q8<7$!6LBds!*H)v^||LS3}t)cp3?XMsv>_suEKl%|(N ztkQ7Kk`)cH`o-`%C5sM?ktL)R#c|2j97k2CGZ zYpv+(Hl~4Jn&rjS#1j3--T6lP?IafNNk0elbe^lJ^om2%+Zflt!>BLlO>1`incBGS zZx@hH7%LII%B59rkH?5KEqbw?-xQ$wKTdWN2g6wyrwx)Ka_|q;^`6|VUQvd0W|jBR zEnomWEJ53+OZ9l@?N{Gb2Bf0{to3T+kvy;UEBzJ7qDSqpbRdUXe5|@velt36h<~vD znpHJ}jXoWiHGl&)>@I=Ss{Lio%T0V*fY6M3a_wEZge1vc)NfISF1CDiH|6$e->_3v zxoXnn$#y!ro-h>lH`kj+P8fD$|KTU30snm3f=#^%aX5VosejFJvT}XI7>`4(*4a*b z!rRBR%94yL4ENM|ynRfVQW~D8rqtl~;{;Q*%J)94Ymh_Q+=~R6x%&iHqWZsjj|x{` zx7Z(e1g?V1PiW{=9WqZoL~P4j*Y-)}<2Hodr<*$fyUrq&5JB#TWVhw9TDguY0?fRk zNGU@gB##B2%ZvsFIAfIyDD2~U*1kDjAJ^=0(N2vGcJ6G=-(A}_k@pt1vrU=)lN+uUHj)h`{Ck@wjFWM69Zw zn8sx|pZ&mg;M7O4mtnw&>UrqPy+UnLDt}+gp`Yo0kze73i};Vljze&szScO4Yf@{o zyYy@SOj(wjL({8z{hP{L9oK+a(TqCXW`5!U(;!zntr2BFIo0qaI!6aJCG`H>MF`$1 z(Wd1!xJwsMTY#F&JdP!eH0k+`HxJ`rZRi|eZ zgyTN=K3qlNlS~ZgzgbIA`$RSJ6#{dt&i_p+k_SXzd^>$=t|Gk(6SPw66LN6ztV44gm&zBjgU5oswTj%5&;=m%bNB9TQY^X?7Sv7Ww0HD}X{oQ764sRyp-?gMc9?50@b^xnMHps-!JlQ=k zl0W~a2<#`O0EKww`j!fBu7u-3ce~j27kicjxwoj=5^oQ0DG4=!`!;ShisDy^w?=c% zE#^KI+{%T%FRfCiC3l^wCs@Aqz|JtE)iajO;Pwg9BOnx~!^sl~zCGJpShyfI2 zco3~z1HppirW=`>rm)nPiL0&?5*HFze=Tf8`e#*RZZaGUbg{tWu z5%z$GoUWm4IxSu!>W-lRfwv^;p1!)?jd|fA6lTogqdK9}?MC!sJf<^1f|4gogr#*~ zAG;VT>@ObtgrEOZ$Eu6ABnSd;UP739E&K0cL~r$ zq1r21m)qt&6nfk#>*Y771fwMq!22haMNB6?&)%cV>eZ2;(@wmKi3#Bju0KGr8T03L z^8IqiiBTho)XLmoY(FwWUo0Qlhl5_p*e#ONH?H;ii|PSc5bhxZGuxxnUnS2Dpg*N) zu}c7YrJa7MyWQ_#zw4q}`RIGj4uIay-_2N zdO#<2pm*)+!eK%NhqrtuemB2vPMpX~WsGx7r8r^Labn$x${PuUuD)TUKj^kJOp@Dq z^E4Zz`Q0@IH})OugZqizsF3r=0U`H~+yWt?J+;I~0-QTzO&8-ff$O9G0<4`PRJkC( z>UXeX`no;t{h5lpqmb02<~o!)>cQO&Os_YLy()zq9t-cr7&83#GJ;edKxe-`?+OFW z){e=PH^p$SjxK`35ZyI)ATPgvP@QHZhTT9Ny zMWqLAj{2r_q1jFx0fIKn#l?ZDbxwj^V{w6qHDiyI^d;LxFKr$g5V~@tcgoA0m%+yV zmF7XOhNlSYJDzXfdg!v43xmn9(+gcS)6`hXy)KT5rRC%Nr*&~l~*EcmT7`a-Mf6MN4qyA`$T&b!sn5SAoAuBsRRHIb2Us^~nj^J|BFa9~v*2XEQ zH{$N*7*KR@_L;GzitEnfHYBzY+|ka|0zV`||Fa9qe%?YM*KkTtHuu|{AcDNe=#2x` z7{V1H&NP?Fhj4zzmEOmmq?dcVe8ObLsC7`)oJ7Bu_&7!F@>Jdsdd!UrQC$l+XK=3G zS&(q7a_)Mx?XHyC09Nr8$$Dz&JdV$<3gtsH+Be$N>&I?q$sq$%`#SDisoGO$$S6lo zs2chRf@i_<5U+F(L|gUloilegDIy}b0aOU?^<%HLXBbr}td@Xof;T!g7Qcw-IP?^- zDvsKz^+;PVX6!bCbq2om+`g7U7YO%;www2|`IjbC7EtB)Q9}J4tRku|%37x!*(_jJ zq*>P}qh+ftOV>qdXW?j-q)v&s*C67+aK8oB!lrm!2of?1?%M?KRzcJn(jPu%HprDb zeCutD&$|TYg)K!XlHK-6G72x{T!c|-;~{E@J0G=gsp`x_~ZK`LyKdHKiP zuadb|w87hV<(MWB&Tsi=_E^jeZ{1(+>hD(O+Sb3ir9!Dv^U#3Doe`a9ud_~53SJG* zv6T@bqdaDTIJ=qhc^V6ksQD|*bve_ zwLvq=lGGrQPLK2PbX9Y)xUL8KE?YL;T3u}u9^(N&t2^tl);%rK;OJ*l5SJ#E1^C|T`0#GT4Acw_X?%&?pV0=90Q0tjtxC(R!KU&4ukII8G za%~;&{2Kld=F;9)io0PfqU~R=bn7OvdLiKTZW0?ArkI6M;udr(c*`q;PE`C z2jBCaXMpeW^oL*bRc66n9X1Okl9ZFpb;rb1Ud&DbX==;u$%R+seE*I=5(o{SszDH! zn4ee#!ukCcA-O+|5b{VR{o#+q((>{W3V$`5oRc4&^DH^me6>^A71;xM!a2bLyj0}c zm2cPAGZr>uG@o_4<4GDJs{idXP%92yJ2iE&K%5=8ey;N`I%Xrsb~qK9uo`$_vzVZ9 z*Xp<%>mf@>Mk=VFNL61xQvxfn!c6)xv1N7xy%c7@4Zrjle^4+JsoFE$L0ncg^YuZ~ z=4zDTnaMzt0qhomM*!^DAK)aJUY)-?zq@3AJo1rT0g~D$k2rx(H6ruF`Sr5huG$W0 z&;(@K_lvBBI*ExzO8EB{_kIR8FcO~0ojWq3s$H|Jpx{0K;>{S0aD(tKXj33ufWVG! zDVR2fD4u4-Fr}3dR8q?Lk*{P3o|JRuwa%Cm6uc3W5V?qL@>qO&b(5{zb%!=d09YFx zU~N>aj;ueMbE+*bd$@J6D-$qXw8B;ReB}1S9VY7~o}Q38i|QQ9Rql@(1q-cY3M?1| zH+g?9>@)SODJQR5FR(F=&A6$)Zu@xNDSTrrb+>RAvqvCD^(Cibhk||XicD-e%TA(=cr}zF=)s7 z8r!ZA@qAz3MAcUL94;a0 zT5BROD7d_8Ym=P6|7c7g7t7kAzQ9{efA2%Z&T44&^z0}x&odOa3)Fr9q4IKmO{1$U%s&5MGL{)K|Wdex@U)KnGo*7+*F|d)}QeD%5 z(A_hU7I)R~OGQgjb($RdZm-;I;_5sN^1wbRG2??_-Dw?-T>*8_@jx+;j~{@Dh4Pp& z4N28YO$@oP<3bVf(NHJv%20(vFYhsOrS)CkF|jPC%UE491^`0$NOVRtxSo1(i%QTZ{9+`3B_HhlP zY=3cK$3Zub5lWskq|l(gq1B>T*mwKTo2z=qs?X8_QM4TWDp%5nzx=hMz3Xt1+eGIU zrp5qH9415B-0JNZQ-{dg$W~Pydv&Yu#mt2tB~L`Jg%Xsz)&xZsV?h#`ux>006#Hba zMOb|iMSQqas&s}n;LdU_c-TeZg7ab5|37HeA(0C-_+lG zfn%?d*KWN*opWHLb||S_-8;gBX%{QPTVG~5B(U6LpWlqC3?cLmh`sC~8)A*G1 z)&A-6kr~oDj=dxB;n&MYvTeSz@rjS$?1pW=sONochG+l|K}er5ytxEXbzXcoRNWfs z+U|lD9*hL41rn87Qs!7~2<1uB)gt^XdOP(n)@o{B!bJ{Z7?D?{mqiv^g?nJ$t6jOW zd%AlW36<}eJ(?k(BlPKf5@ddmU*(xoy3dKI_J!gYkEtXw9<$RcEoj|E-h}k9xnjlM zegJ(O`-=+#S>AR2*Yd)5?@8b%FLBx<)?WI0~Uq-%Y)u2V=|Fm^s8^VH}2N!_eU$aMUR`c>*e==fY>>gtp&hSwV%0nhH+Ym2+V z1Q-ng09@ZBybr24`Wt$XQAab|orK*xM>Ahh&fx^bde5FNI%e~eI6R2Pa8xl@4f|zI znfe^LI$k<7-MdZ|tEE6V?l-n+hEYd<@S-eB%Zq%bR)7~cmjV`oes)@B`RB>j$YH)$ zu{Mq7O*I~{I<|_>#d2-J zUG>J8ewD=mD*7U6NuX09E=P;Jp=pk*az?>obHsQFbyAqfbk51Hc}wc(_~pe6gUUHr zHwjS`kzIZ)^L*?zv62glqQmJ~_CS-RuCPau07G*=!=5gfNK*^DFy+@K-QVwG(j`);_HA-H)NtGD`oh-KMfBI#ah;ic)NFJrn_Ra! zHP6PSWWG}mZqC2AZ_{GQ$C3&w?)mf)*+X9AUAEva?LNO+@D}6X>|>`JewimntS*@W zDtF~+!Z-<=yXy;cW7h<(0YAt65;Pk)J?t5c3bDh+r{ge;M0vK;3qaGVUb56y&kC$6 zB=)D{Ha~(~`_tAN`QPkT%$}Y))Io*GP4PUpEa$y%+NE`?sqgB80r!_;gE(ut+Rq*0 zf)FsK(58!XLAO55rB{wE)JPXm9;hci{G|M1x>+h@&=#Al6Vm8B>Zetlhw1LR;yOWz z-Xf|F(pl3xm#sgcJI5BhJ(%X@={rem$9Jin>(Kw@3$n_eH>vML(~Uin#NV*T%oeUW$N1|Cd!Mr8nv7kE^kFW zcMd==?hf$7WWSOf8Jp{RrmGs~L`{)2z9qkM0z5hlTF%#=$~XXFTHH-tNIQD!)f}L$2mM72s*>VnHW)*GljruS0{)hI7)ZX?q+llzvqDqRyDbbaqw8XtaiQ*nrNstHwUOf_!bqk!50?^{Vc3`a>M8vG$sn z(*pW(mU``Nc=3hjl*bQmReD&?rjAPg&v_-mau0UlSPU~!1^#8~U9FrI1zwZxc=zwk znd!7=CTtA`4}?7V@4-k70HcQ6hAV2+CkodaTiIt$sUE$=gQBEIhM9Od$820%1`O5p z$t8Pm^(QeXLC3ySTp?1xXVCUAkWg3p>Ma_l_erRWb#t)AaW9R6qOf zQs{%tw|l?o!(N|Isg!UpL4d=7gnM^nrfBI@l={24EgVm_CWT#`SKu@xX+c-^W61M) zSk2>Vg&VHF!agm1B9~9Kmq=p-?uGeBRnOLcD3@=`7K>j$z_Bfs4P#hutsVY_z7h_@ zuRA_eaB_-;oW>|tL%>ISV@3*-d&T>CzoCh_wRzL|pS+I7nxyjL3rL6R&818xsx8z^ z>W4A4&HYIyrdTV=DvKxV>qZs1HS@{%XYF#cI0!&v&(hdD=iQe+do1_XvOiO<)TW&9 zy-)AGAzG*$jSIR>H+4XMMLHq9&_GF@hS^L#c}k@8erkR86{5N_aPjl`EdpIY2gq-1 z=o|4qtNWJ{fO`YdxHQc@lw@n-^Q0hzMNPYcA8y9|Op7hB{ixAL;Hk zEZgaj6FrDgmPMuvu{2+5uf;UObc}{-r%d|mY>&E6(_`&Qa!v}DmnGlfVc(jnW;2y5 z1f3GmvffuK#1#1eCkXJ)kP_KH3V{%`|;A0 zp&?}Xi+C#R(HkpA4{HbdA)$Bm=W22(*oN@4%XwtrHx*ofyyCl)JWodF6NQN%#dFs) zp6JWv4byd^J#x-OnU*D3_0oDz~OwZAxa8xBCo@OPx`WwDL0w(9Dus&VD+ z6QDApT-F>=9jP+Wehltn^&5_zID__EYQ7th)}x;vs7Zz2UsUvrdQX)JuclYzU&IBFz9#R@Xdhwr4?zqS{Bep*{yOI_ZelqIkcR%N7PlV?++ z>;dDvELi2k5szck)bp@!8%4d|l9=Z3o6Z|x0_R8^msNkl?%8haLuec-`s72KjnCc% zI#2{+`0!xlWPi3v6115YxKiH86Ovo+uch&Bo&y%(R$s!o2&xwB@))roX>vfs%qAzk zt9ck3ac{B;g`lk;dsDOwB21cqh$XZRz>dF;)$9pUk`6g*%TJw@FVeXO;j(ycQB>*}SvVChBHCTB0b$r?*o2&+6?C_RXZ3kyr@5~gvV1fK5LCAqdv zov7v~;&`amp0b;`^l(*ivFvJ>XJ;t(4DGSM6Y+8lMW z%-?a|{$5T1(|;_hAMjlMjisMMd((99y?K&AbavgNS5u_(DfPcls6XFztKQujtr7eg zV;l98m-*h7AumuK!ubSB1g!@s1oOF_o^Z=tgWBfFnK{n_>7C@=_}aIvS5V=^O}@GgFP(`hC83iVrHEQP zX{*U2*vU1MliP`aOyNE!Q<4gzm9hFq$$0f^z!z;%6~V)epRPkaj=vl`=4Ch4*mMlx zLDDRNLov!&4_kg-Qd4EVA2wI&G z)Z-4=yZ+gAslWLAywTe=Ur~A{b-qE$VrI$y`0<}(!yx3D8?$I~TzluX`b)#&UFN-4 z&_92VN*pzxc7yOKR+qO#$I1W4qa;8DzqJY6$_mG-!CB;L|mCPtQY7CI*hZr`R}-P zl=b0+!*Fw60V`#!PuG$&LO0!sARCDWugq0Oz&Z}cejlXOg2L>Uve?*TA2qZd{6j(I_~zXyGq;;@&eUQ!Q1|#_AAb-YqL0#liZj*2&_5Z{ zDF;nNWVHj(|C@-aLH6useM_3Iq_I>#RW{t83aAG4#lNVFWQ-EDxleYKLm*xj^@5x{ zu%|c(`~t5Hc?aiU9*b)m%r|~!}hT*djPmI zPwO1jW@kFwQMjNYqu?_@oD*v_`%%?CHwQcAUdKnoUTvOxq*6M^@tc_!_WTV>5IXk3 zaqIQVz};LQYNej(x$&wEda1uVi3_0`2VLG?)p{}9t|pV6WueuU=l7I0g5|(}_S(lC zRSt3Drsvk(xW;06)$o<;zQ$M*3sj|a7%fH9et|5=HU}$@zJEcZ9))pnT5l&N(hFhD z{jsPtEfD8zhh)90KmkPs!&Sh}Z=W~Y)>-3LwR>p$A%C_L36x2o5He$-L~u4fn|L-3 z)8wjR*RA7TPyTbzxX;TVg%SGsm{Wig>km-yzQKaixY;|S(q^$}m>Ga2X{0~1Q(d+{0IK- zUG43{+)OCUe!mL#+7(`OfzZFqSwevBmrrF4y6pEdJI@hxTd5Iz?cUp2VigLf|JVrN zrZHe?ujY~Fz`zz|#Src0Rd=a`t!(R<&&+TX+8?hP3IqQjtpLbFZ+i!YZ{4Y#P__S0#b~9dy(>}XMs;($3 zE@WArIv7W%!gT_Eyk`DQzi=tYJlV~}aoTqVqyFWt&)&(;u>OIBx+EYU0~YW+72;$c zl6>^iQttF*>M4$SrA+?x(Jao~(|?YXC#pBn^dMi2(Z6;-Vy={bPUN4D`PgA_jE+qy zfro{5Su9h}#w&pv$SNP<95DdD3c%iD+Azj`)AFed!``=c8E%_-SD!njxjv^GM>{CH zSB#<-jsy}XQc!SfsGpE?K=_xO{Y0zMZVda+gMqE70hJ3?3`*kcO$AQ5cv@A~*mMEl z2z)j^+s9tFp*o&VzgRD>#qQEFx3&muK)^vgxyOB;B632XU_6C1VA+f)wWN}WssW&3 z#saI}v_kzW7+^c5=HYF*_Tm2pHi!yV4jNq~&y`QW3UR8~#-w?ysMMw=kfE35-0BnW`n*I=qhh!4xYwfU&# z%3OSW6&$>icePr7EAG@`!3rQg66hVVX`ao&OSAf2_3Qy{nB$N7&Tz*wbtO^JVY3>~ z|M?sdXv;uD^pVFa&v~I!jaimr8$};pBJPN2F<|}O&(r2A``y&K64)L+hI^|m8Q&jf zH#JZodF(Jp;DdReXCLD_O}EWF1q#p*C$7+p`ie%?bdz4Q`)>#&Hq9w4PY;*lAO64g zu00&ev=3+2xfPpeOT=~+jc=r!LrsWpTV!L1F(`}37!l(Tp<=gr=cd>_dP?|+S;${`}e!P`D13@>w4yW-uL(1zx#Lp?)SakdFo?h z;eb!2vYp7(!b0w}6j$?|z6X1R9`d5mo~fvDV<%uCW#@IVS-5JYuq5M6BH)5X^zB*F zS*>>2z)e+Dv&DGq!8P%_KuZA%n32SVeZ+#2_a*=MXNBP4v$bts=aUWor0cN zOcQhCkiwWwJNtS9DiZ$jqS)Qv$~2h+(X za>b8OUMJQit$}P;RIh_=GQa?eUk?aW2B`J=F#d}!3oXq$r<163?Qwkv(0O{r*Yx-6 zasxr-@mHHBCBHeNHtlnvl5y2;dY4oNNYa&Pq`AT^r3S9&OwWXB)@N1n8tTfaZd~%n z6hKS-Pcn>GoONNR`%7|SO$WM1eR5S_PTaP@-iejF>n2|60g`e8vwP#u@H%mZ5;*}S zQGLzBx-HD60P>K22N^*_(H=Uj;e(e z0yr20LZnsCID(bm5&wWJD-Uu27X zE!g$Ml-E|M+#s?A8sViRoyv<-K|vHc*OwO*Aj1pTiFZ9f7%J>K73q z0c9e@tm(ZBw~*+{%+zSlp5qhth?Q9k`eZK7T;!UMWqS%jV2)jgP>ZwQuMcL04gTQ} z?M)8TH+VgHyXq!-@d(k|)I4>HE=h)tU)2(8RB#73CA@1A;EyF3heSyTC>YJ44cMBr z9j)2ks92`~=-Lf_001vjb$pC^g-+`HW9AAy$n)iI2I%|>6EzUzY#BJ z=Nc4P=JA^N2?+y%#VTkdFw`H{7PD%@JnYLh3O5+5K_X>}!vZs<%84b`C%Xx{)QYE; zt`e23R6|Vy+W((F;th7WhObITKtz=gVNQwq;@}ctV9{0VWpPI7nRbyRa=ID^O_Zl% z6N8;Kh|R2TELJ8} zFB|D{#>2iyk0Y!h1{G|L4D=wxip5m?SX_!D!Q{YoZ}8_kwpuC|$ZD;LaXps(J>}2t zXZ$wtFTt?vpY*$4C+}tI=KYIJz_mkXgdS5-`bF_=TN7uS!%t#z>H06?C-bn8W{ju) zme82`p;}WCo(P+*1=LO8`m<4N^X8O72In%C^#)0-D~y?aSz{f z?}kshif12nPFRDL?{IP#i#gz5T2|E}l+5MDm^cQQyQQ?YQ-unT3`fPu8pVQOXCHIV zh>OuQnz%X7oCGk2EtPa@x%iD2iUTYmXkcq0WMbNHf=p; zTUl2#!p7P1LbbJV3(q5gY3*eL&kxLyy7QE9%I_A3RH)n?#X>Ae~C+ctAfKLyaG11W0c7ZyiuFQq7IkAih8wwFECjvzD`|aJ57~X^G*cN+XvHU zGBRS2fR=>&N`%6%wq&E0<8T$!Or%|3&NxN(9?W;m0CIU_pJI?j#MoY6xIV~*!D?vXPTtIKe0D@RR#Jwxp z_l|);$!kA+vMOXgnh}0bCeZxMw~%KFv%WNU$}noiwTXXdud3U8B6zamcssA5VPdXu z>$#G>;`)+=n(aXg)wlCCkisZvf8iYs@?}hSgGOG5Bsp0i#}sQqEC>JoAz+d@SN(eJ z;HRh()x+bm1WaHbu!wr<+#RdOd1^x6o?E?pcu!-Yf>7SRLQFsM>!LsW_HYV`Nnt}& z3OPBVkI;iGg1Xx=nMrr@sXDc!5WmbCr69&6`=;3mL|&<&u-x2@o)$&)e|ypCMcJXHt(8&|S-m`R8->+7 z(OB*o6J*S5qln`x-l}q6EU)}U(EB3IkSUiwv+f^b;ROud_)Vl^#kNlY^6@}*2Qwfe zMpz*~{WXC<$h4L8R>y#mbtQf8Zz%X=!0K{o<$xTR2{hUB#o1GweS1z$UX`%dH--m(J?yH7TZ#!iU#@ O*Y^kQtw + +### Relationship of primary (distributed store) and secondary (task-local) state snapshots + +Task-local state is always considered a secondary copy, the ground truth of the checkpoint state is the primary copy in the distributed store. This +has implications for problems with local state during checkpointing and recovery: + +- For checkpointing, the *primary copy must be successful* and a failure to produce the *secondary, local copy will not fail* the checkpoint. A checkpoint +will fail if the primary copy could not be created, even if the secondary copy was successfully created. + +- Only the primary copy is acknowledged and managed by the job manager, secondary copies are owned by task managers and their life cycles can be +independent from their primary copies. For example, it is possible to retain a history of the 3 latest checkpoints as primary copies and only keep +the task-local state of the latest checkpoint. + +- For recovery, Flink will always *attempt to restore from task-local state first*, if a matching secondary copy is available. If any problem occurs during +the recovery from the secondary copy, Flink will *transparently retry to recover the task from the primary copy*. Recovery only fails, if primary +and the (optional) secondary copy failed. In this case, depending on the configuration Flink could still fall back to an older checkpoint. + +- It is possible that the task-local copy contains only parts of the full task state (e.g. exception while writing one local file). In this case, +Flink will first try to recover local parts locally, non-local state is restored from the primary copy. Primary state must always be complete and is +a *superset of the task-local state*. + +- Task-local state can have a different format than the primary state, they are not required to be byte identical. For example, it could be even possible +that the task-local state is an in-memory consisting of heap objects, and not stored in any files. + +- If a task manager is lost, the local state from all its task is lost. + +### Configuring task-local recovery + +Task-local recovery is *deactivated by default* and can be activated through Flink's configuration with the key `state.backend.local-recovery` as specified +in `CheckpointingOptions.LOCAL_RECOVERY`. Users have currently two choices: + +- `DISABLED`: Local recovery is disabled (default). +- `ENABLE_FILE_BASED`: Local recovery is activated, based on writing a secondary copies of the task state on local disk. + +### Details on task-local recovery for different state backends + +***Limitation**: Currently, task-local recovery only covers keyed state backends. Keyed state is typically by far the largest part of the state. In the near future, we will +also cover operator state and timers.* + +The following state backends can support task-local recovery. + +- FsStateBackend: task-local recovery is supported for keyed state. The implementation will duplicate the state to a local file. This can introduce additional write costs +and occupy local disk space. In the future, we might also offer an implementation that keeps task-local state in memory. + +- RocksDBStateBackend: task-local recovery is supported for keyed state. For *full checkpoints*, state is duplicated to a local file. This can introduce additional write costs +and occupy local disk space. For *incremental snapshots*, the local state is based on RocksDB's native checkpointing mechanism. This mechanism is also used as the first step +to create the primary copy, which means that in this case no additional cost is introduced for creating the secondary copy. We simply keep the native checkpoint directory around +instead of deleting it after uploading to the distributed store. This local copy can share active files with the working directory of RocksDB (via hard links), so for active +files also no additional disk space is consumed for task-local recovery with incremental snapshots. + +### Allocation-preserving scheduling + +Task-local recovery assumes allocation-preserving task scheduling under failures, which was introduced as part of FLIP-6 and works as follows. Each task remembers its previous +allocation and *requests the exact same slot* to restart in recovery. If this slot is not available, the task will request a *new, fresh slot* from the resource manager. This way, +if a task manager is no longer available, a task that cannot return to its previous location *will not drive other recovering tasks out of their previous slots*. Our reasoning is +that the previous slot can only disappear when a task manager is no longer available, and in this case *some* tasks have to request a new slot anyways. With our scheduling strategy +we give the maximum number of tasks a chance to recover from their local state and avoid the cascading effect of tasks stealing their previous slots from one another. + {% top %} From 08d088103b1b4b2c0c772ec36a8b3e52b2588b5f Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Wed, 17 Jan 2018 18:02:25 +0100 Subject: [PATCH 0004/2294] [hotfix] Suppress emitting non-causal exceptions from closed checkpointing thread This avoids that an exception that is caused by closing a running snapshot is reported. With this we avoid that users get confused by their logs or that this exception could be reported before its actual cause, thus hiding the real cause in logs. --- .../streaming/runtime/tasks/StreamTask.java | 119 +++++++++++------- .../tasks/StreamTaskTerminationTest.java | 26 ++-- 2 files changed, 83 insertions(+), 62 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java index dba4c87ec80c13..7ebbc7178b6532 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java @@ -814,8 +814,8 @@ protected static final class AsyncCheckpointRunnable implements Runnable, Closea private final long asyncStartNanos; - private final AtomicReference asyncCheckpointState = new AtomicReference<>( - CheckpointingOperation.AsynCheckpointState.RUNNING); + private final AtomicReference asyncCheckpointState = new AtomicReference<>( + CheckpointingOperation.AsyncCheckpointState.RUNNING); AsyncCheckpointRunnable( StreamTask owner, @@ -865,8 +865,8 @@ public void run() { checkpointMetrics.setAsyncDurationMillis(asyncDurationMillis); - if (asyncCheckpointState.compareAndSet(CheckpointingOperation.AsynCheckpointState.RUNNING, - CheckpointingOperation.AsynCheckpointState.COMPLETED)) { + if (asyncCheckpointState.compareAndSet(CheckpointingOperation.AsyncCheckpointState.RUNNING, + CheckpointingOperation.AsyncCheckpointState.COMPLETED)) { reportCompletedSnapshotStates( jobManagerTaskOperatorSubtaskStates, @@ -917,63 +917,92 @@ private void reportCompletedSnapshotStates( } private void handleExecutionException(Exception e) { - // the state is completed if an exception occurred in the acknowledgeCheckpoint call - // in order to clean up, we have to set it to RUNNING again. - asyncCheckpointState.compareAndSet( - CheckpointingOperation.AsynCheckpointState.COMPLETED, - CheckpointingOperation.AsynCheckpointState.RUNNING); - try { - cleanup(); - } catch (Exception cleanupException) { - e.addSuppressed(cleanupException); - } + boolean didCleanup = false; + CheckpointingOperation.AsyncCheckpointState currentState = asyncCheckpointState.get(); - Exception checkpointException = new Exception( - "Could not materialize checkpoint " + checkpointMetaData.getCheckpointId() + " for operator " + - owner.getName() + '.', - e); + while (CheckpointingOperation.AsyncCheckpointState.DISCARDED != currentState) { - owner.asynchronousCheckpointExceptionHandler.tryHandleCheckpointException( - checkpointMetaData, - checkpointException); + if (asyncCheckpointState.compareAndSet( + currentState, + CheckpointingOperation.AsyncCheckpointState.DISCARDED)) { + + didCleanup = true; + + try { + cleanup(); + } catch (Exception cleanupException) { + e.addSuppressed(cleanupException); + } + + Exception checkpointException = new Exception( + "Could not materialize checkpoint " + checkpointMetaData.getCheckpointId() + " for operator " + + owner.getName() + '.', + e); + + // We only report the exception for the original cause of fail and cleanup. + // Otherwise this followup exception could race the original exception in failing the task. + owner.asynchronousCheckpointExceptionHandler.tryHandleCheckpointException( + checkpointMetaData, + checkpointException); + + currentState = CheckpointingOperation.AsyncCheckpointState.DISCARDED; + } else { + currentState = asyncCheckpointState.get(); + } + } + + if (!didCleanup) { + LOG.trace("Caught followup exception from a failed checkpoint thread. This can be ignored.", e); + } } @Override public void close() { - try { - cleanup(); - } catch (Exception cleanupException) { - LOG.warn("Could not properly clean up the async checkpoint runnable.", cleanupException); + if (asyncCheckpointState.compareAndSet( + CheckpointingOperation.AsyncCheckpointState.RUNNING, + CheckpointingOperation.AsyncCheckpointState.DISCARDED)) { + + try { + cleanup(); + } catch (Exception cleanupException) { + LOG.warn("Could not properly clean up the async checkpoint runnable.", cleanupException); + } + } else { + logFailedCleanupAttempt(); } } private void cleanup() throws Exception { - if (asyncCheckpointState.compareAndSet(CheckpointingOperation.AsynCheckpointState.RUNNING, CheckpointingOperation.AsynCheckpointState.DISCARDED)) { - LOG.debug("Cleanup AsyncCheckpointRunnable for checkpoint {} of {}.", checkpointMetaData.getCheckpointId(), owner.getName()); - Exception exception = null; + LOG.debug( + "Cleanup AsyncCheckpointRunnable for checkpoint {} of {}.", + checkpointMetaData.getCheckpointId(), + owner.getName()); - // clean up ongoing operator snapshot results and non partitioned state handles - for (OperatorSnapshotFutures operatorSnapshotResult : operatorSnapshotsInProgress.values()) { - if (operatorSnapshotResult != null) { - try { - operatorSnapshotResult.cancel(); - } catch (Exception cancelException) { - exception = ExceptionUtils.firstOrSuppressed(cancelException, exception); - } + Exception exception = null; + + // clean up ongoing operator snapshot results and non partitioned state handles + for (OperatorSnapshotFutures operatorSnapshotResult : operatorSnapshotsInProgress.values()) { + if (operatorSnapshotResult != null) { + try { + operatorSnapshotResult.cancel(); + } catch (Exception cancelException) { + exception = ExceptionUtils.firstOrSuppressed(cancelException, exception); } } + } - if (null != exception) { - throw exception; - } - } else { - LOG.debug("{} - asynchronous checkpointing operation for checkpoint {} has " + - "already been completed. Thus, the state handles are not cleaned up.", - owner.getName(), - checkpointMetaData.getCheckpointId()); + if (null != exception) { + throw exception; } } + + private void logFailedCleanupAttempt() { + LOG.debug("{} - asynchronous checkpointing operation for checkpoint {} has " + + "already been completed. Thus, the state handles are not cleaned up.", + owner.getName(), + checkpointMetaData.getCheckpointId()); + } } public CloseableRegistry getCancelables() { @@ -1088,7 +1117,7 @@ private void checkpointStreamOperator(StreamOperator op) throws Exception { } } - private enum AsynCheckpointState { + private enum AsyncCheckpointState { RUNNING, DISCARDED, COMPLETED diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java index 62a903bafb0e34..e5558f6d246204 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTerminationTest.java @@ -75,6 +75,7 @@ import org.apache.flink.util.SerializedValue; import org.apache.flink.util.TestLogger; +import org.junit.Assert; import org.junit.Test; import java.io.IOException; @@ -82,6 +83,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; @@ -97,7 +99,6 @@ public class StreamTaskTerminationTest extends TestLogger { public static final OneShotLatch RUN_LATCH = new OneShotLatch(); public static final OneShotLatch CHECKPOINTING_LATCH = new OneShotLatch(); private static final OneShotLatch CLEANUP_LATCH = new OneShotLatch(); - private static final OneShotLatch HANDLE_ASYNC_EXCEPTION_LATCH = new OneShotLatch(); /** * FLINK-6833 @@ -209,8 +210,7 @@ public BlockingStreamTask(Environment env) { } @Override - protected void init() throws Exception { - + protected void init() { } @Override @@ -226,24 +226,16 @@ protected void cleanup() throws Exception { // has been stopped CLEANUP_LATCH.trigger(); - // wait until handle async exception has been called to proceed with the termination of the - // StreamTask - HANDLE_ASYNC_EXCEPTION_LATCH.await(); + // wait until all async checkpoint threads are terminated, so that no more exceptions can be reported + Assert.assertTrue(getAsyncOperationsThreadPool().awaitTermination(30L, TimeUnit.SECONDS)); } @Override - protected void cancelTask() throws Exception { - } - - @Override - public void handleAsyncException(String message, Throwable exception) { - super.handleAsyncException(message, exception); - - HANDLE_ASYNC_EXCEPTION_LATCH.trigger(); + protected void cancelTask() { } } - static class NoOpStreamOperator extends AbstractStreamOperator { + private static class NoOpStreamOperator extends AbstractStreamOperator { private static final long serialVersionUID = 4517845269225218312L; } @@ -252,7 +244,7 @@ static class BlockingStateBackend implements StateBackend { private static final long serialVersionUID = -5053068148933314100L; @Override - public CompletedCheckpointStorageLocation resolveCheckpoint(String pointer) throws IOException { + public CompletedCheckpointStorageLocation resolveCheckpoint(String pointer) { throw new UnsupportedOperationException(); } @@ -269,7 +261,7 @@ public AbstractKeyedStateBackend createKeyedStateBackend( TypeSerializer keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, - TaskKvStateRegistry kvStateRegistry) throws IOException { + TaskKvStateRegistry kvStateRegistry) { return null; } From 0f2711618f543248a5dcd6ba8e3c4dc2b55fa568 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Wed, 21 Feb 2018 15:58:50 +0100 Subject: [PATCH 0005/2294] [FLINK-8699][checkpointing] Create deep copy of state meta data to avoid concurrency problem with checkpoints --- .../streaming/state/RocksDBKeyedStateBackend.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 0cb2792f0b2e5c..3accbe52daaaab 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -1818,6 +1818,7 @@ static class RocksDBFullSnapshotOperation private Snapshot snapshot; private ReadOptions readOptions; + private List>> kvStateInformationCopy; private List> kvStateIterators; private CheckpointStreamWithResultProvider checkpointStreamWithResultProvider; @@ -1841,7 +1842,7 @@ static class RocksDBFullSnapshotOperation */ public void takeDBSnapShot() { Preconditions.checkArgument(snapshot == null, "Only one ongoing snapshot allowed!"); - this.kvStateIterators = new ArrayList<>(stateBackend.kvStateInformation.size()); + this.kvStateInformationCopy = new ArrayList<>(stateBackend.kvStateInformation.values()); this.snapshot = stateBackend.db.getSnapshot(); } @@ -1928,20 +1929,22 @@ public void releaseSnapshotResources() { private void writeKVStateMetaData() throws IOException { List> metaInfoSnapshots = - new ArrayList<>(stateBackend.kvStateInformation.size()); + new ArrayList<>(kvStateInformationCopy.size()); + + this.kvStateIterators = new ArrayList<>(kvStateInformationCopy.size()); int kvStateId = 0; - for (Map.Entry>> column : - stateBackend.kvStateInformation.entrySet()) { + for (Tuple2> column : + kvStateInformationCopy) { - metaInfoSnapshots.add(column.getValue().f1.snapshot()); + metaInfoSnapshots.add(column.f1.snapshot()); //retrieve iterator for this k/v states readOptions = new ReadOptions(); readOptions.setSnapshot(snapshot); kvStateIterators.add( - new Tuple2<>(stateBackend.db.newIterator(column.getValue().f0, readOptions), kvStateId)); + new Tuple2<>(stateBackend.db.newIterator(column.f0, readOptions), kvStateId)); ++kvStateId; } From ca523fd556d3c8411ca550e20087d18fd3e72a0c Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Wed, 17 Jan 2018 13:05:17 +0100 Subject: [PATCH 0006/2294] [hotfix] RocksDB make default column family first According to the documentation of RocksDB, the default column family should always be created first. --- .../state/RocksDBKeyedStateBackend.java | 52 +++++++++--------- .../state/RocksDBMergeIteratorTest.java | 54 ++++++++++--------- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 3accbe52daaaab..c02f130a791ac0 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -156,9 +156,6 @@ public class RocksDBKeyedStateBackend extends AbstractKeyedStateBackend { /** File suffix of sstable files. */ private static final String SST_FILE_SUFFIX = ".sst"; - /** Bytes for the name of the column descriptor for the default column family. */ - public static final byte[] DEFAULT_COLUMN_FAMILY_NAME_BYTES = "default".getBytes(ConfigConstants.DEFAULT_CHARSET); - /** String that identifies the operator that owns this backend. */ private final String operatorIdentifier; @@ -349,29 +346,31 @@ public void dispose() { if (db != null) { // RocksDB's native memory management requires that *all* CFs (including default) are closed before the - // DB is closed. So we start with the ones created by Flink... + // DB is closed. See: + // https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families + // Start with default CF ... + IOUtils.closeQuietly(defaultColumnFamily); + + // ... continue with the ones created by Flink... for (Tuple2> columnMetaData : kvStateInformation.values()) { IOUtils.closeQuietly(columnMetaData.f0); } - // ... close the default CF ... - IOUtils.closeQuietly(defaultColumnFamily); - // ... and finally close the DB instance ... IOUtils.closeQuietly(db); - // invalidate the reference before releasing the lock so that other accesses will not cause crashes + // invalidate the reference db = null; - } - kvStateInformation.clear(); - restoredKvStateMetaInfos.clear(); + kvStateInformation.clear(); + restoredKvStateMetaInfos.clear(); - IOUtils.closeQuietly(dbOptions); - IOUtils.closeQuietly(columnOptions); + IOUtils.closeQuietly(dbOptions); + IOUtils.closeQuietly(columnOptions); - cleanInstanceBasePath(); + cleanInstanceBasePath(); + } } private void cleanInstanceBasePath() { @@ -475,11 +474,11 @@ private RocksDB openDB( List columnFamilyDescriptors = new ArrayList<>(1 + stateColumnFamilyDescriptors.size()); + // we add the required descriptor for the default CF in FIRST position, see + // https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnOptions)); columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors); - // we add the required descriptor for the default CF in last position. - columnFamilyDescriptors.add(new ColumnFamilyDescriptor(DEFAULT_COLUMN_FAMILY_NAME_BYTES, columnOptions)); - RocksDB dbRef; try { @@ -602,7 +601,6 @@ private void restoreKVStateMetaData() throws IOException, StateMigrationExceptio List> restoredMetaInfos = serializationProxy.getStateMetaInfoSnapshots(); currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size()); - //rocksDBKeyedStateBackend.restoredKvStateMetaInfos = new HashMap<>(restoredMetaInfos.size()); for (RegisteredKeyedBackendStateMetaInfo.Snapshot restoredMetaInfo : restoredMetaInfos) { @@ -845,8 +843,8 @@ private void restoreLocalStateIntoFullInstance( stateBackend.instanceRocksDBPath.getAbsolutePath(), columnFamilyDescriptors, columnFamilyHandles); - // extract and store the default column family which is located at the last index - stateBackend.defaultColumnFamily = columnFamilyHandles.remove(columnFamilyHandles.size() - 1); + // extract and store the default column family which is located at the first index + stateBackend.defaultColumnFamily = columnFamilyHandles.remove(0); for (int i = 0; i < columnFamilyDescriptors.size(); ++i) { RegisteredKeyedBackendStateMetaInfo.Snapshot stateMetaInfoSnapshot = stateMetaInfoSnapshots.get(i); @@ -1027,8 +1025,11 @@ private void restoreKeyGroupsShardWithTemporaryHelperInstance( columnFamilyDescriptors, columnFamilyHandles)) { + final ColumnFamilyHandle defaultColumnFamily = columnFamilyHandles.remove(0); + + Preconditions.checkState(columnFamilyHandles.size() == columnFamilyDescriptors.size()); + try { - // iterating only the requested descriptors automatically skips the default column family handle for (int i = 0; i < columnFamilyDescriptors.size(); ++i) { ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i); ColumnFamilyDescriptor columnFamilyDescriptor = columnFamilyDescriptors.get(i); @@ -1085,9 +1086,12 @@ private void restoreKeyGroupsShardWithTemporaryHelperInstance( } // releases native iterator resources } } finally { + //release native tmp db column family resources - for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandles) { - IOUtils.closeQuietly(columnFamilyHandle); + IOUtils.closeQuietly(defaultColumnFamily); + + for (ColumnFamilyHandle flinkColumnFamilyHandle : columnFamilyHandles) { + IOUtils.closeQuietly(flinkColumnFamilyHandle); } } } // releases native tmp db resources @@ -1165,7 +1169,7 @@ protected ColumnFamilyHandle getColumnFamily( } byte[] nameBytes = descriptor.getName().getBytes(ConfigConstants.DEFAULT_CHARSET); - Preconditions.checkState(!Arrays.equals(DEFAULT_COLUMN_FAMILY_NAME_BYTES, nameBytes), + Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!"); ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java index 1d14f6e92fe72e..19f49f841d1287 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/RocksDBMergeIteratorTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; +import org.apache.flink.util.IOUtils; import org.junit.Assert; import org.junit.Rule; @@ -53,7 +54,7 @@ public class RocksDBMergeIteratorTest { @Test public void testEmptyMergeIterator() throws IOException { RocksDBKeyedStateBackend.RocksDBMergeIterator emptyIterator = - new RocksDBKeyedStateBackend.RocksDBMergeIterator(Collections.EMPTY_LIST, 2); + new RocksDBKeyedStateBackend.RocksDBMergeIterator(Collections.emptyList(), 2); Assert.assertFalse(emptyIterator.isValid()); } @@ -74,8 +75,7 @@ public void testMergeIteratorShort() throws Exception { public void testMergeIterator(int maxParallelism) throws Exception { Random random = new Random(1234); - RocksDB rocksDB = RocksDB.open(tempFolder.getRoot().getAbsolutePath()); - try { + try (RocksDB rocksDB = RocksDB.open(tempFolder.getRoot().getAbsolutePath())) { List> rocksIteratorsWithKVStateId = new ArrayList<>(); List> columnFamilyHandlesWithKeyCount = new ArrayList<>(); @@ -83,7 +83,7 @@ public void testMergeIterator(int maxParallelism) throws Exception { for (int c = 0; c < NUM_KEY_VAL_STATES; ++c) { ColumnFamilyHandle handle = rocksDB.createColumnFamily( - new ColumnFamilyDescriptor(("column-" + c).getBytes(ConfigConstants.DEFAULT_CHARSET))); + new ColumnFamilyDescriptor(("column-" + c).getBytes(ConfigConstants.DEFAULT_CHARSET))); ByteArrayOutputStreamWithPos bos = new ByteArrayOutputStreamWithPos(); DataOutputStream dos = new DataOutputStream(bos); @@ -113,39 +113,41 @@ public void testMergeIterator(int maxParallelism) throws Exception { ++id; } - RocksDBKeyedStateBackend.RocksDBMergeIterator mergeIterator = new RocksDBKeyedStateBackend.RocksDBMergeIterator(rocksIteratorsWithKVStateId, maxParallelism <= Byte.MAX_VALUE ? 1 : 2); + try (RocksDBKeyedStateBackend.RocksDBMergeIterator mergeIterator = new RocksDBKeyedStateBackend.RocksDBMergeIterator( + rocksIteratorsWithKVStateId, + maxParallelism <= Byte.MAX_VALUE ? 1 : 2)) { - int prevKVState = -1; - int prevKey = -1; - int prevKeyGroup = -1; - int totalKeysActual = 0; + int prevKVState = -1; + int prevKey = -1; + int prevKeyGroup = -1; + int totalKeysActual = 0; - while (mergeIterator.isValid()) { - ByteBuffer bb = ByteBuffer.wrap(mergeIterator.key()); + while (mergeIterator.isValid()) { + ByteBuffer bb = ByteBuffer.wrap(mergeIterator.key()); - int keyGroup = maxParallelism > Byte.MAX_VALUE ? bb.getShort() : bb.get(); - int key = bb.getInt(); + int keyGroup = maxParallelism > Byte.MAX_VALUE ? bb.getShort() : bb.get(); + int key = bb.getInt(); - Assert.assertTrue(keyGroup >= prevKeyGroup); - Assert.assertTrue(key >= prevKey); - Assert.assertEquals(prevKeyGroup != keyGroup, mergeIterator.isNewKeyGroup()); - Assert.assertEquals(prevKVState != mergeIterator.kvStateId(), mergeIterator.isNewKeyValueState()); + Assert.assertTrue(keyGroup >= prevKeyGroup); + Assert.assertTrue(key >= prevKey); + Assert.assertEquals(prevKeyGroup != keyGroup, mergeIterator.isNewKeyGroup()); + Assert.assertEquals(prevKVState != mergeIterator.kvStateId(), mergeIterator.isNewKeyValueState()); - prevKeyGroup = keyGroup; - prevKVState = mergeIterator.kvStateId(); + prevKeyGroup = keyGroup; + prevKVState = mergeIterator.kvStateId(); - //System.out.println(keyGroup + " " + key + " " + mergeIterator.kvStateId()); - mergeIterator.next(); - ++totalKeysActual; + mergeIterator.next(); + ++totalKeysActual; + } + + Assert.assertEquals(totalKeysExpected, totalKeysActual); } - Assert.assertEquals(totalKeysExpected, totalKeysActual); + IOUtils.closeQuietly(rocksDB.getDefaultColumnFamily()); for (Tuple2 handleWithCount : columnFamilyHandlesWithKeyCount) { - rocksDB.dropColumnFamily(handleWithCount.f0); + IOUtils.closeQuietly(handleWithCount.f0); } - } finally { - rocksDB.close(); } } From c2c4f492fd44f6252e33f0b63b865ac07d51b51c Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Thu, 25 Jan 2018 23:02:20 +0100 Subject: [PATCH 0007/2294] [hotfix] RocksDB improve resource cleanup (disposal order, dispose all WriteOptions) This commit ensures that all WriteOption objects are closed and that we do not create unessesary WriteOption objects for each state. --- .../streaming/state/AbstractRocksDBState.java | 9 +++----- .../state/RocksDBAggregatingState.java | 10 --------- .../streaming/state/RocksDBFoldingState.java | 10 --------- .../state/RocksDBKeyedStateBackend.java | 21 +++++++++++++++---- .../streaming/state/RocksDBListState.java | 10 --------- .../streaming/state/RocksDBMapState.java | 10 --------- .../streaming/state/RocksDBReducingState.java | 10 --------- .../streaming/state/RocksDBValueState.java | 10 --------- 8 files changed, 20 insertions(+), 70 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java index 6db0e8611fb4e2..346435502ef592 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java @@ -64,12 +64,10 @@ public abstract class AbstractRocksDBState /** User-specified aggregation function. */ private final AggregateFunction aggFunction; - /** - * We disable writes to the write-ahead-log here. We can't have these in the base class - * because JNI segfaults for some reason if they are. - */ - private final WriteOptions writeOptions; - /** * Creates a new {@code RocksDBFoldingState}. * @@ -77,9 +70,6 @@ public RocksDBAggregatingState( this.valueSerializer = stateDesc.getSerializer(); this.aggFunction = stateDesc.getAggregateFunction(); - - writeOptions = new WriteOptions(); - writeOptions.setDisableWAL(true); } @Override diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java index 479565e359e484..d886f4460d4d2c 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBFoldingState.java @@ -29,7 +29,6 @@ import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; -import org.rocksdb.WriteOptions; import java.io.IOException; @@ -54,12 +53,6 @@ public class RocksDBFoldingState /** User-specified fold function. */ private final FoldFunction foldFunction; - /** - * We disable writes to the write-ahead-log here. We can't have these in the base class - * because JNI segfaults for some reason if they are. - */ - private final WriteOptions writeOptions; - /** * Creates a new {@code RocksDBFoldingState}. * @@ -76,9 +69,6 @@ public RocksDBFoldingState(ColumnFamilyHandle columnFamily, this.valueSerializer = stateDesc.getSerializer(); this.foldFunction = stateDesc.getFoldFunction(); - - writeOptions = new WriteOptions(); - writeOptions.setDisableWAL(true); } @Override diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index c02f130a791ac0..8f95b1812d844b 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -102,6 +102,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.Snapshot; +import org.rocksdb.WriteOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -120,6 +121,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; @@ -190,6 +192,11 @@ public class RocksDBKeyedStateBackend extends AbstractKeyedStateBackend { */ private ColumnFamilyHandle defaultColumnFamily; + /** + * The write options to use in the states. We disable write ahead logging. + */ + private final WriteOptions writeOptions; + /** * Information about the k/v states as we create them. This is used to retrieve the * column family that is used for a state and also for sanity checks when restoring. @@ -266,7 +273,7 @@ public RocksDBKeyedStateBackend( this.localRecoveryConfig = Preconditions.checkNotNull(localRecoveryConfig); this.keyGroupPrefixBytes = getNumberOfKeyGroups() > (Byte.MAX_VALUE + 1) ? 2 : 1; - this.kvStateInformation = new HashMap<>(); + this.kvStateInformation = new LinkedHashMap<>(); this.restoredKvStateMetaInfos = new HashMap<>(); this.materializedSstFiles = new TreeMap<>(); this.backendUID = UUID.randomUUID(); @@ -275,6 +282,8 @@ public RocksDBKeyedStateBackend( new IncrementalSnapshotStrategy() : new FullSnapshotStrategy(); + this.writeOptions = new WriteOptions().setDisableWAL(true); + LOG.debug("Setting initial keyed backend uid for operator {} to {}.", this.operatorIdentifier, this.backendUID); } @@ -363,12 +372,12 @@ public void dispose() { // invalidate the reference db = null; + IOUtils.closeQuietly(columnOptions); + IOUtils.closeQuietly(dbOptions); + IOUtils.closeQuietly(writeOptions); kvStateInformation.clear(); restoredKvStateMetaInfos.clear(); - IOUtils.closeQuietly(dbOptions); - IOUtils.closeQuietly(columnOptions); - cleanInstanceBasePath(); } } @@ -387,6 +396,10 @@ public int getKeyGroupPrefixBytes() { return keyGroupPrefixBytes; } + public WriteOptions getWriteOptions() { + return writeOptions; + } + /** * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java index f0481ec45be960..62c169b059e613 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBListState.java @@ -28,7 +28,6 @@ import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; -import org.rocksdb.WriteOptions; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -54,12 +53,6 @@ public class RocksDBListState /** Serializer for the values. */ private final TypeSerializer valueSerializer; - /** - * We disable writes to the write-ahead-log here. We can't have these in the base class - * because JNI segfaults for some reason if they are. - */ - private final WriteOptions writeOptions; - /** * Separator of StringAppendTestOperator in RocksDB. */ @@ -79,9 +72,6 @@ public RocksDBListState(ColumnFamilyHandle columnFamily, super(columnFamily, namespaceSerializer, stateDesc, backend); this.valueSerializer = stateDesc.getElementSerializer(); - - writeOptions = new WriteOptions(); - writeOptions.setDisableWAL(true); } @Override diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java index 6b7177b3c42e58..d1e72c98cef2e1 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java @@ -35,7 +35,6 @@ import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.WriteOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,12 +67,6 @@ public class RocksDBMapState private final TypeSerializer userKeySerializer; private final TypeSerializer userValueSerializer; - /** - * We disable writes to the write-ahead-log here. We can't have these in the base class - * because JNI segfaults for some reason if they are. - */ - private final WriteOptions writeOptions; - /** The offset of User Key offset in raw key bytes. */ private int userKeyOffset; @@ -92,9 +85,6 @@ public RocksDBMapState(ColumnFamilyHandle columnFamily, this.userKeySerializer = stateDesc.getKeySerializer(); this.userValueSerializer = stateDesc.getValueSerializer(); - - writeOptions = new WriteOptions(); - writeOptions.setDisableWAL(true); } // ------------------------------------------------------------------------ diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java index b4c3f5141375d1..2a7f6e0284ad88 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBReducingState.java @@ -29,7 +29,6 @@ import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; -import org.rocksdb.WriteOptions; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -52,12 +51,6 @@ public class RocksDBReducingState /** User-specified reduce function. */ private final ReduceFunction reduceFunction; - /** - * We disable writes to the write-ahead-log here. We can't have these in the base class - * because JNI segfaults for some reason if they are. - */ - private final WriteOptions writeOptions; - /** * Creates a new {@code RocksDBReducingState}. * @@ -73,9 +66,6 @@ public RocksDBReducingState(ColumnFamilyHandle columnFamily, super(columnFamily, namespaceSerializer, stateDesc, backend); this.valueSerializer = stateDesc.getSerializer(); this.reduceFunction = stateDesc.getReduceFunction(); - - writeOptions = new WriteOptions(); - writeOptions.setDisableWAL(true); } @Override diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java index da21e8afb25850..99718bed0f6f2e 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBValueState.java @@ -27,7 +27,6 @@ import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; -import org.rocksdb.WriteOptions; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -46,12 +45,6 @@ public class RocksDBValueState /** Serializer for the values. */ private final TypeSerializer valueSerializer; - /** - * We disable writes to the write-ahead-log here. We can't have these in the base class - * because JNI segfaults for some reason if they are. - */ - private final WriteOptions writeOptions; - /** * Creates a new {@code RocksDBValueState}. * @@ -66,9 +59,6 @@ public RocksDBValueState(ColumnFamilyHandle columnFamily, super(columnFamily, namespaceSerializer, stateDesc, backend); this.valueSerializer = stateDesc.getSerializer(); - - writeOptions = new WriteOptions(); - writeOptions.setDisableWAL(true); } @Override From ff6662c979e7f8660a7f6039339d82a37abfc623 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Thu, 22 Feb 2018 10:15:37 +0100 Subject: [PATCH 0008/2294] [hotfix] Replace use of deprecated remove calls to RocksDB with delete --- .../flink/contrib/streaming/state/AbstractRocksDBState.java | 2 +- .../apache/flink/contrib/streaming/state/RocksDBMapState.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java index 346435502ef592..89152619b5a8ee 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/AbstractRocksDBState.java @@ -102,7 +102,7 @@ public void clear() { try { writeCurrentKeyWithGroupAndNamespace(); byte[] key = keySerializationStream.toByteArray(); - backend.db.remove(columnFamily, writeOptions, key); + backend.db.delete(columnFamily, writeOptions, key); } catch (IOException | RocksDBException e) { throw new RuntimeException("Error while removing entry from RocksDB", e); } diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java index d1e72c98cef2e1..af789ac4aace92 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBMapState.java @@ -123,7 +123,7 @@ public void putAll(Map map) throws IOException, RocksDBException { public void remove(UK userKey) throws IOException, RocksDBException { byte[] rawKeyBytes = serializeUserKeyWithCurrentKeyAndNamespace(userKey); - backend.db.remove(columnFamily, writeOptions, rawKeyBytes); + backend.db.delete(columnFamily, writeOptions, rawKeyBytes); } @Override @@ -339,7 +339,7 @@ public void remove() { rawValueBytes = null; try { - db.remove(columnFamily, writeOptions, rawKeyBytes); + db.delete(columnFamily, writeOptions, rawKeyBytes); } catch (RocksDBException e) { throw new RuntimeException("Error while removing data from RocksDB.", e); } From a8fc3b14697379fedaf4f6f9d4568e54228cb61f Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Thu, 22 Feb 2018 10:20:28 +0100 Subject: [PATCH 0009/2294] [hotfix] Update RocksDB version to 5.7.5 --- flink-state-backends/flink-statebackend-rocksdb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/pom.xml b/flink-state-backends/flink-statebackend-rocksdb/pom.xml index 16416c6593f995..c6a9f92081a2a8 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/pom.xml +++ b/flink-state-backends/flink-statebackend-rocksdb/pom.xml @@ -57,7 +57,7 @@ under the License. org.rocksdb rocksdbjni - 5.6.1 + 5.7.5 From f9a583b727c9aecbec3213b12266f1d598223400 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Fri, 23 Feb 2018 18:21:24 +0100 Subject: [PATCH 0010/2294] [hotfix] Clear interrupted flag in stream task cancellation We clear the interrupted flag before the cleanup code block of task cancellation. Otherwise, code that would like to wait until services are properly shutdown will always immediately return from calls that are supposed to be blocking waits. --- .../org/apache/flink/streaming/runtime/tasks/StreamTask.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java index 7ebbc7178b6532..55d0132ee5d941 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java @@ -348,6 +348,9 @@ public final void invoke() throws Exception { // clean up everything we initialized isRunning = false; + // clear the interrupted status so that we can wait for the following resource shutdowns to complete + Thread.interrupted(); + // stop all timers and threads if (timerService != null && !timerService.isTerminated()) { try { From bbb63531b2118dfde1584a7f61ee908d1188698d Mon Sep 17 00:00:00 2001 From: gyao Date: Sun, 25 Feb 2018 12:53:53 +0100 Subject: [PATCH 0011/2294] [FLINK-8776][flip6] Use correct port for job submission from Web UI. Use address of local WebMonitorEndpoint for the job submission from the Web UI. Rename TestingLeaderRetrievalService to SettableLeaderRetrievalService and move class out of test directory. This closes #5577. --- .../program/rest/RestClusterClient.java | 28 +++++++- .../client/program/ClientConnectionTest.java | 6 +- .../program/rest/RestClusterClientTest.java | 7 +- .../MesosFlinkResourceManagerTest.java | 4 +- .../MesosResourceManagerTest.java | 6 +- .../webmonitor/WebSubmissionExtension.java | 12 +++- .../SettableLeaderRetrievalService.java} | 28 ++++---- .../runtime/client/JobClientActorTest.java | 30 ++++----- .../clusterframework/ResourceManagerTest.java | 8 +-- .../runtime/dispatcher/DispatcherTest.java | 4 +- .../highavailability/ManualLeaderService.java | 14 ++-- .../jobmanager/JobManagerHARecoveryTest.java | 4 +- .../jobmaster/JobManagerRunnerTest.java | 4 +- .../runtime/jobmaster/JobMasterTest.java | 6 +- .../SettableLeaderRetrievalServiceTest.java | 67 +++++++++++++++++++ .../JobLeaderIdServiceTest.java | 10 +-- .../ResourceManagerJobMasterTest.java | 12 ++-- .../taskexecutor/TaskExecutorITCase.java | 6 +- .../taskexecutor/TaskExecutorTest.java | 10 +-- .../TaskManagerRegistrationTest.java | 13 ++-- .../runtime/taskmanager/TaskManagerTest.java | 4 +- .../retriever/LeaderGatewayRetrieverTest.java | 8 +-- .../impl/AkkaJobManagerRetrieverTest.java | 10 +-- .../impl/RpcGatewayRetrieverTest.java | 10 +-- .../java/org/apache/flink/yarn/UtilsTest.java | 4 +- 25 files changed, 211 insertions(+), 104 deletions(-) rename flink-runtime/src/{test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java => main/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalService.java} (65%) create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalServiceTest.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 8ad571f3758178..3a377f3ef49b14 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -142,11 +142,29 @@ public RestClusterClient(Configuration config, T clusterId) throws Exception { config, null, clusterId, - new ExponentialWaitStrategy(10L, 2000L)); + new ExponentialWaitStrategy(10L, 2000L), + null); + } + + public RestClusterClient( + Configuration config, + T clusterId, + LeaderRetrievalService webMonitorRetrievalService) throws Exception { + this( + config, + null, + clusterId, + new ExponentialWaitStrategy(10L, 2000L), + webMonitorRetrievalService); } @VisibleForTesting - RestClusterClient(Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy) throws Exception { + RestClusterClient( + Configuration configuration, + @Nullable RestClient restClient, + T clusterId, + WaitStrategy waitStrategy, + @Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception { super(configuration); this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration); @@ -159,7 +177,11 @@ public RestClusterClient(Configuration config, T clusterId) throws Exception { this.waitStrategy = Preconditions.checkNotNull(waitStrategy); this.clusterId = Preconditions.checkNotNull(clusterId); - this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever(); + if (webMonitorRetrievalService == null) { + this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever(); + } else { + this.webMonitorRetrievalService = webMonitorRetrievalService; + } this.dispatcherRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever(); this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry")); startLeaderRetrievers(); diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java index 72767c7ba8f41b..1dd4787c12d2a3 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java @@ -27,8 +27,8 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.instance.ActorGateway; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.util.NetUtils; import org.apache.flink.util.TestLogger; @@ -136,9 +136,9 @@ public void testJobManagerRetrievalWithHAServices() throws Exception { final String expectedAddress = AkkaUtils.getAkkaURL(actorSystem, actorRef); - final TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService(expectedAddress, leaderId); + final SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService(expectedAddress, leaderId); - highAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, testingLeaderRetrievalService); + highAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, settableLeaderRetrievalService); StandaloneClusterClient client = new StandaloneClusterClient(configuration, highAvailabilityServices); diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index f587e33f6c1a54..ca2ba223ebeefa 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -182,7 +182,12 @@ public void setUp() throws Exception { } } }; - restClusterClient = new RestClusterClient<>(config, restClient, StandaloneClusterId.getInstance(), (attempt) -> 0); + restClusterClient = new RestClusterClient<>( + config, + restClient, + StandaloneClusterId.getInstance(), + (attempt) -> 0, + null); jobGraph = new JobGraph("testjob"); jobId = jobGraph.getJobID(); diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java index 56735effcab3da..330a2c60df3b95 100644 --- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java +++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosFlinkResourceManagerTest.java @@ -50,8 +50,8 @@ import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.instance.AkkaActorGateway; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.util.TestLogger; @@ -213,7 +213,7 @@ public Context() { highAvailabilityServices.setJobMasterLeaderRetriever( HighAvailabilityServices.DEFAULT_JOB_ID, - new TestingLeaderRetrievalService( + new SettableLeaderRetrievalService( jobManager.path(), HighAvailabilityServices.DEFAULT_LEADER_ID)); diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java index 2b38b8587b69ac..412e18da65a30b 100644 --- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java +++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java @@ -53,7 +53,7 @@ import org.apache.flink.runtime.jobmaster.JobMasterId; import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.MetricRegistryImpl; import org.apache.flink.runtime.registration.RegistrationResponse; @@ -412,7 +412,7 @@ class MockJobMaster { public final String address; public final JobMasterGateway gateway; public final JobMasterId jobMasterId; - public final TestingLeaderRetrievalService leaderRetrievalService; + public final SettableLeaderRetrievalService leaderRetrievalService; MockJobMaster(JobID jobID) { this.jobID = jobID; @@ -420,7 +420,7 @@ class MockJobMaster { this.address = "/" + jobID; this.gateway = mock(JobMasterGateway.class); this.jobMasterId = JobMasterId.generate(); - this.leaderRetrievalService = new TestingLeaderRetrievalService(this.address, this.jobMasterId.toUUID()); + this.leaderRetrievalService = new SettableLeaderRetrievalService(this.address, this.jobMasterId.toUUID()); } } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java index e0ac2500e9ab58..df36483c4e4d3f 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java @@ -22,6 +22,8 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.highavailability.HighAvailabilityServices; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.rest.handler.RestHandlerSpecification; import org.apache.flink.runtime.webmonitor.handlers.JarDeleteHandler; import org.apache.flink.runtime.webmonitor.handlers.JarDeleteHeaders; @@ -62,7 +64,15 @@ public WebSubmissionExtension( Executor executor, Time timeout) throws Exception { - restClusterClient = new RestClusterClient<>(configuration, "WebSubmissionHandlers"); + final SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService(); + restAddressFuture.thenAccept(restAddress -> settableLeaderRetrievalService.notifyListener( + restAddress, + HighAvailabilityServices.DEFAULT_LEADER_ID)); + + restClusterClient = new RestClusterClient<>( + configuration, + "WebSubmissionHandlers", + settableLeaderRetrievalService); webSubmissionHandlers = new ArrayList<>(3); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalService.java similarity index 65% rename from flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalService.java index 15d3bde75042de..1f2711a6aa7f50 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderRetrievalService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalService.java @@ -16,36 +16,38 @@ * limitations under the License. */ -package org.apache.flink.runtime.leaderelection; +package org.apache.flink.runtime.leaderretrieval; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.util.Preconditions; +import javax.annotation.Nullable; + import java.util.UUID; /** - * Test {@link LeaderRetrievalService} implementation which directly forwards calls of + * {@link LeaderRetrievalService} implementation which directly forwards calls of * notifyListener to the listener. */ -public class TestingLeaderRetrievalService implements LeaderRetrievalService { +public class SettableLeaderRetrievalService implements LeaderRetrievalService { - private volatile String leaderAddress; - private volatile UUID leaderSessionID; + private String leaderAddress; + private UUID leaderSessionID; - private volatile LeaderRetrievalListener listener; + private LeaderRetrievalListener listener; - public TestingLeaderRetrievalService() { + public SettableLeaderRetrievalService() { this(null, null); } - public TestingLeaderRetrievalService(String leaderAddress, UUID leaderSessionID) { + public SettableLeaderRetrievalService( + @Nullable String leaderAddress, + @Nullable UUID leaderSessionID) { this.leaderAddress = leaderAddress; this.leaderSessionID = leaderSessionID; } @Override - public void start(LeaderRetrievalListener listener) throws Exception { + public synchronized void start(LeaderRetrievalListener listener) throws Exception { this.listener = Preconditions.checkNotNull(listener); if (leaderSessionID != null && leaderAddress != null) { @@ -58,7 +60,9 @@ public void stop() throws Exception { } - public void notifyListener(String address, UUID leaderSessionID) { + public synchronized void notifyListener( + @Nullable String address, + @Nullable UUID leaderSessionID) { this.leaderAddress = address; this.leaderSessionID = leaderSessionID; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java index 919a784f388867..9050d12f7c7fa2 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/client/JobClientActorTest.java @@ -31,7 +31,7 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.JobClientMessages; import org.apache.flink.runtime.messages.JobClientMessages.AttachToJobAndWait; @@ -85,13 +85,13 @@ public void testSubmissionTimeout() throws Exception { PlainActor.class, leaderSessionID)); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( jobManager.path().toString(), leaderSessionID ); Props jobClientActorProps = JobSubmissionClientActor.createActorProps( - testingLeaderRetrievalService, + settableLeaderRetrievalService, jobClientActorTimeout, false, clientConfig); @@ -124,13 +124,13 @@ public void testRegistrationTimeout() throws Exception { PlainActor.class, leaderSessionID)); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( jobManager.path().toString(), leaderSessionID ); Props jobClientActorProps = JobAttachmentClientActor.createActorProps( - testingLeaderRetrievalService, + settableLeaderRetrievalService, jobClientActorTimeout, false); @@ -154,12 +154,12 @@ public void testConnectionTimeoutWithoutJobManagerForSubmission() throws Excepti FiniteDuration jobClientActorTimeout = new FiniteDuration(1L, TimeUnit.SECONDS); FiniteDuration timeout = jobClientActorTimeout.$times(2); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( "localhost", HighAvailabilityServices.DEFAULT_LEADER_ID); Props jobClientActorProps = JobSubmissionClientActor.createActorProps( - testingLeaderRetrievalService, + settableLeaderRetrievalService, jobClientActorTimeout, false, clientConfig); @@ -183,12 +183,12 @@ public void testConnectionTimeoutWithoutJobManagerForRegistration() throws Excep FiniteDuration jobClientActorTimeout = new FiniteDuration(1L, TimeUnit.SECONDS); FiniteDuration timeout = jobClientActorTimeout.$times(2); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( "localhost", HighAvailabilityServices.DEFAULT_LEADER_ID); Props jobClientActorProps = JobAttachmentClientActor.createActorProps( - testingLeaderRetrievalService, + settableLeaderRetrievalService, jobClientActorTimeout, false); @@ -219,13 +219,13 @@ public void testConnectionTimeoutAfterJobSubmission() throws Exception { JobAcceptingActor.class, leaderSessionID)); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( jobManager.path().toString(), leaderSessionID ); Props jobClientActorProps = JobSubmissionClientActor.createActorProps( - testingLeaderRetrievalService, + settableLeaderRetrievalService, jobClientActorTimeout, false, clientConfig); @@ -261,13 +261,13 @@ public void testConnectionTimeoutAfterJobRegistration() throws Exception { JobAcceptingActor.class, leaderSessionID)); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( jobManager.path().toString(), leaderSessionID ); Props jobClientActorProps = JobAttachmentClientActor.createActorProps( - testingLeaderRetrievalService, + settableLeaderRetrievalService, jobClientActorTimeout, false); @@ -302,13 +302,13 @@ public void testGuaranteedAnswerIfJobClientDies() throws Exception { JobAcceptingActor.class, leaderSessionID)); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( jobManager.path().toString(), leaderSessionID ); TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); - highAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, testingLeaderRetrievalService); + highAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, settableLeaderRetrievalService); JobListeningContext jobListeningContext = JobClient.submitJob( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java index 241da8f1bd6dbe..a1b3fb6acd79cb 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ResourceManagerTest.java @@ -42,7 +42,7 @@ import org.apache.flink.runtime.jobmaster.JobMasterId; import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.JobManagerMessages; import org.apache.flink.runtime.metrics.MetricRegistryImpl; @@ -107,7 +107,7 @@ public class ResourceManagerTest extends TestLogger { private final Time timeout = Time.seconds(10L); private TestingHighAvailabilityServices highAvailabilityServices; - private TestingLeaderRetrievalService jobManagerLeaderRetrievalService; + private SettableLeaderRetrievalService jobManagerLeaderRetrievalService; @BeforeClass public static void setup() { @@ -121,7 +121,7 @@ public static void teardown() { @Before public void setupTest() { - jobManagerLeaderRetrievalService = new TestingLeaderRetrievalService(); + jobManagerLeaderRetrievalService = new SettableLeaderRetrievalService(); highAvailabilityServices = new TestingHighAvailabilityServices(); @@ -602,7 +602,7 @@ public void testHeartbeatTimeoutWithJobManager() throws Exception { Time.seconds(5L)); final TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService(); - final TestingLeaderRetrievalService jmLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jobMasterId.toUUID()); + final SettableLeaderRetrievalService jmLeaderRetrievalService = new SettableLeaderRetrievalService(jobMasterAddress, jobMasterId.toUUID()); final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService); highAvailabilityServices.setJobMasterLeaderRetriever(jobId, jmLeaderRetrievalService); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java index 5d264e2010f99e..82679216205fc7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java @@ -42,7 +42,7 @@ import org.apache.flink.runtime.jobmaster.JobManagerSharedServices; import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.FlinkJobNotFoundException; import org.apache.flink.runtime.metrics.MetricRegistry; @@ -161,7 +161,7 @@ public void setUp() throws Exception { haServices.setSubmittedJobGraphStore(submittedJobGraphStore); haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService); haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory()); - haServices.setResourceManagerLeaderRetriever(new TestingLeaderRetrievalService()); + haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService()); runningJobsRegistry = haServices.getRunningJobsRegistry(); final Configuration blobServerConfig = new Configuration(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java index 423c0ceded065d..a055edfdd390de 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/ManualLeaderService.java @@ -20,7 +20,7 @@ import org.apache.flink.runtime.leaderelection.LeaderElectionService; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.util.Preconditions; @@ -32,13 +32,13 @@ /** * Leader service for {@link TestingManualHighAvailabilityServices} implementation. The leader * service allows to create multiple {@link TestingLeaderElectionService} and - * {@link TestingLeaderRetrievalService} and allows to manually trigger the services identified + * {@link SettableLeaderRetrievalService} and allows to manually trigger the services identified * by a continuous index. */ public class ManualLeaderService { private final List leaderElectionServices; - private final List leaderRetrievalServices; + private final List leaderRetrievalServices; private int currentLeaderIndex; @@ -54,13 +54,13 @@ public ManualLeaderService() { } public LeaderRetrievalService createLeaderRetrievalService() { - final TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + final SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( getLeaderAddress(currentLeaderIndex), currentLeaderId); - leaderRetrievalServices.add(testingLeaderRetrievalService); + leaderRetrievalServices.add(settableLeaderRetrievalService); - return testingLeaderRetrievalService; + return settableLeaderRetrievalService; } public LeaderElectionService createLeaderElectionService() { @@ -100,7 +100,7 @@ public void revokeLeadership() { } public void notifyRetrievers(int index, UUID leaderId) { - for (TestingLeaderRetrievalService retrievalService: leaderRetrievalServices) { + for (SettableLeaderRetrievalService retrievalService: leaderRetrievalServices) { retrievalService.notifyListener(getLeaderAddress(index), leaderId); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java index 005dd987078838..309ac1236d4c5e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java @@ -64,7 +64,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; import org.apache.flink.runtime.leaderelection.LeaderElectionService; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.JobManagerMessages; import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; @@ -184,7 +184,7 @@ public void testJobRecoveryWhenLosingLeadership() throws Exception { CheckpointIDCounter checkpointCounter = new StandaloneCheckpointIDCounter(); CheckpointRecoveryFactory checkpointStateFactory = new TestingCheckpointRecoveryFactory(checkpointStore, checkpointCounter); TestingLeaderElectionService myLeaderElectionService = new TestingLeaderElectionService(); - TestingLeaderRetrievalService myLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService myLeaderRetrievalService = new SettableLeaderRetrievalService( null, null); TestingHighAvailabilityServices testingHighAvailabilityServices = new TestingHighAvailabilityServices(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java index 0c238cece7dc32..9730ddef27e5a6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java @@ -31,7 +31,7 @@ import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder; @@ -113,7 +113,7 @@ public static void setupClass() throws Exception { public void setup() { haServices = new TestingHighAvailabilityServices(); haServices.setJobMasterLeaderElectionService(jobGraph.getJobID(), new TestingLeaderElectionService()); - haServices.setResourceManagerLeaderRetriever(new TestingLeaderRetrievalService()); + haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService()); haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory()); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java index e40102006b2932..b5430569e1913c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java @@ -45,7 +45,7 @@ import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings; import org.apache.flink.runtime.jobmanager.OnCompletionActions; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.ResourceManagerId; @@ -113,7 +113,7 @@ public class JobMasterTest extends TestLogger { private TestingHighAvailabilityServices haServices; - private TestingLeaderRetrievalService rmLeaderRetrievalService; + private SettableLeaderRetrievalService rmLeaderRetrievalService; private TestingFatalErrorHandler testingFatalErrorHandler; @@ -135,7 +135,7 @@ public void setup() throws IOException { haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory()); - rmLeaderRetrievalService = new TestingLeaderRetrievalService( + rmLeaderRetrievalService = new SettableLeaderRetrievalService( null, null); haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalServiceTest.java new file mode 100644 index 00000000000000..667008b2638d9e --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderretrieval/SettableLeaderRetrievalServiceTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.leaderretrieval; + +import org.apache.flink.runtime.highavailability.HighAvailabilityServices; +import org.apache.flink.runtime.leaderelection.TestingListener; +import org.apache.flink.util.TestLogger; + +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * Tests for {@link SettableLeaderRetrievalService}. + */ +public class SettableLeaderRetrievalServiceTest extends TestLogger { + + private SettableLeaderRetrievalService settableLeaderRetrievalService; + + @Before + public void setUp() { + settableLeaderRetrievalService = new SettableLeaderRetrievalService(); + } + + @Test + public void testNotifyListenerLater() throws Exception { + final String localhost = "localhost"; + settableLeaderRetrievalService.notifyListener(localhost, HighAvailabilityServices.DEFAULT_LEADER_ID); + + final TestingListener listener = new TestingListener(); + settableLeaderRetrievalService.start(listener); + + assertThat(listener.getAddress(), equalTo(localhost)); + assertThat(listener.getLeaderSessionID(), equalTo(HighAvailabilityServices.DEFAULT_LEADER_ID)); + } + + @Test + public void testNotifyListenerImmediately() throws Exception { + final TestingListener listener = new TestingListener(); + settableLeaderRetrievalService.start(listener); + + final String localhost = "localhost"; + settableLeaderRetrievalService.notifyListener(localhost, HighAvailabilityServices.DEFAULT_LEADER_ID); + + assertThat(listener.getAddress(), equalTo(localhost)); + assertThat(listener.getLeaderSessionID(), equalTo(HighAvailabilityServices.DEFAULT_LEADER_ID)); + } + +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdServiceTest.java index bb99a0ce4231db..301ab46c577e62 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdServiceTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/JobLeaderIdServiceTest.java @@ -23,7 +23,7 @@ import org.apache.flink.runtime.concurrent.ScheduledExecutor; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.jobmaster.JobMasterId; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.testutils.category.Flip6; import org.apache.flink.util.TestLogger; import org.junit.Test; @@ -68,7 +68,7 @@ public void testAddingJob() throws Exception { final String address = "foobar"; final JobMasterId leaderId = JobMasterId.generate(); TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); - TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService( null, null); @@ -104,7 +104,7 @@ public void testAddingJob() throws Exception { public void testRemovingJob() throws Exception { final JobID jobId = new JobID(); TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); - TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService(null, null); + SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(null, null); highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService); @@ -145,7 +145,7 @@ public void testRemovingJob() throws Exception { public void testInitialJobTimeout() throws Exception { final JobID jobId = new JobID(); TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); - TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService( null, null); @@ -189,7 +189,7 @@ public void jobTimeoutAfterLostLeadership() throws Exception { final String address = "foobar"; final JobMasterId leaderId = JobMasterId.generate(); TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices(); - TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService( null, null); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java index acd87748f7e68b..9854f5a3e2f4b2 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java @@ -31,7 +31,7 @@ import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess; import org.apache.flink.runtime.leaderelection.LeaderElectionService; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.runtime.metrics.MetricRegistryImpl; import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException; @@ -86,7 +86,7 @@ public void testRegisterJobMaster() throws Exception { JobID jobID = mockJobMaster(jobMasterAddress); JobMasterId jobMasterId = JobMasterId.generate(); final ResourceID jmResourceId = new ResourceID(jobMasterAddress); - TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jobMasterId.toUUID()); + SettableLeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService(jobMasterAddress, jobMasterId.toUUID()); TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService(); TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler(); final ResourceManager resourceManager = createAndStartResourceManager(resourceManagerLeaderElectionService, jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler); @@ -119,7 +119,7 @@ public void testRegisterJobMasterWithUnmatchedLeaderSessionId1() throws Exceptio JobID jobID = mockJobMaster(jobMasterAddress); JobMasterId jobMasterId = JobMasterId.generate(); final ResourceID jmResourceId = new ResourceID(jobMasterAddress); - TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService(jobMasterAddress, jobMasterId.toUUID()); + SettableLeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService(jobMasterAddress, jobMasterId.toUUID()); TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler(); final ResourceManager resourceManager = createAndStartResourceManager(mock(LeaderElectionService.class), jobID, jobMasterLeaderRetrievalService, testingFatalErrorHandler); final ResourceManagerGateway wronglyFencedGateway = rpcService.connect(resourceManager.getAddress(), ResourceManagerId.generate(), ResourceManagerGateway.class) @@ -153,7 +153,7 @@ public void testRegisterJobMasterWithUnmatchedLeaderSessionId2() throws Exceptio String jobMasterAddress = "/jobMasterAddress1"; JobID jobID = mockJobMaster(jobMasterAddress); TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService(); - TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService( "localhost", HighAvailabilityServices.DEFAULT_LEADER_ID); TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler(); @@ -187,7 +187,7 @@ public void testRegisterJobMasterFromInvalidAddress() throws Exception { String jobMasterAddress = "/jobMasterAddress1"; JobID jobID = mockJobMaster(jobMasterAddress); TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService(); - TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService( "localhost", HighAvailabilityServices.DEFAULT_LEADER_ID); TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler(); @@ -221,7 +221,7 @@ public void testRegisterJobMasterWithFailureLeaderListener() throws Exception { String jobMasterAddress = "/jobMasterAddress1"; JobID jobID = mockJobMaster(jobMasterAddress); TestingLeaderElectionService resourceManagerLeaderElectionService = new TestingLeaderElectionService(); - TestingLeaderRetrievalService jobMasterLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService( "localhost", HighAvailabilityServices.DEFAULT_LEADER_ID); TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java index d693f477affdf8..fc8337f086fcbd 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java @@ -35,7 +35,7 @@ import org.apache.flink.runtime.jobmaster.JobMasterId; import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.NoOpMetricRegistry; @@ -100,7 +100,7 @@ public void testSlotAllocation() throws Exception { final ResourceID taskManagerResourceId = new ResourceID("foobar"); final UUID rmLeaderId = UUID.randomUUID(); final TestingLeaderElectionService rmLeaderElectionService = new TestingLeaderElectionService(); - final TestingLeaderRetrievalService rmLeaderRetrievalService = new TestingLeaderRetrievalService(null, null); + final SettableLeaderRetrievalService rmLeaderRetrievalService = new SettableLeaderRetrievalService(null, null); final String rmAddress = "rm"; final String jmAddress = "jm"; final JobMasterId jobMasterId = JobMasterId.generate(); @@ -111,7 +111,7 @@ public void testSlotAllocation() throws Exception { testingHAServices.setResourceManagerLeaderElectionService(rmLeaderElectionService); testingHAServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService); - testingHAServices.setJobMasterLeaderRetriever(jobId, new TestingLeaderRetrievalService(jmAddress, jobMasterId.toUUID())); + testingHAServices.setJobMasterLeaderRetriever(jobId, new SettableLeaderRetrievalService(jmAddress, jobMasterId.toUUID())); TestingRpcService rpcService = new TestingRpcService(); ResourceManagerConfiguration resourceManagerConfiguration = new ResourceManagerConfiguration( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java index e972feb0c3bed0..7aae28704a0313 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorTest.java @@ -59,7 +59,7 @@ import org.apache.flink.runtime.jobmaster.JobMasterGateway; import org.apache.flink.runtime.jobmaster.JobMasterId; import org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGateway; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; @@ -165,9 +165,9 @@ public class TaskExecutorTest extends TestLogger { private TestingHighAvailabilityServices haServices; - private TestingLeaderRetrievalService resourceManagerLeaderRetriever; + private SettableLeaderRetrievalService resourceManagerLeaderRetriever; - private TestingLeaderRetrievalService jobManagerLeaderRetriever; + private SettableLeaderRetrievalService jobManagerLeaderRetriever; @Before public void setup() throws IOException { @@ -188,8 +188,8 @@ public void setup() throws IOException { testingFatalErrorHandler = new TestingFatalErrorHandler(); haServices = new TestingHighAvailabilityServices(); - resourceManagerLeaderRetriever = new TestingLeaderRetrievalService(); - jobManagerLeaderRetriever = new TestingLeaderRetrievalService(); + resourceManagerLeaderRetriever = new SettableLeaderRetrievalService(); + jobManagerLeaderRetriever = new SettableLeaderRetrievalService(); haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever); haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java index ceb92c42ffa32c..6b6509543f720f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerRegistrationTest.java @@ -24,7 +24,6 @@ import akka.actor.Terminated; import akka.testkit.JavaTestKit; import org.apache.flink.configuration.AkkaOptions; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.akka.AkkaUtils; @@ -38,7 +37,7 @@ import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.instance.AkkaActorGateway; import org.apache.flink.runtime.instance.InstanceID; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService; import org.apache.flink.runtime.messages.JobManagerMessages; import org.apache.flink.runtime.messages.JobManagerMessages.LeaderSessionMessage; @@ -273,7 +272,7 @@ public void testShutdownAfterRegistrationDurationExpired() { highAvailabilityServices.setJobMasterLeaderRetriever( HighAvailabilityServices.DEFAULT_JOB_ID, // Give a non-existent job manager address to the task manager - new TestingLeaderRetrievalService( + new SettableLeaderRetrievalService( "foobar", HighAvailabilityServices.DEFAULT_LEADER_ID)); @@ -330,7 +329,7 @@ public void testTaskManagerResumesConnectAfterRefusedRegistration() { highAvailabilityServices.setJobMasterLeaderRetriever( HighAvailabilityServices.DEFAULT_JOB_ID, - new TestingLeaderRetrievalService( + new SettableLeaderRetrievalService( jm.path(), HighAvailabilityServices.DEFAULT_LEADER_ID)); @@ -397,7 +396,7 @@ public void testTaskManagerNoExcessiveRegistrationMessages() throws Exception { highAvailabilityServices.setJobMasterLeaderRetriever( HighAvailabilityServices.DEFAULT_JOB_ID, - new TestingLeaderRetrievalService( + new SettableLeaderRetrievalService( jm.path(), HighAvailabilityServices.DEFAULT_LEADER_ID)); @@ -496,13 +495,13 @@ public void testTaskManagerResumesConnectAfterJobManagerFailure() { Option.apply(JOB_MANAGER_NAME)); final ActorGateway fakeJM1Gateway = fakeJobManager1Gateway; - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( fakeJM1Gateway.path(), HighAvailabilityServices.DEFAULT_LEADER_ID); highAvailabilityServices.setJobMasterLeaderRetriever( HighAvailabilityServices.DEFAULT_JOB_ID, - testingLeaderRetrievalService); + settableLeaderRetrievalService); // we make the test actor (the test kit) the JobManager to intercept // the messages diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java index 9d41bfb39000a1..93ec3adcb2a42c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java @@ -53,7 +53,7 @@ import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; import org.apache.flink.runtime.jobmanager.Tasks; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.RegistrationMessages; @@ -1529,7 +1529,7 @@ protected void run() { public void testTerminationOnFatalError() { highAvailabilityServices.setJobMasterLeaderRetriever( HighAvailabilityServices.DEFAULT_JOB_ID, - new TestingLeaderRetrievalService()); + new SettableLeaderRetrievalService()); new JavaTestKit(system){{ diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/LeaderGatewayRetrieverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/LeaderGatewayRetrieverTest.java index 7be06f354441ad..9fffc48f2ab06d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/LeaderGatewayRetrieverTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/LeaderGatewayRetrieverTest.java @@ -20,7 +20,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.concurrent.FutureUtils; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.rpc.RpcGateway; import org.apache.flink.util.FlinkException; import org.apache.flink.util.TestLogger; @@ -52,14 +52,14 @@ public void testGatewayRetrievalFailures() throws Exception { RpcGateway rpcGateway = mock(RpcGateway.class); TestingLeaderGatewayRetriever leaderGatewayRetriever = new TestingLeaderGatewayRetriever(rpcGateway); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService(); + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService(); - testingLeaderRetrievalService.start(leaderGatewayRetriever); + settableLeaderRetrievalService.start(leaderGatewayRetriever); CompletableFuture gatewayFuture = leaderGatewayRetriever.getFuture(); // this triggers the first gateway retrieval attempt - testingLeaderRetrievalService.notifyListener(address, leaderId); + settableLeaderRetrievalService.notifyListener(address, leaderId); // check that the first future has been failed try { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/AkkaJobManagerRetrieverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/AkkaJobManagerRetrieverTest.java index 5d01087a10990c..94473b9581c01b 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/AkkaJobManagerRetrieverTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/AkkaJobManagerRetrieverTest.java @@ -23,7 +23,7 @@ import org.apache.flink.runtime.client.JobClientActorTest; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.jobmaster.JobManagerGateway; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.util.TestLogger; @@ -70,7 +70,7 @@ public static void teardown() { @Test public void testAkkaJobManagerRetrieval() throws Exception { AkkaJobManagerRetriever akkaJobManagerRetriever = new AkkaJobManagerRetriever(actorSystem, timeout, 0, Time.milliseconds(0L)); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService(); + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService(); CompletableFuture gatewayFuture = akkaJobManagerRetriever.getFuture(); final UUID leaderSessionId = UUID.randomUUID(); @@ -83,18 +83,18 @@ public void testAkkaJobManagerRetrieval() throws Exception { final String address = actorRef.path().toString(); - testingLeaderRetrievalService.start(akkaJobManagerRetriever); + settableLeaderRetrievalService.start(akkaJobManagerRetriever); // check that the gateway future has not been completed since there is no leader yet assertFalse(gatewayFuture.isDone()); - testingLeaderRetrievalService.notifyListener(address, leaderSessionId); + settableLeaderRetrievalService.notifyListener(address, leaderSessionId); JobManagerGateway jobManagerGateway = gatewayFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS); assertEquals(address, jobManagerGateway.getAddress()); } finally { - testingLeaderRetrievalService.stop(); + settableLeaderRetrievalService.stop(); if (actorRef != null) { TestingUtils.stopActorGracefully(actorRef); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/RpcGatewayRetrieverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/RpcGatewayRetrieverTest.java index 5f59d59a98cc91..f4d66d1a6a0202 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/RpcGatewayRetrieverTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/retriever/impl/RpcGatewayRetrieverTest.java @@ -20,7 +20,7 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.rpc.FencedRpcGateway; import org.apache.flink.runtime.rpc.RpcEndpoint; import org.apache.flink.runtime.rpc.RpcService; @@ -74,7 +74,7 @@ public void testRpcGatewayRetrieval() throws Exception { final UUID leaderSessionId = UUID.randomUUID(); RpcGatewayRetriever gatewayRetriever = new RpcGatewayRetriever<>(rpcService, DummyGateway.class, Function.identity(), 0, Time.milliseconds(0L)); - TestingLeaderRetrievalService testingLeaderRetrievalService = new TestingLeaderRetrievalService(); + SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService(); DummyRpcEndpoint dummyRpcEndpoint = new DummyRpcEndpoint(rpcService, "dummyRpcEndpoint1", expectedValue); DummyRpcEndpoint dummyRpcEndpoint2 = new DummyRpcEndpoint(rpcService, "dummyRpcEndpoint2", expectedValue2); rpcService.registerGateway(dummyRpcEndpoint.getAddress(), dummyRpcEndpoint.getSelfGateway(DummyGateway.class)); @@ -84,13 +84,13 @@ public void testRpcGatewayRetrieval() throws Exception { dummyRpcEndpoint.start(); dummyRpcEndpoint2.start(); - testingLeaderRetrievalService.start(gatewayRetriever); + settableLeaderRetrievalService.start(gatewayRetriever); final CompletableFuture gatewayFuture = gatewayRetriever.getFuture(); assertFalse(gatewayFuture.isDone()); - testingLeaderRetrievalService.notifyListener(dummyRpcEndpoint.getAddress(), leaderSessionId); + settableLeaderRetrievalService.notifyListener(dummyRpcEndpoint.getAddress(), leaderSessionId); final DummyGateway dummyGateway = gatewayFuture.get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS); @@ -98,7 +98,7 @@ public void testRpcGatewayRetrieval() throws Exception { assertEquals(expectedValue, dummyGateway.foobar(TIMEOUT).get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS)); // elect a new leader - testingLeaderRetrievalService.notifyListener(dummyRpcEndpoint2.getAddress(), leaderSessionId); + settableLeaderRetrievalService.notifyListener(dummyRpcEndpoint2.getAddress(), leaderSessionId); final CompletableFuture gatewayFuture2 = gatewayRetriever.getFuture(); final DummyGateway dummyGateway2 = gatewayFuture2.get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS); diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/UtilsTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/UtilsTest.java index aea3a670be9a32..578e8e202011f3 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/UtilsTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/UtilsTest.java @@ -25,7 +25,7 @@ import org.apache.flink.runtime.clusterframework.messages.RegisterResourceManager; import org.apache.flink.runtime.clusterframework.messages.RegisterResourceManagerSuccessful; import org.apache.flink.runtime.instance.AkkaActorGateway; -import org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService; +import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.util.TestLogger; @@ -98,7 +98,7 @@ public void testYarnFlinkResourceManagerJobManagerLostLeadership() throws Except Configuration flinkConfig = new Configuration(); YarnConfiguration yarnConfig = new YarnConfiguration(); - TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService( + SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService( null, null); String applicationMasterHostName = "localhost"; From 6fba46e6af926bdb349a4e4b10d213385739040a Mon Sep 17 00:00:00 2001 From: gyao Date: Sun, 25 Feb 2018 13:23:41 +0100 Subject: [PATCH 0012/2294] [hotfix] Initialize webSubmissionHandlers list in WebSubmissionExtension with correct size. --- .../apache/flink/runtime/webmonitor/WebSubmissionExtension.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java index df36483c4e4d3f..bf3bc34ca71f83 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebSubmissionExtension.java @@ -74,7 +74,7 @@ public WebSubmissionExtension( "WebSubmissionHandlers", settableLeaderRetrievalService); - webSubmissionHandlers = new ArrayList<>(3); + webSubmissionHandlers = new ArrayList<>(5); final JarUploadHandler jarUploadHandler = new JarUploadHandler( restAddressFuture, From be2ecac40f2689e190a6021d763bf07d94373ba1 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Wed, 21 Feb 2018 20:20:51 +0100 Subject: [PATCH 0013/2294] [FLINK-8762] [quickstarts] Make 'StreamingJob' the default main class and remove WordCount example from the quickstart. The packaged example jobs have been reported to not be terribly helpful and simply create noise in the initial project setup. --- .../resources/archetype-resources/pom.xml | 6 +- .../src/main/java/BatchJob.java | 20 +--- .../main/java/SocketTextStreamWordCount.java | 108 ------------------ .../src/main/java/StreamingJob.java | 21 +--- .../src/main/java/WordCount.java | 94 --------------- .../resources/archetype-resources/pom.xml | 6 +- .../src/main/scala/BatchJob.scala | 25 ++-- .../scala/SocketTextStreamWordCount.scala | 69 ----------- .../src/main/scala/StreamingJob.scala | 24 ++-- .../src/main/scala/WordCount.scala | 53 --------- 10 files changed, 27 insertions(+), 399 deletions(-) delete mode 100644 flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java delete mode 100644 flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java delete mode 100644 flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala delete mode 100644 flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index ef900a18b9bba9..b78bf691736d46 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -189,15 +189,11 @@ under the License. - - diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java index d0e68a4b265bef..971192422b96dd 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java @@ -23,22 +23,12 @@ /** * Skeleton for a Flink Batch Job. * - *

For a full example of a Flink Batch Job, see the WordCountJob.java file in the - * same package/directory or have a look at the website. + *

For a tutorial how to write a Flink batch application, check the + * tutorials and examples on the Flink Website. * - *

You can also generate a .jar file that you can submit on your Flink - * cluster. - * Just type - * mvn clean package - * in the projects root directory. - * You will find the jar in - * target/${artifactId}-${version}.jar - * From the CLI you can then run - * ./bin/flink run -c ${package}.BatchJob target/${artifactId}-${version}.jar - * - *

For more information on the CLI see: - * - *

http://flink.apache.org/docs/latest/apis/cli.html + *

To package your appliation into a JAR file for execution, + * change the main class in the POM.xml file to this class (simply search for 'mainClass') + * and run 'mvn clean package' on the command line. */ public class BatchJob { diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java deleted file mode 100644 index 97df489afbc425..00000000000000 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java +++ /dev/null @@ -1,108 +0,0 @@ -package ${package}; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.flink.api.common.functions.FlatMapFunction; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.util.Collector; - -/** - * This example shows an implementation of WordCount with data from a text - * socket. To run the example make sure that the service providing the text data - * is already up and running. - * - *

To start an example socket text stream on your local machine run netcat from - * a command line: nc -lk 9999, where the parameter specifies the - * port number. - * - *

Usage: - * SocketTextStreamWordCount <hostname> <port> - *
- * - *

This example shows how to: - *

    - *
  • use StreamExecutionEnvironment.socketTextStream - *
  • write a simple Flink program - *
  • write and use user-defined functions - *
- * - * @see netcat - */ -public class SocketTextStreamWordCount { - - // - // Program - // - - public static void main(String[] args) throws Exception { - - if (args.length != 2){ - System.err.println("USAGE:\nSocketTextStreamWordCount "); - return; - } - - String hostName = args[0]; - Integer port = Integer.parseInt(args[1]); - - // set up the execution environment - final StreamExecutionEnvironment env = StreamExecutionEnvironment - .getExecutionEnvironment(); - - // get input data - DataStream text = env.socketTextStream(hostName, port); - - DataStream> counts = - // split up the lines in pairs (2-tuples) containing: (word,1) - text.flatMap(new LineSplitter()) - // group by the tuple field "0" and sum up tuple field "1" - .keyBy(0) - .sum(1); - - counts.print(); - - // execute program - env.execute("Java WordCount from SocketTextStream Example"); - } - - // - // User Functions - // - - /** - * Implements the string tokenizer that splits sentences into words as a user-defined - * FlatMapFunction. The function takes a line (String) and splits it into - * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>). - */ - public static final class LineSplitter implements FlatMapFunction> { - - @Override - public void flatMap(String value, Collector> out) { - // normalize and split the line - String[] tokens = value.toLowerCase().split("\\W+"); - - // emit the pairs - for (String token : tokens) { - if (token.length() > 0) { - out.collect(new Tuple2(token, 1)); - } - } - } - } -} diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java index 45a67ae3b7bcf0..6027e751650859 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java @@ -20,26 +20,17 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - /** * Skeleton for a Flink Streaming Job. * - *

For a full example of a Flink Streaming Job, see the SocketTextStreamWordCount.java - * file in the same package/directory or have a look at the website. - * - *

You can also generate a .jar file that you can submit on your Flink - * cluster. - * Just type - * mvn clean package - * in the projects root directory. - * You will find the jar in - * target/${artifactId}-${version}.jar - * From the CLI you can then run - * ./bin/flink run -c ${package}.StreamingJob target/${artifactId}-${version}.jar + *

For a tutorial how to write a Flink streaming application, check the + * tutorials and examples on the Flink Website. * - *

For more information on the CLI see: + *

To package your appliation into a JAR file for execution, run + * 'mvn clean package' on the command line. * - *

http://flink.apache.org/docs/latest/apis/cli.html + *

If you change the name of the main class (with the public static void main(String[] args)) + * method, change the respective entry in the POM.xml file (simply search for 'mainClass'). */ public class StreamingJob { diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java deleted file mode 100644 index 6c953890ff356d..00000000000000 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java +++ /dev/null @@ -1,94 +0,0 @@ -package ${package}; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.flink.api.common.functions.FlatMapFunction; -import org.apache.flink.api.java.DataSet; -import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.util.Collector; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram - * over some sample data - * - *

This example shows how to: - *

    - *
  • write a simple Flink program. - *
  • use Tuple data types. - *
  • write and use user-defined functions. - *
- * - */ -public class WordCount { - - // - // Program - // - - public static void main(String[] args) throws Exception { - - // set up the execution environment - final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - - // get input data - DataSet text = env.fromElements( - "To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", - "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles," - ); - - DataSet> counts = - // split up the lines in pairs (2-tuples) containing: (word,1) - text.flatMap(new LineSplitter()) - // group by the tuple field "0" and sum up tuple field "1" - .groupBy(0) - .sum(1); - - // execute and print result - counts.print(); - - } - - // - // User Functions - // - - /** - * Implements the string tokenizer that splits sentences into words as a user-defined - * FlatMapFunction. The function takes a line (String) and splits it into - * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>). - */ - public static final class LineSplitter implements FlatMapFunction> { - - @Override - public void flatMap(String value, Collector> out) { - // normalize and split the line - String[] tokens = value.toLowerCase().split("\\W+"); - - // emit the pairs - for (String token : tokens) { - if (token.length() > 0) { - out.collect(new Tuple2(token, 1)); - } - } - } - } -} diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml index c82c3859a6f602..2af71185d68ff1 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml @@ -201,15 +201,11 @@ under the License. - - diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala index 4ecfeed17bc4bc..a533da90f22e08 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala @@ -23,26 +23,15 @@ import org.apache.flink.api.scala._ /** * Skeleton for a Flink Batch Job. * - * For a full example of a Flink Batch Job, see the WordCountJob.scala file in the - * same package/directory or have a look at the website. + * For a tutorial how to write a Flink batch application, check the + * tutorials and examples on the Flink Website. * - * You can also generate a .jar file that you can submit on your Flink - * cluster. Just type - * {{{ - * mvn clean package - * }}} - * in the projects root directory. You will find the jar in - * target/${artifactId}-${version}.jar - * From the CLI you can then run - * {{{ - * ./bin/flink run -c ${package}.BatchJob target/${artifactId}-${version}.jar - * }}} - * - * For more information on the CLI see: - * - * http://flink.apache.org/docs/latest/apis/cli.html + * To package your appliation into a JAR file for execution, + * change the main class in the POM.xml file to this class (simply search for 'mainClass') + * and run 'mvn clean package' on the command line. */ object BatchJob { + def main(args: Array[String]) { // set up the batch execution environment val env = ExecutionEnvironment.getExecutionEnvironment @@ -74,4 +63,4 @@ object BatchJob { // execute program env.execute("Flink Batch Scala API Skeleton") } -} \ No newline at end of file +} diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala deleted file mode 100644 index a6987acc613b91..00000000000000 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala +++ /dev/null @@ -1,69 +0,0 @@ -package ${package} - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.flink.streaming.api.scala._ - -/** - * This example shows an implementation of WordCount with data from a text socket. - * To run the example make sure that the service providing the text data is already up and running. - * - * To start an example socket text stream on your local machine run netcat from a command line, - * where the parameter specifies the port number: - * - * {{{ - * nc -lk 9999 - * }}} - * - * Usage: - * {{{ - * SocketTextStreamWordCount - * }}} - * - * This example shows how to: - * - * - use StreamExecutionEnvironment.socketTextStream - * - write a simple Flink Streaming program in scala - * - write and use user-defined functions - */ -object SocketTextStreamWordCount { - - def main(args: Array[String]) { - if (args.length != 2) { - System.err.println("USAGE:\nSocketTextStreamWordCount ") - return - } - - val hostName = args(0) - val port = args(1).toInt - - val env = StreamExecutionEnvironment.getExecutionEnvironment - - // create streams for names and ages by mapping the inputs to the corresponding objects - val text = env.socketTextStream(hostName, port) - val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } } - .map { (_, 1) } - .keyBy(0) - .sum(1) - - counts print - - env.execute("Scala WordCount from SocketTextStream Example") - } -} diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala index 7a45fc226661b8..7c950b14aceac9 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala @@ -23,24 +23,14 @@ import org.apache.flink.streaming.api.scala._ /** * Skeleton for a Flink Streaming Job. * - * For a full example of a Flink Streaming Job, see the SocketTextStreamWordCount.java - * file in the same package/directory or have a look at the website. + * For a tutorial how to write a Flink streaming application, check the + * tutorials and examples on the Flink Website. * - * You can also generate a .jar file that you can submit on your Flink - * cluster. Just type - * {{{ - * mvn clean package - * }}} - * in the projects root directory. You will find the jar in - * target/${artifactId}-${version}.jar - * From the CLI you can then run - * {{{ - * ./bin/flink run -c ${package}.StreamingJob target/${artifactId}-${version}.jar - * }}} + * To package your appliation into a JAR file for execution, run + * 'mvn clean package' on the command line. * - * For more information on the CLI see: - * - * http://flink.apache.org/docs/latest/apis/cli.html + * If you change the name of the main class (with the public static void main(String[] args)) + * method, change the respective entry in the POM.xml file (simply search for 'mainClass'). */ object StreamingJob { def main(args: Array[String]) { @@ -70,4 +60,4 @@ object StreamingJob { // execute program env.execute("Flink Streaming Scala API Skeleton") } -} \ No newline at end of file +} diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala deleted file mode 100644 index b88dcc6d2299be..00000000000000 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala +++ /dev/null @@ -1,53 +0,0 @@ -package ${package} - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.flink.api.scala._ - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram - * over some sample data - * - * This example shows how to: - * - * - write a simple Flink program. - * - use Tuple data types. - * - write and use user-defined functions. - */ -object WordCount { - def main(args: Array[String]) { - - // set up the execution environment - val env = ExecutionEnvironment.getExecutionEnvironment - - // get input data - val text = env.fromElements("To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles,") - - val counts = text.flatMap { _.toLowerCase.split("\\W+") } - .map { (_, 1) } - .groupBy(0) - .sum(1) - - // execute and print result - counts.print() - - } -} From 78bf90d8def2d88e1480e229e49430a0decfb2a0 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Wed, 21 Feb 2018 20:30:35 +0100 Subject: [PATCH 0014/2294] [hotfix] [quickstarts] Fix header and package declaration order. --- .../archetype-resources/src/main/java/BatchJob.java | 6 +++--- .../archetype-resources/src/main/java/StreamingJob.java | 6 +++--- .../archetype-resources/src/main/scala/BatchJob.scala | 6 +++--- .../archetype-resources/src/main/scala/StreamingJob.scala | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java index 971192422b96dd..9515791af809d4 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java @@ -1,6 +1,4 @@ -package ${package}; - -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,6 +16,8 @@ * limitations under the License. */ +package ${package}; + import org.apache.flink.api.java.ExecutionEnvironment; /** diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java index 6027e751650859..40918894f873f9 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java @@ -1,6 +1,4 @@ -package ${package}; - -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,6 +16,8 @@ * limitations under the License. */ +package ${package}; + import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; /** diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala index a533da90f22e08..46520b7dfcccd7 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala @@ -1,6 +1,4 @@ -package ${package} - -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,6 +16,8 @@ package ${package} * limitations under the License. */ +package ${package} + import org.apache.flink.api.scala._ /** diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala index 7c950b14aceac9..20115a75a2532d 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala @@ -1,6 +1,4 @@ -package ${package} - -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,6 +16,8 @@ package ${package} * limitations under the License. */ +package ${package} + import org.apache.flink.streaming.api.scala._ /** From d05248900b433735db05da57de3d0e837300077c Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Wed, 21 Feb 2018 20:40:25 +0100 Subject: [PATCH 0015/2294] [FLINK-8763] [quickstarts] Remove obsolete Dummy.java classes from quickstart projects. --- .../org/apache/flink/quickstart/Dummy.java | 27 ------------------- .../org/apache/flink/quickstart/Dummy.java | 27 ------------------- 2 files changed, 54 deletions(-) delete mode 100644 flink-quickstart/flink-quickstart-java/src/main/java/org/apache/flink/quickstart/Dummy.java delete mode 100644 flink-quickstart/flink-quickstart-scala/src/main/java/org/apache/flink/quickstart/Dummy.java diff --git a/flink-quickstart/flink-quickstart-java/src/main/java/org/apache/flink/quickstart/Dummy.java b/flink-quickstart/flink-quickstart-java/src/main/java/org/apache/flink/quickstart/Dummy.java deleted file mode 100644 index b590d76983423f..00000000000000 --- a/flink-quickstart/flink-quickstart-java/src/main/java/org/apache/flink/quickstart/Dummy.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.quickstart; - -/** - * This class solely exists to generate - * javadocs for the "quickstart-java" project. - **/ -public class Dummy { - // -} diff --git a/flink-quickstart/flink-quickstart-scala/src/main/java/org/apache/flink/quickstart/Dummy.java b/flink-quickstart/flink-quickstart-scala/src/main/java/org/apache/flink/quickstart/Dummy.java deleted file mode 100644 index b590d76983423f..00000000000000 --- a/flink-quickstart/flink-quickstart-scala/src/main/java/org/apache/flink/quickstart/Dummy.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.quickstart; - -/** - * This class solely exists to generate - * javadocs for the "quickstart-java" project. - **/ -public class Dummy { - // -} From cb0ea0f873c499b579a405fd12a5ecf644a5dee0 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Wed, 21 Feb 2018 21:05:23 +0100 Subject: [PATCH 0016/2294] [hotfix] [quickstarts] Fix block comments in program stubs. --- .../resources/archetype-resources/src/main/java/BatchJob.java | 2 +- .../archetype-resources/src/main/java/StreamingJob.java | 2 +- .../resources/archetype-resources/src/main/scala/BatchJob.scala | 2 +- .../archetype-resources/src/main/scala/StreamingJob.scala | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java index 9515791af809d4..db2ee601c02343 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java @@ -36,7 +36,7 @@ public static void main(String[] args) throws Exception { // set up the batch execution environment final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - /** + /* * Here, you can start creating your execution plan for Flink. * * Start with getting some data from the environment, like diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java index 40918894f873f9..5bcee21cf45d8d 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java @@ -38,7 +38,7 @@ public static void main(String[] args) throws Exception { // set up the streaming execution environment final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - /** + /* * Here, you can start creating your execution plan for Flink. * * Start with getting some data from the environment, like diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala index 46520b7dfcccd7..329e6c253a8f94 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala @@ -36,7 +36,7 @@ object BatchJob { // set up the batch execution environment val env = ExecutionEnvironment.getExecutionEnvironment - /** + /* * Here, you can start creating your execution plan for Flink. * * Start with getting some data from the environment, like diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala index 20115a75a2532d..bab0bb97abf33a 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala @@ -37,7 +37,7 @@ object StreamingJob { // set up the streaming execution environment val env = StreamExecutionEnvironment.getExecutionEnvironment - /** + /* * Here, you can start creating your execution plan for Flink. * * Start with getting some data from the environment, like From 51262c89c7fad933ac267d606f7c85f46249d7f3 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Fri, 23 Feb 2018 10:53:22 +0100 Subject: [PATCH 0017/2294] [FLINK-8764] [quickstarts] Make quickstarts work out of the box for IDE and JAR packaging - All Flink and Scala dependencies are properly set to provided - That way, Maven JAR packaging behaves correctly by default - Eclipse adds 'provided' dependencies to the classpath when running programs, so works out of the box - There is a profile that automatically activates in IntelliJ that adds the necessary dependencies in 'compile' scope to make it run out of the box. --- .../resources/archetype-resources/pom.xml | 230 +++++++--------- .../resources/archetype-resources/pom.xml | 246 ++++++++---------- 2 files changed, 206 insertions(+), 270 deletions(-) diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index b78bf691736d46..6ac05b0419b634 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -50,162 +50,53 @@ under the License. - - - - org.apache.flink - flink-core - ${flink.version} - + org.apache.flink flink-java ${flink.version} + provided - org.apache.flink - flink-clients_${scala.binary.version} + flink-streaming-java_${scala.binary.version} ${flink.version} + provided + + + + - + + org.slf4j slf4j-log4j12 ${slf4j.version} + runtime log4j log4j ${log4j.version} + runtime - - - - build-jar - - - false - - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - org.apache.flink - flink-java - ${flink.version} - provided - - - org.apache.flink - flink-clients_${scala.binary.version} - ${flink.version} - provided - - - org.apache.flink - flink-streaming-java_${scala.binary.version} - ${flink.version} - provided - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - provided - - - log4j - log4j - ${log4j.version} - provided - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.0.0 - - - - package - - shade - - - - - org.apache.flink:force-shading - com.google.code.findbugs:jsr305 - org.slf4j:* - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - ${package}.StreamingJob - - - - - - - - - - - + org.apache.maven.plugins maven-compiler-plugin @@ -215,12 +106,57 @@ under the License. 1.8 + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + + package + + shade + + + + + org.apache.flink:force-shading + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + ${package}.StreamingJob + + + + + + - - + + org.eclipse.m2e lifecycle-mapping @@ -247,10 +185,10 @@ under the License. org.apache.maven.plugins - maven-assembly-plugin - [2.4,) + maven-shade-plugin + [3.0.0,) - single + shade @@ -277,6 +215,36 @@ under the License. - --> + + + + + + + add-dependencies-for-IDEA + + + + idea.version + + + + + + org.apache.flink + flink-java + ${flink.version} + compile + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${flink.version} + compile + + + + + diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml index 2af71185d68ff1..5828d7374c614d 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml @@ -52,171 +52,104 @@ under the License. 2.11.12 - - - - org.apache.flink - flink-core - ${flink.version} - - - - org.apache.flink - flink-clients_${scala.binary.version} - ${flink.version} - + org.apache.flink flink-scala_${scala.binary.version} ${flink.version} + provided org.apache.flink flink-streaming-scala_${scala.binary.version} ${flink.version} + provided + org.scala-lang scala-library ${scala.version} + provided + + + + + - + + org.slf4j slf4j-log4j12 ${slf4j.version} + runtime log4j log4j ${log4j.version} + runtime - - - - build-jar - - false - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - org.apache.flink - flink-clients_${scala.binary.version} - ${flink.version} - provided - - - org.apache.flink - flink-scala_${scala.binary.version} - ${flink.version} - provided - - - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${flink.version} - provided - - - org.scala-lang - scala-library - ${scala.version} - provided - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - provided - - - log4j - log4j - ${log4j.version} - provided - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.0.0 - - - - package - - shade - - - - - org.apache.flink:force-shading - com.google.code.findbugs:jsr305 - org.slf4j:* - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - ${package}.StreamingJob - - - - - - - - - - - + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + + package + + shade + + + + + org.apache.flink:force-shading + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + ${package}.StreamingJob + + + + + + + + org.apache.maven.plugins maven-compiler-plugin @@ -226,6 +159,8 @@ under the License. 1.8 + + net.alchim31.maven scala-maven-plugin @@ -240,7 +175,7 @@ under the License. - + org.apache.maven.plugins maven-eclipse-plugin @@ -255,10 +190,8 @@ under the License. org.scala-ide.sdt.core.scalabuilder - org.scala-ide.sdt.launching.SCALA_CONTAINER - - org.eclipse.jdt.launching.JRE_CONTAINER - + org.scala-ide.sdt.launching.SCALA_CONTAINER + org.eclipse.jdt.launching.JRE_CONTAINER org.scala-lang:scala-library @@ -270,8 +203,6 @@ under the License. - - org.codehaus.mojo build-helper-maven-plugin @@ -307,4 +238,41 @@ under the License. + + + + + + + add-dependencies-for-IDEA + + + + idea.version + + + + + + org.apache.flink + flink-scala_${scala.binary.version} + ${flink.version} + compile + + + org.apache.flink + flink-streaming-scala_${scala.binary.version} + ${flink.version} + compile + + + org.scala-lang + scala-library + ${scala.version} + compile + + + + + From 509f85489a3523a42e958062029ebaf3e3e714e4 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Fri, 23 Feb 2018 11:10:43 +0100 Subject: [PATCH 0018/2294] [FLINK-8765] [quickstarts] Simplify quickstart properties This does not pull out the slf4j and log4j version into properties any more, making the quickstarts a bit simpler. Given that both versions are used only once, and only for the feature to have convenience logging in the IDE, the versions might as well be defined directly in the dependencies. --- .../src/main/resources/archetype-resources/pom.xml | 6 ++---- .../src/main/resources/archetype-resources/pom.xml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index 6ac05b0419b634..50b035192a9d41 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -31,8 +31,6 @@ under the License. UTF-8 @project.version@ - @slf4j.version@ - @log4j.version@ @scala.binary.version@ @@ -82,13 +80,13 @@ under the License. org.slf4j slf4j-log4j12 - ${slf4j.version} + 1.7.7 runtime log4j log4j - ${log4j.version} + 1.2.17 runtime diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml index 5828d7374c614d..e0f50f1e1c274e 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml @@ -46,8 +46,6 @@ under the License. UTF-8 @project.version@ - @slf4j.version@ - @log4j.version@ 2.11 2.11.12 @@ -92,13 +90,13 @@ under the License. org.slf4j slf4j-log4j12 - ${slf4j.version} + 1.7.7 runtime log4j log4j - ${log4j.version} + 1.2.17 runtime From 2239ba3c8faf4078581b5ee764bb5863648b549f Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Fri, 23 Feb 2018 11:13:01 +0100 Subject: [PATCH 0019/2294] [FLINK-8766] [quickstarts] Pin scala runtime version for Java Quickstart Followup to FLINK-7414, which pinned the scala version for the Scala Quickstart --- .../src/main/resources/archetype-resources/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index 50b035192a9d41..d53415a5f7ac5a 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -31,7 +31,7 @@ under the License. UTF-8 @project.version@ - @scala.binary.version@ + 2.11 From c6f840623d3555557f51327cb4ea2d745bb19b64 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Fri, 23 Feb 2018 11:18:36 +0100 Subject: [PATCH 0020/2294] [FLINK-8767] [quickstarts] Set the maven.compiler.source and .target properties for Java Quickstart Setting these properties helps properly pinning the Java version in IntelliJ. Without these properties, Java version keeps switching back to 1.5 in some setups. --- .../src/main/resources/archetype-resources/pom.xml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index d53415a5f7ac5a..0ca6eb925987e5 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -31,7 +31,10 @@ under the License. UTF-8 @project.version@ + 1.8 2.11 + ${java.version} + ${java.version} @@ -100,8 +103,8 @@ under the License. maven-compiler-plugin 3.1 - 1.8 - 1.8 + ${java.version} + ${java.version} @@ -158,8 +161,8 @@ under the License. maven-compiler-plugin - 1.8 - 1.8 + ${java.version} + ${java.version} jdt From 647c552a26cbe5f37dfb1d69f26574ef0853fba3 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Mon, 26 Feb 2018 12:19:00 +0100 Subject: [PATCH 0021/2294] [FLINK-8764] [docs] Adjust quickstart documentation --- docs/quickstart/java_api_quickstart.md | 119 +++++------------------- docs/quickstart/scala_api_quickstart.md | 95 ++++++------------- 2 files changed, 52 insertions(+), 162 deletions(-) diff --git a/docs/quickstart/java_api_quickstart.md b/docs/quickstart/java_api_quickstart.md index baf14deb7c5cca..9a325911dd27b1 100644 --- a/docs/quickstart/java_api_quickstart.md +++ b/docs/quickstart/java_api_quickstart.md @@ -1,6 +1,6 @@ --- -title: "Sample Project using the Java API" -nav-title: Sample Project in Java +title: "Project Template for Java" +nav-title: Project Template for Java nav-parent_id: start nav-pos: 0 --- @@ -86,120 +86,51 @@ quickstart/ │   └── myorg │   └── quickstart │   ├── BatchJob.java - │   ├── SocketTextStreamWordCount.java - │   ├── StreamingJob.java - │   └── WordCount.java + │   └── StreamingJob.java └── resources └── log4j.properties {% endhighlight %} -The sample project is a __Maven project__, which contains four classes. _StreamingJob_ and _BatchJob_ are basic skeleton programs, _SocketTextStreamWordCount_ is a working streaming example and _WordCountJob_ is a working batch example. Please note that the _main_ method of all classes allow you to start Flink in a development/testing mode. +The sample project is a __Maven project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program. +The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments. We recommend you __import this project into your IDE__ to develop and -test it. If you use Eclipse, the [m2e plugin](http://www.eclipse.org/m2e/) +test it. IntelliJ IDEA supports Maven projects out of the box. +If you use Eclipse, the [m2e plugin](http://www.eclipse.org/m2e/) allows to [import Maven projects](http://books.sonatype.com/m2eclipse-book/reference/creating-sect-importing-projects.html#fig-creating-import). Some Eclipse bundles include that plugin by default, others require you -to install it manually. The IntelliJ IDE supports Maven projects out of -the box. +to install it manually. - -*A note to Mac OS X users*: The default JVM heapsize for Java is too +*A note to Mac OS X users*: The default JVM heapsize for Java mey be too small for Flink. You have to manually increase it. In Eclipse, choose `Run Configurations -> Arguments` and write into the `VM Arguments` box: `-Xmx800m`. ## Build Project -If you want to __build your project__, go to your project directory and -issue the `mvn clean install -Pbuild-jar` command. You will -__find a jar__ that runs on every Flink cluster with a compatible -version, __target/original-your-artifact-id-your-version.jar__. There -is also a fat-jar in __target/your-artifact-id-your-version.jar__ which, -additionally, contains all dependencies that were added to the Maven -project. +If you want to __build/package your project__, go to your project directory and +run the '`mvn clean package`' command. +You will __find a JAR file__ that contains your application, plus connectors and libraries +that you may have added as dependencoes to the application: `target/-.jar`. + +__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, +we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, the Flink +can run time application from the JAR file without additionally specifying the main class. ## Next Steps Write your application! -The quickstart project contains a `WordCount` implementation, the -"Hello World" of Big Data processing systems. The goal of `WordCount` -is to determine the frequencies of words in a text, e.g., how often do -the terms "the" or "house" occur in all Wikipedia texts. - -__Sample Input__: - -~~~bash -big data is big -~~~ - -__Sample Output__: - -~~~bash -big 2 -data 1 -is 1 -~~~ - -The following code shows the `WordCount` implementation from the -Quickstart which processes some text lines with two operators (a FlatMap -and a Reduce operation via aggregating a sum), and prints the resulting -words and counts to std-out. - -~~~java -public class WordCount { - - public static void main(String[] args) throws Exception { - - // set up the execution environment - final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - - // get input data - DataSet text = env.fromElements( - "To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", - "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles," - ); - - DataSet> counts = - // split up the lines in pairs (2-tuples) containing: (word,1) - text.flatMap(new LineSplitter()) - // group by the tuple field "0" and sum up tuple field "1" - .groupBy(0) - .sum(1); - - // execute and print result - counts.print(); - } -} -~~~ - -The operations are defined by specialized classes, here the LineSplitter class. - -~~~java -public static final class LineSplitter implements FlatMapFunction> { - - @Override - public void flatMap(String value, Collector> out) { - // normalize and split the line - String[] tokens = value.toLowerCase().split("\\W+"); - - // emit the pairs - for (String token : tokens) { - if (token.length() > 0) { - out.collect(new Tuple2(token, 1)); - } - } - } -} -~~~ - -{% gh_link /flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java "Check GitHub" %} for the full example code. - -For a complete overview over our API, have a look at the +If you are writing a streaming application and you are looking for inspiration what to write, +take a look at the [Stream Processing Application Tutorial]({{ site.baseurl }}/quickstart/run_example_quickstart.html#writing-a-flink-program) + +If you are writing a batch processing application and you are looking for inspiration what to write, +take a look at the [Batch Application Examples]({{ site.baseurl }}/dev/batch/examples.html) + +For a complete overview over the APIa, have a look at the [DataStream API]({{ site.baseurl }}/dev/datastream_api.html) and [DataSet API]({{ site.baseurl }}/dev/batch/index.html) sections. + If you have any trouble, ask on our [Mailing List](http://mail-archives.apache.org/mod_mbox/flink-user/). We are happy to provide help. diff --git a/docs/quickstart/scala_api_quickstart.md b/docs/quickstart/scala_api_quickstart.md index 40c02a9b8913aa..a7b73e322c43bd 100644 --- a/docs/quickstart/scala_api_quickstart.md +++ b/docs/quickstart/scala_api_quickstart.md @@ -1,6 +1,6 @@ --- -title: "Sample Project using the Scala API" -nav-title: Sample Project in Scala +title: "Project Template for Scala" +nav-title: Project Template for Scala nav-parent_id: start nav-pos: 1 --- @@ -173,14 +173,18 @@ quickstart/ └── myorg └── quickstart ├── BatchJob.scala - ├── SocketTextStreamWordCount.scala - ├── StreamingJob.scala - └── WordCount.scala + └── StreamingJob.scala {% endhighlight %} -The sample project is a __Maven project__, which contains four classes. _StreamingJob_ and _BatchJob_ are basic skeleton programs, _SocketTextStreamWordCount_ is a working streaming example and _WordCountJob_ is a working batch example. Please note that the _main_ method of all classes allow you to start Flink in a development/testing mode. +The sample project is a __Maven project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program. +The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments. -We recommend you __import this project into your IDE__. For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites: +We recommend you __import this project into your IDE__. + +IntelliJ IDEA supports Maven out of the box and offers a plugin for Scala development. +From our experience, IntelliJ provides the best experience for developing Flink applications. + +For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites: * _Eclipse 4.x_ * [Scala IDE](http://download.scala-ide.org/sdk/lithium/e44/scala211/stable/site) @@ -191,78 +195,33 @@ We recommend you __import this project into your IDE__. For Eclipse, you need th * [m2eclipse-scala](http://alchim31.free.fr/m2e-scala/update-site) * [Build Helper Maven Plugin](https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/) -The IntelliJ IDE supports Maven out of the box and offers a plugin for -Scala development. +### Build Project +If you want to __build/package your project__, go to your project directory and +run the '`mvn clean package`' command. +You will __find a JAR file__ that contains your application, plus connectors and libraries +that you may have added as dependencoes to the application: `target/-.jar`. -### Build Project +__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point, +we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, the Flink +can run time application from the JAR file without additionally specifying the main class. -If you want to __build your project__, go to your project directory and -issue the `mvn clean package -Pbuild-jar` command. You will -__find a jar__ that runs on every Flink cluster with a compatible -version, __target/original-your-artifact-id-your-version.jar__. There -is also a fat-jar in __target/your-artifact-id-your-version.jar__ which, -additionally, contains all dependencies that were added to the Maven -project. ## Next Steps Write your application! -The quickstart project contains a `WordCount` implementation, the -"Hello World" of Big Data processing systems. The goal of `WordCount` -is to determine the frequencies of words in a text, e.g., how often do -the terms "the" or "house" occur in all Wikipedia texts. - -__Sample Input__: - -~~~bash -big data is big -~~~ +If you are writing a streaming application and you are looking for inspiration what to write, +take a look at the [Stream Processing Application Tutorial]({{ site.baseurl }}/quickstart/run_example_quickstart.html#writing-a-flink-program) -__Sample Output__: - -~~~bash -big 2 -data 1 -is 1 -~~~ - -The following code shows the `WordCount` implementation from the -Quickstart which processes some text lines with two operators (a FlatMap -and a Reduce operation via aggregating a sum), and prints the resulting -words and counts to std-out. - -~~~scala -object WordCountJob { - def main(args: Array[String]) { - - // set up the execution environment - val env = ExecutionEnvironment.getExecutionEnvironment - - // get input data - val text = env.fromElements("To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles,") - - val counts = text.flatMap { _.toLowerCase.split("\\W+") } - .map { (_, 1) } - .groupBy(0) - .sum(1) - - // emit result and print result - counts.print() - } -} -~~~ +If you are writing a batch processing application and you are looking for inspiration what to write, +take a look at the [Batch Application Examples]({{ site.baseurl }}/dev/batch/examples.html) -{% gh_link flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala "Check GitHub" %} for the full example code. +For a complete overview over the APIa, have a look at the +[DataStream API]({{ site.baseurl }}/dev/datastream_api.html) and +[DataSet API]({{ site.baseurl }}/dev/batch/index.html) sections. -For a complete overview over our API, have a look at the -[DataStream API]({{ site.baseurl }}/dev/datastream_api.html), -[DataSet API]({{ site.baseurl }}/dev/batch/index.html), and -[Scala API Extensions]({{ site.baseurl }}/dev/scala_api_extensions.html) -sections. If you have any trouble, ask on our +If you have any trouble, ask on our [Mailing List](http://mail-archives.apache.org/mod_mbox/flink-user/). We are happy to provide help. From d63bc75ffa3ad1b0d4de82cd218b9c4d268b41ab Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Thu, 1 Feb 2018 16:02:28 +0100 Subject: [PATCH 0022/2294] [FLINK-8781][scheduler] Try to reschedule failed tasks to previous allocation This closes #5403. --- .../clusterframework/types/SlotProfile.java | 283 ++++++++++++++++++ .../runtime/executiongraph/Execution.java | 29 +- .../executiongraph/ExecutionVertex.java | 27 +- .../jobmanager/scheduler/Scheduler.java | 14 +- .../jobmanager/slots/SlotAndLocality.java | 14 +- .../runtime/jobmaster/slotpool/SlotPool.java | 134 +++------ .../jobmaster/slotpool/SlotPoolGateway.java | 7 +- .../jobmaster/slotpool/SlotProvider.java | 11 +- .../slotpool/SlotSharingManager.java | 102 +------ .../types/SlotProfileTest.java | 133 ++++++++ .../ExecutionGraphMetricsTest.java | 4 +- .../ExecutionVertexSchedulingTest.java | 8 +- .../ProgrammedSlotProvider.java | 5 +- .../utils/SimpleSlotProvider.java | 16 +- .../ScheduleWithCoLocationHintTest.java | 128 ++++---- .../scheduler/SchedulerIsolatedTasksTest.java | 50 ++-- .../scheduler/SchedulerSlotSharingTest.java | 228 +++++++------- .../jobmanager/scheduler/SchedulerTest.java | 4 +- .../scheduler/SchedulerTestBase.java | 19 +- .../slotpool/AvailableSlotsTest.java | 5 +- .../slotpool/SlotPoolCoLocationTest.java | 10 +- .../jobmaster/slotpool/SlotPoolRpcTest.java | 16 +- .../slotpool/SlotPoolSlotSharingTest.java | 22 +- .../jobmaster/slotpool/SlotPoolTest.java | 39 ++- .../slotpool/SlotSharingManagerTest.java | 12 +- 25 files changed, 833 insertions(+), 487 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotProfile.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/SlotProfileTest.java diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotProfile.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotProfile.java new file mode 100644 index 00000000000000..5b1fa0893a0df3 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/types/SlotProfile.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.clusterframework.types; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.runtime.jobmanager.scheduler.Locality; +import org.apache.flink.runtime.jobmaster.SlotContext; +import org.apache.flink.runtime.taskmanager.TaskManagerLocation; +import org.apache.flink.util.Preconditions; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Stream; + +/** + * A slot profile describes the profile of a slot into which a task wants to be scheduled. The profile contains + * attributes such as resource or locality constraints, some of which may be hard or soft. A matcher can be generated + * to filter out candidate slots by matching their {@link SlotContext} against the slot profile and, potentially, + * further requirements. + */ +public class SlotProfile { + + /** Singleton object for a slot profile without any requirements. */ + private static final SlotProfile NO_REQUIREMENTS = noLocality(ResourceProfile.UNKNOWN); + + /** This specifies the desired resource profile for the slot. */ + @Nonnull + private final ResourceProfile resourceProfile; + + /** This specifies the preferred locations for the slot. */ + @Nonnull + private final Collection preferredLocations; + + /** This contains desired allocation ids of the slot. */ + @Nonnull + private final Collection priorAllocations; + + public SlotProfile( + @Nonnull ResourceProfile resourceProfile, + @Nonnull Collection preferredLocations, + @Nonnull Collection priorAllocations) { + + this.resourceProfile = resourceProfile; + this.preferredLocations = preferredLocations; + this.priorAllocations = priorAllocations; + } + + /** + * Returns the desired resource profile for the slot. + */ + @Nonnull + public ResourceProfile getResourceProfile() { + return resourceProfile; + } + + /** + * Returns the preferred locations for the slot. + */ + @Nonnull + public Collection getPreferredLocations() { + return preferredLocations; + } + + /** + * Returns the desired allocation ids for the slot. + */ + @Nonnull + public Collection getPriorAllocations() { + return priorAllocations; + } + + /** + * Returns the matcher for this profile that helps to find slots that fit the profile. + */ + public ProfileToSlotContextMatcher matcher() { + if (priorAllocations.isEmpty()) { + return new LocalityAwareRequirementsToSlotMatcher(preferredLocations); + } else { + return new PreviousAllocationProfileToSlotContextMatcher(priorAllocations); + } + } + + /** + * Classes that implement this interface provide a method to match objects to somehow represent slot candidates + * against the {@link SlotProfile} that produced the matcher object. A matching candidate is transformed into a + * desired result. If the matcher does not find a matching candidate, it returns null. + */ + public interface ProfileToSlotContextMatcher { + + /** + * This method takes the candidate slots, extracts slot contexts from them, filters them by the profile + * requirements and potentially by additional requirements, and produces a result from a match. + * + * @param candidates stream of candidates to match against. + * @param contextExtractor function to extract the {@link SlotContext} from the candidates. + * @param additionalRequirementsFilter predicate to specify additional requirements for each candidate. + * @param resultProducer function to produce a result from a matching candidate input. + * @param type of the objects against we match the profile. + * @param type of the produced output from a matching object. + * @return the result produced by resultProducer if a matching candidate was found or null otherwise. + */ + @Nullable + OUT findMatchWithLocality( + @Nonnull Stream candidates, + @Nonnull Function contextExtractor, + @Nonnull Predicate additionalRequirementsFilter, + @Nonnull BiFunction resultProducer); + } + + /** + * This matcher implementation is the presence of prior allocations. Prior allocations are supposed to overrule + * other locality requirements, such as preferred locations. Prior allocations also require strict matching and + * this matcher returns null if it cannot find a candidate for the same prior allocation. The background is that + * this will force the scheduler tor request a new slot that is guaranteed to be not the prior location of any + * other subtask, so that subtasks do not steal another subtasks prior allocation in case that the own prior + * allocation is no longer available (e.g. machine failure). This is important to enable local recovery for all + * tasks that can still return to their prior allocation. + */ + @VisibleForTesting + public static class PreviousAllocationProfileToSlotContextMatcher implements ProfileToSlotContextMatcher { + + /** Set of prior allocations. */ + private final HashSet priorAllocations; + + @VisibleForTesting + PreviousAllocationProfileToSlotContextMatcher(@Nonnull Collection priorAllocations) { + this.priorAllocations = new HashSet<>(priorAllocations); + Preconditions.checkState( + this.priorAllocations.size() > 0, + "This matcher should only be used if there are prior allocations!"); + } + + public O findMatchWithLocality( + @Nonnull Stream candidates, + @Nonnull Function contextExtractor, + @Nonnull Predicate additionalRequirementsFilter, + @Nonnull BiFunction resultProducer) { + + Predicate filterByAllocation = + (candidate) -> priorAllocations.contains(contextExtractor.apply(candidate).getAllocationId()); + + return candidates + .filter(filterByAllocation.and(additionalRequirementsFilter)) + .findFirst() + .map((result) -> resultProducer.apply(result, Locality.LOCAL)) // TODO introduce special locality? + .orElse(null); + } + } + + /** + * This matcher is used whenever no prior allocation was specified in the {@link SlotProfile}. This implementation + * tries to achieve best possible locality if a preferred location is specified in the profile. + */ + @VisibleForTesting + public static class LocalityAwareRequirementsToSlotMatcher implements ProfileToSlotContextMatcher { + + private final Collection locationPreferences; + + @VisibleForTesting + public LocalityAwareRequirementsToSlotMatcher(@Nonnull Collection locationPreferences) { + this.locationPreferences = new ArrayList<>(locationPreferences); + } + + @Override + public OUT findMatchWithLocality( + @Nonnull Stream candidates, + @Nonnull Function contextExtractor, + @Nonnull Predicate additionalRequirementsFilter, + @Nonnull BiFunction resultProducer) { + + // if we have no location preferences, we can only filter by the additional requirements. + if (locationPreferences.isEmpty()) { + return candidates + .filter(additionalRequirementsFilter) + .findFirst() + .map((result) -> resultProducer.apply(result, Locality.UNCONSTRAINED)) + .orElse(null); + } + + // we build up two indexes, one for resource id and one for host names of the preferred locations. + HashSet preferredResourceIDs = new HashSet<>(locationPreferences.size()); + HashSet preferredFQHostNames = new HashSet<>(locationPreferences.size()); + + for (TaskManagerLocation locationPreference : locationPreferences) { + preferredResourceIDs.add(locationPreference.getResourceID()); + preferredFQHostNames.add(locationPreference.getFQDNHostname()); + } + + Iterator iterator = candidates.iterator(); + + IN matchByHostName = null; + IN matchByAdditionalRequirements = null; + + while (iterator.hasNext()) { + + IN candidate = iterator.next(); + SlotContext slotContext = contextExtractor.apply(candidate); + + // this if checks if the candidate has is a local slot + if (preferredResourceIDs.contains(slotContext.getTaskManagerLocation().getResourceID())) { + if (additionalRequirementsFilter.test(candidate)) { + // we can stop, because we found a match with best possible locality. + return resultProducer.apply(candidate, Locality.LOCAL); + } else { + // next candidate because this failed on the additional requirements. + continue; + } + } + + // this if checks if the candidate is at least host-local, if we did not find another host-local + // candidate before. + if (matchByHostName == null) { + if (preferredFQHostNames.contains(slotContext.getTaskManagerLocation().getFQDNHostname())) { + if (additionalRequirementsFilter.test(candidate)) { + // We remember the candidate, but still continue because there might still be a candidate + // that is local to the desired task manager. + matchByHostName = candidate; + } else { + // next candidate because this failed on the additional requirements. + continue; + } + } + + // this if checks if the candidate at least fulfils the resource requirements, and is only required + // if we did not yet find a valid candidate with better locality. + if (matchByAdditionalRequirements == null + && additionalRequirementsFilter.test(candidate)) { + // Again, we remember but continue in hope for a candidate with better locality. + matchByAdditionalRequirements = candidate; + } + } + } + + // at the end of the iteration, we return the candidate with best possible locality or null. + if (matchByHostName != null) { + return resultProducer.apply(matchByHostName, Locality.HOST_LOCAL); + } else if (matchByAdditionalRequirements != null) { + return resultProducer.apply(matchByAdditionalRequirements, Locality.NON_LOCAL); + } else { + return null; + } + } + } + + /** + * Returns a slot profile that has no requirements. + */ + public static SlotProfile noRequirements() { + return NO_REQUIREMENTS; + } + + /** + * Returns a slot profile for the given resource profile, without any locality requirements. + */ + public static SlotProfile noLocality(ResourceProfile resourceProfile) { + return new SlotProfile(resourceProfile, Collections.emptyList(), Collections.emptyList()); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java index 946f6e42cf2416..2fb831a2c62b8d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java @@ -26,7 +26,10 @@ import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult; import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.checkpoint.JobManagerTaskRestore; +import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor; import org.apache.flink.runtime.deployment.PartialInputChannelDeploymentDescriptor; @@ -155,6 +158,10 @@ public class Execution implements AccessExecution, Archiveable allocateAndAssignSlotForExecution( new ScheduledUnit(this, slotSharingGroupId) : new ScheduledUnit(this, slotSharingGroupId, locationConstraint); + // try to extract previous allocation ids, if applicable, so that we can reschedule to the same slot + ExecutionVertex executionVertex = getVertex(); + AllocationID lastAllocation = executionVertex.getLatestPriorAllocation(); + + Collection previousAllocationIDs = + lastAllocation != null ? Collections.singletonList(lastAllocation) : Collections.emptyList(); + // calculate the preferred locations - final CompletableFuture> preferredLocationsFuture = calculatePreferredLocations(locationPreferenceConstraint); + final CompletableFuture> preferredLocationsFuture = + calculatePreferredLocations(locationPreferenceConstraint); final SlotRequestId slotRequestId = new SlotRequestId(); @@ -470,7 +490,10 @@ public CompletableFuture allocateAndAssignSlotForExecution( slotRequestId, toSchedule, queued, - preferredLocations, + new SlotProfile( + ResourceProfile.UNKNOWN, + preferredLocations, + previousAllocationIDs), allocationTimeout)); // register call back to cancel slot request in case that the execution gets canceled diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java index f13e42c14318bc..8b57a7a6a93771 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java @@ -26,6 +26,7 @@ import org.apache.flink.runtime.JobException; import org.apache.flink.runtime.blob.PermanentBlobKey; import org.apache.flink.runtime.checkpoint.JobManagerTaskRestore; +import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor; import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor; import org.apache.flink.runtime.deployment.PartialInputChannelDeploymentDescriptor; @@ -295,17 +296,11 @@ public Execution getPriorExecutionAttempt(int attemptNumber) { } } - /** - * Gets the location where the latest completed/canceled/failed execution of the vertex's - * task happened. - * - * @return The latest prior execution location, or null, if there is none, yet. - */ - public TaskManagerLocation getLatestPriorLocation() { + public Execution getLatestPriorExecution() { synchronized (priorExecutions) { final int size = priorExecutions.size(); if (size > 0) { - return priorExecutions.get(size - 1).getAssignedResourceLocation(); + return priorExecutions.get(size - 1); } else { return null; @@ -313,6 +308,22 @@ public TaskManagerLocation getLatestPriorLocation() { } } + /** + * Gets the location where the latest completed/canceled/failed execution of the vertex's + * task happened. + * + * @return The latest prior execution location, or null, if there is none, yet. + */ + public TaskManagerLocation getLatestPriorLocation() { + Execution latestPriorExecution = getLatestPriorExecution(); + return latestPriorExecution != null ? latestPriorExecution.getAssignedResourceLocation() : null; + } + + public AllocationID getLatestPriorAllocation() { + Execution latestPriorExecution = getLatestPriorExecution(); + return latestPriorExecution != null ? latestPriorExecution.getAssignedAllocationID() : null; + } + EvictingBoundedList getCopyOfPriorExecutionsList() { synchronized (priorExecutions) { return new EvictingBoundedList<>(priorExecutions); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java index fc79d40518de0b..0116fdb8f24c29 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/Scheduler.java @@ -21,19 +21,20 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.executiongraph.ExecutionVertex; import org.apache.flink.runtime.instance.Instance; import org.apache.flink.runtime.instance.InstanceDiedException; import org.apache.flink.runtime.instance.InstanceListener; -import org.apache.flink.runtime.instance.SlotSharingGroupId; -import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.instance.SharedSlot; import org.apache.flink.runtime.instance.SimpleSlot; -import org.apache.flink.runtime.jobmaster.SlotRequestId; -import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; import org.apache.flink.runtime.instance.SlotSharingGroupAssignment; +import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobgraph.JobVertexID; +import org.apache.flink.runtime.jobmaster.LogicalSlot; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import org.apache.flink.util.ExceptionUtils; @@ -49,7 +50,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -148,11 +148,11 @@ public CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit task, boolean allowQueued, - Collection preferredLocations, + SlotProfile slotProfile, Time allocationTimeout) { try { - final Object ret = scheduleTask(task, allowQueued, preferredLocations); + final Object ret = scheduleTask(task, allowQueued, slotProfile.getPreferredLocations()); if (ret instanceof SimpleSlot) { return CompletableFuture.completedFuture((SimpleSlot) ret); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/SlotAndLocality.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/SlotAndLocality.java index 85871c89987cde..fed26c91b5c483 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/SlotAndLocality.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/SlotAndLocality.java @@ -18,31 +18,35 @@ package org.apache.flink.runtime.jobmanager.slots; -import org.apache.flink.runtime.jobmaster.slotpool.AllocatedSlot; import org.apache.flink.runtime.jobmanager.scheduler.Locality; +import org.apache.flink.runtime.jobmaster.slotpool.AllocatedSlot; -import static org.apache.flink.util.Preconditions.checkNotNull; +import javax.annotation.Nonnull; /** * A combination of a {@link AllocatedSlot} and a {@link Locality}. */ public class SlotAndLocality { + @Nonnull private final AllocatedSlot slot; + @Nonnull private final Locality locality; - public SlotAndLocality(AllocatedSlot slot, Locality locality) { - this.slot = checkNotNull(slot); - this.locality = checkNotNull(locality); + public SlotAndLocality(@Nonnull AllocatedSlot slot, @Nonnull Locality locality) { + this.slot = slot; + this.locality = locality; } // ------------------------------------------------------------------------ + @Nonnull public AllocatedSlot getSlot() { return slot; } + @Nonnull public Locality getLocality() { return locality; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java index a49e6ed9c04341..8a2dd45ea7ef83 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java @@ -26,6 +26,7 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.executiongraph.ExecutionGraph; import org.apache.flink.runtime.instance.SlotSharingGroupId; @@ -303,16 +304,14 @@ public void disconnectResourceManager() { public CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit scheduledUnit, - ResourceProfile resourceProfile, - Collection locationPreferences, + SlotProfile slotProfile, boolean allowQueuedScheduling, Time timeout) { return internalAllocateSlot( slotRequestId, scheduledUnit, - resourceProfile, - locationPreferences, + slotProfile, allowQueuedScheduling, timeout); } @@ -320,8 +319,7 @@ public CompletableFuture allocateSlot( private CompletableFuture internalAllocateSlot( SlotRequestId slotRequestId, ScheduledUnit task, - ResourceProfile resourceProfile, - Collection locationPreferences, + SlotProfile slotProfile, boolean allowQueuedScheduling, Time allocationTimeout) { @@ -343,16 +341,14 @@ private CompletableFuture internalAllocateSlot( multiTaskSlotLocality = allocateCoLocatedMultiTaskSlot( task.getCoLocationConstraint(), multiTaskSlotManager, - resourceProfile, - locationPreferences, + slotProfile, allowQueuedScheduling, allocationTimeout); } else { multiTaskSlotLocality = allocateMultiTaskSlot( task.getJobVertexId(), multiTaskSlotManager, - resourceProfile, - locationPreferences, + slotProfile, allowQueuedScheduling, allocationTimeout); } @@ -373,8 +369,7 @@ private CompletableFuture internalAllocateSlot( // request an allocated slot to assign a single logical slot to CompletableFuture slotAndLocalityFuture = requestAllocatedSlot( slotRequestId, - resourceProfile, - locationPreferences, + slotProfile, allowQueuedScheduling, allocationTimeout); @@ -408,8 +403,7 @@ private CompletableFuture internalAllocateSlot( * * @param coLocationConstraint for which to allocate a {@link SlotSharingManager.MultiTaskSlot} * @param multiTaskSlotManager responsible for the slot sharing group for which to allocate the slot - * @param resourceProfile specifying the requirements for the requested slot - * @param locationPreferences containing preferred TaskExecutors on which to allocate the slot + * @param slotProfile specifying the requirements for the requested slot * @param allowQueuedScheduling true if queued scheduling (the returned task slot must not be completed yet) is allowed, otherwise false * @param allocationTimeout timeout before the slot allocation times out * @return A {@link SlotSharingManager.MultiTaskSlotLocality} which contains the allocated{@link SlotSharingManager.MultiTaskSlot} @@ -419,8 +413,7 @@ private CompletableFuture internalAllocateSlot( private SlotSharingManager.MultiTaskSlotLocality allocateCoLocatedMultiTaskSlot( CoLocationConstraint coLocationConstraint, SlotSharingManager multiTaskSlotManager, - ResourceProfile resourceProfile, - Collection locationPreferences, + SlotProfile slotProfile, boolean allowQueuedScheduling, Time allocationTimeout) throws NoResourceAvailableException { final SlotRequestId coLocationSlotRequestId = coLocationConstraint.getSlotRequestId(); @@ -438,19 +431,18 @@ private SlotSharingManager.MultiTaskSlotLocality allocateCoLocatedMultiTaskSlot( } } - final Collection actualLocationPreferences; - if (coLocationConstraint.isAssigned()) { - actualLocationPreferences = Collections.singleton(coLocationConstraint.getLocation()); - } else { - actualLocationPreferences = locationPreferences; + // refine the preferred locations of the slot profile + slotProfile = new SlotProfile( + slotProfile.getResourceProfile(), + Collections.singleton(coLocationConstraint.getLocation()), + slotProfile.getPriorAllocations()); } // get a new multi task slot final SlotSharingManager.MultiTaskSlotLocality multiTaskSlotLocality = allocateMultiTaskSlot( coLocationConstraint.getGroupId(), multiTaskSlotManager, - resourceProfile, - actualLocationPreferences, + slotProfile, allowQueuedScheduling, allocationTimeout); @@ -505,8 +497,7 @@ private SlotSharingManager.MultiTaskSlotLocality allocateCoLocatedMultiTaskSlot( * * @param groupId for which to allocate a new {@link SlotSharingManager.MultiTaskSlot} * @param slotSharingManager responsible for the slot sharing group for which to allocate the slot - * @param resourceProfile specifying the requirements for the requested slot - * @param locationPreferences containing preferred TaskExecutors on which to allocate the slot + * @param slotProfile slot profile that specifies the requirements for the slot * @param allowQueuedScheduling true if queued scheduling (the returned task slot must not be completed yet) is allowed, otherwise false * @param allocationTimeout timeout before the slot allocation times out * @return A {@link SlotSharingManager.MultiTaskSlotLocality} which contains the allocated {@link SlotSharingManager.MultiTaskSlot} @@ -516,15 +507,14 @@ private SlotSharingManager.MultiTaskSlotLocality allocateCoLocatedMultiTaskSlot( private SlotSharingManager.MultiTaskSlotLocality allocateMultiTaskSlot( AbstractID groupId, SlotSharingManager slotSharingManager, - ResourceProfile resourceProfile, - Collection locationPreferences, + SlotProfile slotProfile, boolean allowQueuedScheduling, Time allocationTimeout) throws NoResourceAvailableException { // check first whether we have a resolved root slot which we can use SlotSharingManager.MultiTaskSlotLocality multiTaskSlotLocality = slotSharingManager.getResolvedRootSlot( groupId, - locationPreferences); + slotProfile.matcher()); if (multiTaskSlotLocality != null && multiTaskSlotLocality.getLocality() == Locality.LOCAL) { return multiTaskSlotLocality; @@ -534,7 +524,7 @@ private SlotSharingManager.MultiTaskSlotLocality allocateMultiTaskSlot( final SlotRequestId multiTaskSlotRequestId = new SlotRequestId(); // check whether we have an allocated slot available which we can use to create a new multi task slot in - final SlotAndLocality polledSlotAndLocality = pollAndAllocateSlot(allocatedSlotRequestId, resourceProfile, locationPreferences); + final SlotAndLocality polledSlotAndLocality = pollAndAllocateSlot(allocatedSlotRequestId, slotProfile); if (polledSlotAndLocality != null && (polledSlotAndLocality.getLocality() == Locality.LOCAL || multiTaskSlotLocality == null)) { @@ -571,7 +561,7 @@ private SlotSharingManager.MultiTaskSlotLocality allocateMultiTaskSlot( // it seems as if we have to request a new slot from the resource manager, this is always the last resort!!! final CompletableFuture futureSlot = requestNewAllocatedSlot( allocatedSlotRequestId, - resourceProfile, + slotProfile.getResourceProfile(), allocationTimeout); multiTaskSlotFuture = slotSharingManager.createRootSlot( @@ -613,24 +603,21 @@ private SlotSharingManager.MultiTaskSlotLocality allocateMultiTaskSlot( * Allocates an allocated slot first by polling from the available slots and then requesting a new * slot from the ResourceManager if no fitting slot could be found. * - * @param slotRequestId identifying the slot allocation request - * @param resourceProfile which the allocated slot should fulfill - * @param locationPreferences for the allocated slot + * @param slotProfile slot profile that specifies the requirements for the slot * @param allowQueuedScheduling true if the slot allocation can be completed in the future * @param allocationTimeout timeout before the slot allocation times out * @return Future containing the allocated simple slot */ private CompletableFuture requestAllocatedSlot( SlotRequestId slotRequestId, - ResourceProfile resourceProfile, - Collection locationPreferences, + SlotProfile slotProfile, boolean allowQueuedScheduling, Time allocationTimeout) { final CompletableFuture allocatedSlotLocalityFuture; // (1) do we have a slot available already? - SlotAndLocality slotFromPool = pollAndAllocateSlot(slotRequestId, resourceProfile, locationPreferences); + SlotAndLocality slotFromPool = pollAndAllocateSlot(slotRequestId, slotProfile); if (slotFromPool != null) { allocatedSlotLocalityFuture = CompletableFuture.completedFuture(slotFromPool); @@ -638,7 +625,7 @@ private CompletableFuture requestAllocatedSlot( // we have to request a new allocated slot CompletableFuture allocatedSlotFuture = requestNewAllocatedSlot( slotRequestId, - resourceProfile, + slotProfile.getResourceProfile(), allocationTimeout); allocatedSlotLocalityFuture = allocatedSlotFuture.thenApply((AllocatedSlot allocatedSlot) -> new SlotAndLocality(allocatedSlot, Locality.UNKNOWN)); @@ -832,11 +819,8 @@ private void failPendingRequest(PendingRequest pendingRequest, Exception e) { } @Nullable - private SlotAndLocality pollAndAllocateSlot( - SlotRequestId slotRequestId, - ResourceProfile resourceProfile, - Collection locationPreferences) { - SlotAndLocality slotFromPool = availableSlots.poll(resourceProfile, locationPreferences); + private SlotAndLocality pollAndAllocateSlot(SlotRequestId slotRequestId, SlotProfile slotProfile) { + SlotAndLocality slotFromPool = availableSlots.poll(slotProfile); if (slotFromPool != null) { allocatedSlots.add(slotRequestId, slotFromPool.getSlot()); @@ -1404,63 +1388,34 @@ boolean contains(AllocationID slotId) { * Poll a slot which matches the required resource profile. The polling tries to satisfy the * location preferences, by TaskManager and by host. * - * @param resourceProfile The required resource profile. - * @param locationPreferences The location preferences, in order to be checked. + * @param slotProfile slot profile that specifies the requirements for the slot * * @return Slot which matches the resource profile, null if we can't find a match */ - SlotAndLocality poll(ResourceProfile resourceProfile, Collection locationPreferences) { + SlotAndLocality poll(SlotProfile slotProfile) { // fast path if no slots are available if (availableSlots.isEmpty()) { return null; } - boolean hadLocationPreference = false; - - if (locationPreferences != null && !locationPreferences.isEmpty()) { - - // first search by TaskManager - for (TaskManagerLocation location : locationPreferences) { - hadLocationPreference = true; - - final Set onTaskManager = availableSlotsByTaskManager.get(location.getResourceID()); - if (onTaskManager != null) { - for (AllocatedSlot candidate : onTaskManager) { - if (candidate.getResourceProfile().isMatching(resourceProfile)) { - remove(candidate.getAllocationId()); - return new SlotAndLocality(candidate, Locality.LOCAL); - } - } - } - } - - // now, search by host - for (TaskManagerLocation location : locationPreferences) { - final Set onHost = availableSlotsByHost.get(location.getFQDNHostname()); - if (onHost != null) { - for (AllocatedSlot candidate : onHost) { - if (candidate.getResourceProfile().isMatching(resourceProfile)) { - remove(candidate.getAllocationId()); - return new SlotAndLocality(candidate, Locality.HOST_LOCAL); - } - } - } - } - } + SlotProfile.ProfileToSlotContextMatcher matcher = slotProfile.matcher(); + Collection slotAndTimestamps = availableSlots.values(); - // take any slot - for (SlotAndTimestamp candidate : availableSlots.values()) { - final AllocatedSlot slot = candidate.slot(); + SlotAndLocality matchingSlotAndLocality = matcher.findMatchWithLocality( + slotAndTimestamps.stream(), + SlotAndTimestamp::slot, + (SlotAndTimestamp slot) -> slot.slot().getResourceProfile().isMatching(slotProfile.getResourceProfile()), + (SlotAndTimestamp slotAndTimestamp, Locality locality) -> { + AllocatedSlot slot = slotAndTimestamp.slot(); + return new SlotAndLocality(slot, locality); + }); - if (slot.getResourceProfile().isMatching(resourceProfile)) { - remove(slot.getAllocationId()); - return new SlotAndLocality( - slot, hadLocationPreference ? Locality.NON_LOCAL : Locality.UNCONSTRAINED); - } + if (matchingSlotAndLocality != null) { + AllocatedSlot slot = matchingSlotAndLocality.getSlot(); + remove(slot.getAllocationId()); } - // nothing available that matches - return null; + return matchingSlotAndLocality; } /** @@ -1574,14 +1529,13 @@ public CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit task, boolean allowQueued, - Collection preferredLocations, + SlotProfile slotProfile, Time timeout) { CompletableFuture slotFuture = gateway.allocateSlot( slotRequestId, task, - ResourceProfile.UNKNOWN, - preferredLocations, + slotProfile, allowQueued, timeout); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolGateway.java index 7d11681c188493..1aad92a8484167 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolGateway.java @@ -22,6 +22,7 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; @@ -143,8 +144,7 @@ CompletableFuture> offerSlots( * * @param slotRequestId identifying the requested slot * @param scheduledUnit for which to allocate slot - * @param resourceProfile which the allocated slot must fulfill - * @param locationPreferences which define where the allocated slot should be placed, this can also be empty + * @param slotProfile profile that specifies the requirements for the requested slot * @param allowQueuedScheduling true if the slot request can be queued (e.g. the returned future must not be completed) * @param timeout for the operation * @return Future which is completed with the allocated {@link LogicalSlot} @@ -152,8 +152,7 @@ CompletableFuture> offerSlots( CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit scheduledUnit, - ResourceProfile resourceProfile, - Collection locationPreferences, + SlotProfile slotProfile, boolean allowQueuedScheduling, @RpcTimeout Time timeout); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java index 80b2689787af09..1653138949b3f2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java @@ -19,16 +19,15 @@ package org.apache.flink.runtime.jobmaster.slotpool; import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.jobmaster.SlotRequestId; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import javax.annotation.Nullable; -import java.util.Collection; import java.util.concurrent.CompletableFuture; /** @@ -57,7 +56,7 @@ CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit task, boolean allowQueued, - Collection preferredLocations, + SlotProfile slotProfile, Time timeout); /** @@ -65,20 +64,20 @@ CompletableFuture allocateSlot( * * @param task The task to allocate the slot for * @param allowQueued Whether allow the task be queued if we do not have enough resource - * @param preferredLocations preferred locations for the slot allocation + * @param slotProfile profile of the requested slot * @param timeout after which the allocation fails with a timeout exception * @return The future of the allocation */ default CompletableFuture allocateSlot( ScheduledUnit task, boolean allowQueued, - Collection preferredLocations, + SlotProfile slotProfile, Time timeout) { return allocateSlot( new SlotRequestId(), task, allowQueued, - preferredLocations, + slotProfile, timeout); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManager.java index fd6be46bec6b77..242d645b945eaf 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManager.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.jobmaster.slotpool; import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobmanager.scheduler.Locality; import org.apache.flink.runtime.jobmaster.LogicalSlot; @@ -175,107 +176,22 @@ MultiTaskSlot createRootSlot( * preferred locations is checked. * * @param groupId which the returned slot must not contain - * @param locationPreferences specifying which locations are preferred + * @param matcher slot profile matcher to match slot with the profile requirements * @return the resolved root slot and its locality wrt to the specified location preferences * or null if there was no root slot which did not contain the given groupId */ @Nullable - MultiTaskSlotLocality getResolvedRootSlot(AbstractID groupId, Collection locationPreferences) { - Preconditions.checkNotNull(locationPreferences); - - final MultiTaskSlotLocality multiTaskSlotLocality; - - if (locationPreferences.isEmpty()) { - multiTaskSlotLocality = getResolvedRootSlotWithoutLocationPreferences(groupId); - } else { - multiTaskSlotLocality = getResolvedRootSlotWithLocationPreferences(groupId, locationPreferences); - } - - return multiTaskSlotLocality; - } - - /** - * Gets a resolved root slot which does not yet contain the given groupId. The method will try to - * find a slot of a TaskManager contained in the collection of preferred locations. If there is no such slot - * with free capacities available, then the method will look for slots of TaskManager which run on the same - * machine as the TaskManager in the collection of preferred locations. If there is no such slot, then any slot - * with free capacities is returned. If there is no such slot, then null is returned. - * - * @param groupId which the returned slot must not contain - * @param locationPreferences specifying which locations are preferred - * @return the resolved root slot and its locality wrt to the specified location preferences - * or null if there was not root slot which did not contain the given groupId - */ - @Nullable - private MultiTaskSlotLocality getResolvedRootSlotWithLocationPreferences(AbstractID groupId, Collection locationPreferences) { - Preconditions.checkNotNull(groupId); - Preconditions.checkNotNull(locationPreferences); - final Set hostnameSet = new HashSet<>(16); - MultiTaskSlot nonLocalMultiTaskSlot = null; - + MultiTaskSlotLocality getResolvedRootSlot(AbstractID groupId, SlotProfile.ProfileToSlotContextMatcher matcher) { synchronized (lock) { - for (TaskManagerLocation locationPreference : locationPreferences) { - final Set multiTaskSlots = resolvedRootSlots.get(locationPreference); - - if (multiTaskSlots != null) { - for (MultiTaskSlot multiTaskSlot : multiTaskSlots) { - if (!multiTaskSlot.contains(groupId)) { - return MultiTaskSlotLocality.of(multiTaskSlot, Locality.LOCAL); - } - } - - hostnameSet.add(locationPreference.getHostname()); - } - } - - for (Map.Entry> taskManagerLocationSetEntry : resolvedRootSlots.entrySet()) { - if (hostnameSet.contains(taskManagerLocationSetEntry.getKey().getHostname())) { - for (MultiTaskSlot multiTaskSlot : taskManagerLocationSetEntry.getValue()) { - if (!multiTaskSlot.contains(groupId)) { - return MultiTaskSlotLocality.of(multiTaskSlot, Locality.HOST_LOCAL); - } - } - } else if (nonLocalMultiTaskSlot == null) { - for (MultiTaskSlot multiTaskSlot : taskManagerLocationSetEntry.getValue()) { - if (!multiTaskSlot.contains(groupId)) { - nonLocalMultiTaskSlot = multiTaskSlot; - } - } - } - } - } - - if (nonLocalMultiTaskSlot != null) { - return MultiTaskSlotLocality.of(nonLocalMultiTaskSlot, Locality.NON_LOCAL); - } else { - return null; + Collection> resolvedRootSlotsValues = this.resolvedRootSlots.values(); + return matcher.findMatchWithLocality( + resolvedRootSlotsValues.stream().flatMap(Collection::stream), + (MultiTaskSlot multiTaskSlot) -> multiTaskSlot.getSlotContextFuture().join(), + (MultiTaskSlot multiTaskSlot) -> !multiTaskSlot.contains(groupId), + MultiTaskSlotLocality::of); } } - /** - * Gets a resolved slot which does not yet contain the given groupId without any location - * preferences. - * - * @param groupId which the returned slot must not contain - * @return the resolved slot or null if there was no root slot with free capacities - */ - @Nullable - private MultiTaskSlotLocality getResolvedRootSlotWithoutLocationPreferences(AbstractID groupId) { - Preconditions.checkNotNull(groupId); - - synchronized (lock) { - for (Set multiTaskSlots : resolvedRootSlots.values()) { - for (MultiTaskSlot multiTaskSlot : multiTaskSlots) { - if (!multiTaskSlot.contains(groupId)) { - return MultiTaskSlotLocality.of(multiTaskSlot, Locality.UNCONSTRAINED); - } - } - } - } - - return null; - } - /** * Gets an unresolved slot which does not yet contain the given groupId. An unresolved * slot is a slot whose underlying allocated slot has not been allocated yet. diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/SlotProfileTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/SlotProfileTest.java new file mode 100644 index 00000000000000..c09d4bbb7199f8 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/types/SlotProfileTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.clusterframework.types; + +import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; +import org.apache.flink.runtime.instance.SimpleSlotContext; +import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; +import org.apache.flink.runtime.jobmaster.SlotContext; +import org.apache.flink.runtime.taskmanager.TaskManagerLocation; + +import org.junit.Assert; +import org.junit.Test; + +import java.net.InetAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class SlotProfileTest { + + private final ResourceProfile resourceProfile = new ResourceProfile(2, 1024); + + private final AllocationID aid1 = new AllocationID(); + private final AllocationID aid2 = new AllocationID(); + private final AllocationID aid3 = new AllocationID(); + private final AllocationID aid4 = new AllocationID(); + private final AllocationID aidX = new AllocationID(); + + private final TaskManagerLocation tml1 = new TaskManagerLocation(new ResourceID("tm-1"), InetAddress.getLoopbackAddress(), 42); + private final TaskManagerLocation tml2 = new TaskManagerLocation(new ResourceID("tm-2"), InetAddress.getLoopbackAddress(), 43); + private final TaskManagerLocation tml3 = new TaskManagerLocation(new ResourceID("tm-3"), InetAddress.getLoopbackAddress(), 44); + private final TaskManagerLocation tml4 = new TaskManagerLocation(new ResourceID("tm-4"), InetAddress.getLoopbackAddress(), 45); + private final TaskManagerLocation tmlX = new TaskManagerLocation(new ResourceID("tm-X"), InetAddress.getLoopbackAddress(), 46); + + private final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); + + private SimpleSlotContext ssc1 = new SimpleSlotContext(aid1, tml1, 1, taskManagerGateway); + private SimpleSlotContext ssc2 = new SimpleSlotContext(aid2, tml2, 2, taskManagerGateway); + private SimpleSlotContext ssc3 = new SimpleSlotContext(aid3, tml3, 3, taskManagerGateway); + private SimpleSlotContext ssc4 = new SimpleSlotContext(aid4, tml4, 4, taskManagerGateway); + + private final Set candidates = Collections.unmodifiableSet(createCandidates()); + + private Set createCandidates() { + Set candidates = new HashSet<>(4); + candidates.add(ssc1); + candidates.add(ssc2); + candidates.add(ssc3); + candidates.add(ssc4); + return candidates; + } + + @Test + public void matchNoRequirements() { + + SlotProfile slotProfile = new SlotProfile(resourceProfile, Collections.emptyList(), Collections.emptyList()); + SlotContext match = runMatching(slotProfile); + + Assert.assertTrue(candidates.contains(match)); + } + + @Test + public void matchPreferredLocationNotAvailable() { + + SlotProfile slotProfile = new SlotProfile(resourceProfile, Collections.singletonList(tmlX), Collections.emptyList()); + SlotContext match = runMatching(slotProfile); + + Assert.assertTrue(candidates.contains(match)); + } + + @Test + public void matchPreferredLocation() { + + SlotProfile slotProfile = new SlotProfile(resourceProfile, Collections.singletonList(tml2), Collections.emptyList()); + SlotContext match = runMatching(slotProfile); + + Assert.assertEquals(ssc2, match); + + slotProfile = new SlotProfile(resourceProfile, Arrays.asList(tmlX, tml4), Collections.emptyList()); + match = runMatching(slotProfile); + + Assert.assertEquals(ssc4, match); + } + + @Test + public void matchPreviousAllocationOverridesPreferredLocation() { + + SlotProfile slotProfile = new SlotProfile(resourceProfile, Collections.singletonList(tml2), Collections.singletonList(aid3)); + SlotContext match = runMatching(slotProfile); + + Assert.assertEquals(ssc3, match); + + slotProfile = new SlotProfile(resourceProfile, Arrays.asList(tmlX, tml1), Arrays.asList(aidX, aid2)); + match = runMatching(slotProfile); + + Assert.assertEquals(ssc2, match); + } + + @Test + public void matchPreviousLocationNotAvailable() { + + SlotProfile slotProfile = new SlotProfile(resourceProfile, Collections.singletonList(tml4), Collections.singletonList(aidX)); + SlotContext match = runMatching(slotProfile); + + Assert.assertEquals(null, match); + } + + private SlotContext runMatching(SlotProfile slotProfile) { + SlotProfile.ProfileToSlotContextMatcher matcher = slotProfile.matcher(); + return matcher.findMatchWithLocality( + candidates.stream(), + (candidate) -> candidate, + (candidate) -> true, + (candidate, locality) -> candidate); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java index 1b835ad796d4b6..63b32381899ae7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphMetricsTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.JobException; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.concurrent.ScheduledExecutor; import org.apache.flink.runtime.execution.ExecutionState; import org.apache.flink.runtime.execution.SuppressRestartsException; @@ -44,7 +45,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; @@ -80,7 +80,7 @@ public void testExecutionGraphRestartTimeMetric() throws JobException, IOExcepti CompletableFuture slotFuture1 = CompletableFuture.completedFuture(new TestingLogicalSlot()); CompletableFuture slotFuture2 = CompletableFuture.completedFuture(new TestingLogicalSlot()); - when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(slotFuture1, slotFuture2); + when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(SlotProfile.class), any(Time.class))).thenReturn(slotFuture1, slotFuture2); TestingRestartStrategy testingRestartStrategy = new TestingRestartStrategy(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java index c0d5dc0a6b2852..51d1827e668341 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexSchedulingTest.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.akka.AkkaUtils; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.execution.ExecutionState; import org.apache.flink.runtime.instance.DummyActorGateway; import org.apache.flink.runtime.instance.Instance; @@ -35,7 +36,6 @@ import org.junit.Test; -import java.util.Collection; import java.util.concurrent.CompletableFuture; import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.getExecutionVertex; @@ -67,7 +67,7 @@ public void testSlotReleasedWhenScheduledImmediately() { Scheduler scheduler = mock(Scheduler.class); CompletableFuture future = new CompletableFuture<>(); future.complete(slot); - when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(future); + when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(SlotProfile.class), any(Time.class))).thenReturn(future); assertEquals(ExecutionState.CREATED, vertex.getExecutionState()); // try to deploy to the slot @@ -99,7 +99,7 @@ public void testSlotReleasedWhenScheduledQueued() { final CompletableFuture future = new CompletableFuture<>(); Scheduler scheduler = mock(Scheduler.class); - when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(future); + when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(SlotProfile.class), any(Time.class))).thenReturn(future); assertEquals(ExecutionState.CREATED, vertex.getExecutionState()); // try to deploy to the slot @@ -133,7 +133,7 @@ public void testScheduleToDeploying() { Scheduler scheduler = mock(Scheduler.class); CompletableFuture future = new CompletableFuture<>(); future.complete(slot); - when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(Collection.class), any(Time.class))).thenReturn(future); + when(scheduler.allocateSlot(any(SlotRequestId.class), any(ScheduledUnit.class), anyBoolean(), any(SlotProfile.class), any(Time.class))).thenReturn(future); assertEquals(ExecutionState.CREATED, vertex.getExecutionState()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java index 2e7ebc9771c3c9..daaf63b1cf7ec5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ProgrammedSlotProvider.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.executiongraph; import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; @@ -26,11 +27,9 @@ import org.apache.flink.runtime.jobmaster.SlotRequestId; import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import javax.annotation.Nullable; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -117,7 +116,7 @@ public CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit task, boolean allowQueued, - Collection preferredLocations, + SlotProfile slotProfile, Time allocationTimeout) { JobVertexID vertexId = task.getTaskToExecute().getVertex().getJobvertexId(); int subtask = task.getTaskToExecute().getParallelSubtaskIndex(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java index c082e9a1fc7d2d..7d11d379c6636c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java @@ -22,19 +22,20 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.concurrent.FutureUtils; -import org.apache.flink.runtime.instance.SlotSharingGroupId; -import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.instance.SimpleSlot; +import org.apache.flink.runtime.instance.SimpleSlotContext; import org.apache.flink.runtime.instance.Slot; -import org.apache.flink.runtime.jobmaster.SlotRequestId; -import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; +import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException; import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; -import org.apache.flink.runtime.instance.SimpleSlotContext; +import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; +import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.jobmaster.SlotContext; import org.apache.flink.runtime.jobmaster.SlotOwner; -import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import org.apache.flink.util.FlinkException; @@ -44,7 +45,6 @@ import java.net.InetAddress; import java.util.ArrayDeque; -import java.util.Collection; import java.util.HashMap; import java.util.concurrent.CompletableFuture; @@ -89,7 +89,7 @@ public CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit task, boolean allowQueued, - Collection preferredLocations, + SlotProfile slotProfile, Time allocationTimeout) { final SlotContext slot; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java index ecd88bcc4dbccd..ed3c361fd5663b 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleWithCoLocationHintTest.java @@ -19,8 +19,10 @@ package org.apache.flink.runtime.jobmanager.scheduler; import org.apache.flink.runtime.clusterframework.types.ResourceID; -import org.apache.flink.runtime.jobmaster.LogicalSlot; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.jobgraph.JobVertexID; +import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import org.apache.flink.runtime.testingUtils.TestingUtils; @@ -67,18 +69,18 @@ public void scheduleAllSharedAndCoLocated() throws Exception { CoLocationConstraint c6 = new CoLocationConstraint(ccg); // schedule 4 tasks from the first vertex group - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s9 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s10 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s11 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s12 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c2), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c3), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s9 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s10 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c4), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s11 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c5), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s12 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 6, sharingGroup), sharingGroup.getSlotSharingGroupId(), c6), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1); assertNotNull(s2); @@ -128,7 +130,7 @@ public void scheduleAllSharedAndCoLocated() throws Exception { assertTrue(testingSlotProvider.getNumberOfAvailableSlots() >= 1); LogicalSlot single = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(new JobVertexID(), 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(new JobVertexID(), 0, 1, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(single); s1.releaseSlot(); @@ -165,11 +167,11 @@ public void scheduleWithIntermediateRelease() throws Exception { CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup()); LogicalSlot s1 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot s2 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); - LogicalSlot sSolo = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot sSolo = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 1, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); ResourceID taskManager = s1.getTaskManagerLocation().getResourceID(); @@ -178,7 +180,7 @@ public void scheduleWithIntermediateRelease() throws Exception { sSolo.releaseSlot(); LogicalSlot sNew = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertEquals(taskManager, sNew.getTaskManagerLocation().getResourceID()); assertEquals(2, testingSlotProvider.getNumberOfLocalizedAssignments()); @@ -201,14 +203,14 @@ public void scheduleWithReleaseNoResource() throws Exception { CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup()); LogicalSlot s1 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); s1.releaseSlot(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduled even though no resource was available."); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof NoResourceAvailableException); @@ -242,35 +244,35 @@ public void scheduleMixedCoLocationSlotSharing() throws Exception { SlotSharingGroup shareGroup = new SlotSharingGroup(); // first wave - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // second wave LogicalSlot s21 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid2, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid2, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot s22 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid2, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid2, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot s23 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid2, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid2, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot s24 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid2, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid2, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // third wave LogicalSlot s31 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid3, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid3, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc2), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot s32 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid3, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid3, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc3), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot s33 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid3, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid3, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc4), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot s34 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertex(jid3, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertex(jid3, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId(), clc1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, shareGroup), shareGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertEquals(s21.getTaskManagerLocation(), s34.getTaskManagerLocation()); assertEquals(s22.getTaskManagerLocation(), s31.getTaskManagerLocation()); @@ -302,25 +304,25 @@ public void testGetsNonLocalFromSharingGroupFirst() throws Exception { // schedule something into the shared group so that both instances are in the sharing group LogicalSlot s1 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s2 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); // schedule one locally to instance 1 LogicalSlot s3 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); // schedule with co location constraint (yet unassigned) and a preference for // instance 1, but it can only get instance 2 LogicalSlot s4 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); // schedule something into the assigned co-location constraints and check that they override the // other preferences LogicalSlot s5 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid3, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid3, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); LogicalSlot s6 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid3, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid3, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); // check that each slot got three assertEquals(s1.getTaskManagerLocation(), s3.getTaskManagerLocation()); @@ -362,9 +364,9 @@ public void testSlotReleasedInBetween() throws Exception { CoLocationConstraint cc2 = new CoLocationConstraint(ccg); LogicalSlot s1 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s2 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); s1.releaseSlot(); s2.releaseSlot(); @@ -373,9 +375,9 @@ public void testSlotReleasedInBetween() throws Exception { assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots()); LogicalSlot s3 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); LogicalSlot s4 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); // still preserves the previous instance mapping) assertEquals(loc1, s3.getTaskManagerLocation()); @@ -409,9 +411,9 @@ public void testSlotReleasedInBetweenAndNoNewLocal() throws Exception { CoLocationConstraint cc2 = new CoLocationConstraint(ccg); LogicalSlot s1 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s2 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); s1.releaseSlot(); s2.releaseSlot(); @@ -420,13 +422,13 @@ public void testSlotReleasedInBetweenAndNoNewLocal() throws Exception { assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfSlots()); LogicalSlot sa = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jidx, 0, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jidx, 0, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); LogicalSlot sb = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jidx, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jidx, 1, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); try { testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); fail("should not be able to find a resource"); } catch (ExecutionException e) { @@ -466,14 +468,14 @@ public void testScheduleOutOfOrder() throws Exception { // and give locality preferences that hint at using the same shared slot for both // co location constraints (which we seek to prevent) LogicalSlot s1 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s2 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s3 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s4 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); // check that each slot got three assertEquals(s1.getTaskManagerLocation(), s3.getTaskManagerLocation()); @@ -515,14 +517,14 @@ public void nonColocationFollowsCoLocation() throws Exception { CoLocationConstraint cc2 = new CoLocationConstraint(ccg); LogicalSlot s1 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s2 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); LogicalSlot s3 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); LogicalSlot s4 = testingSlotProvider.allocateSlot( - new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); // check that each slot got two assertEquals(s1.getTaskManagerLocation(), s3.getTaskManagerLocation()); @@ -539,4 +541,8 @@ public void nonColocationFollowsCoLocation() throws Exception { assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid1)); assertEquals(0, sharingGroup.getTaskAssignment().getNumberOfAvailableSlotsForGroup(jid2)); } + + private static SlotProfile slotProfileForLocation(TaskManagerLocation location) { + return new SlotProfile(ResourceProfile.UNKNOWN, Collections.singletonList(location), Collections.emptyList()); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java index 86f565952ce821..2abf9fbddad1ce 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerIsolatedTasksTest.java @@ -19,6 +19,8 @@ package org.apache.flink.runtime.jobmanager.scheduler; import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.instance.Instance; import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; @@ -68,17 +70,17 @@ public void testScheduleImmediately() throws Exception { assertEquals(5, testingSlotProvider.getNumberOfAvailableSlots()); // schedule something into all slots - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // the slots should all be different assertTrue(areAllDistinct(s1, s2, s3, s4, s5)); try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted scheduling request without available resource."); } catch (ExecutionException e) { @@ -91,8 +93,8 @@ public void testScheduleImmediately() throws Exception { assertEquals(2, testingSlotProvider.getNumberOfAvailableSlots()); // now we can schedule some more slots - LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertTrue(areAllDistinct(s1, s2, s3, s4, s5, s6, s7)); @@ -172,7 +174,7 @@ public void run() { disposeThread.start(); for (int i = 0; i < NUM_TASKS_TO_SCHEDULE; i++) { - CompletableFuture future = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), true, Collections.emptyList(), TestingUtils.infiniteTime()); + CompletableFuture future = testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), true, SlotProfile.noRequirements(), TestingUtils.infiniteTime()); future.thenAcceptAsync( (LogicalSlot slot) -> { synchronized (toRelease) { @@ -207,11 +209,11 @@ public void testScheduleWithDyingInstances() throws Exception { final TaskManagerLocation taskManagerLocation3 = testingSlotProvider.addTaskManager(1); List slots = new ArrayList<>(); - slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get()); - slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get()); - slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get()); - slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get()); - slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get()); + slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get()); + slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get()); + slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get()); + slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get()); + slots.add(testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get()); testingSlotProvider.releaseTaskManager(taskManagerLocation2.getResourceID()); @@ -232,7 +234,7 @@ public void testScheduleWithDyingInstances() throws Exception { // cannot get another slot, since all instances are dead try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getDummyTask()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler served a slot from a dead instance"); } catch (ExecutionException e) { @@ -254,7 +256,7 @@ public void testSchedulingLocation() throws Exception { final TaskManagerLocation taskManagerLocation3 = testingSlotProvider.addTaskManager(2); // schedule something on an arbitrary instance - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(new Instance[0])), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(new Instance[0])), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // figure out how we use the location hints ResourceID firstResourceId = s1.getTaskManagerLocation().getResourceID(); @@ -276,32 +278,36 @@ public void testSchedulingLocation() throws Exception { TaskManagerLocation third = taskManagerLocations.get((index + 2) % taskManagerLocations.size()); // something that needs to go to the first instance again - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(s1.getTaskManagerLocation())), false, Collections.singleton(s1.getTaskManagerLocation()), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(s1.getTaskManagerLocation())), false, slotProfileForLocation(s1.getTaskManagerLocation()), TestingUtils.infiniteTime()).get(); assertEquals(first.getResourceID(), s2.getTaskManagerLocation().getResourceID()); // first or second --> second, because first is full - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, second)), false, Arrays.asList(first, second), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, second)), false, slotProfileForLocation(first, second), TestingUtils.infiniteTime()).get(); assertEquals(second.getResourceID(), s3.getTaskManagerLocation().getResourceID()); // first or third --> third (because first is full) - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get(); - LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, slotProfileForLocation(first, third), TestingUtils.infiniteTime()).get(); + LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, slotProfileForLocation(first, third), TestingUtils.infiniteTime()).get(); assertEquals(third.getResourceID(), s4.getTaskManagerLocation().getResourceID()); assertEquals(third.getResourceID(), s5.getTaskManagerLocation().getResourceID()); // first or third --> second, because all others are full - LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get(); + LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, slotProfileForLocation(first, third), TestingUtils.infiniteTime()).get(); assertEquals(second.getResourceID(), s6.getTaskManagerLocation().getResourceID()); // release something on the first and second instance s2.releaseSlot(); s6.releaseSlot(); - LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, Arrays.asList(first, third), TestingUtils.infiniteTime()).get(); + LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(first, third)), false, slotProfileForLocation(first, third), TestingUtils.infiniteTime()).get(); assertEquals(first.getResourceID(), s7.getTaskManagerLocation().getResourceID()); assertEquals(1, testingSlotProvider.getNumberOfUnconstrainedAssignments()); assertTrue(1 == testingSlotProvider.getNumberOfNonLocalizedAssignments() || 1 == testingSlotProvider.getNumberOfHostLocalizedAssignments()); assertEquals(5, testingSlotProvider.getNumberOfLocalizedAssignments()); } + + private static SlotProfile slotProfileForLocation(TaskManagerLocation... location) { + return new SlotProfile(ResourceProfile.UNKNOWN, Arrays.asList(location), Collections.emptyList()); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java index 1cc930152aa984..aab8132a07b322 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerSlotSharingTest.java @@ -19,6 +19,8 @@ package org.apache.flink.runtime.jobmanager.scheduler; import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; @@ -68,10 +70,10 @@ public void scheduleSingleVertexType() { testingSlotProvider.addTaskManager(2); // schedule 4 tasks from the first vertex group - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1); assertNotNull(s2); @@ -82,7 +84,7 @@ public void scheduleSingleVertexType() { // we cannot schedule another task from the first vertex group try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -96,7 +98,7 @@ public void scheduleSingleVertexType() { s3.releaseSlot(); // allocate another slot from that group - LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s5); // release all old slots @@ -104,9 +106,9 @@ public void scheduleSingleVertexType() { s2.releaseSlot(); s4.releaseSlot(); - LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 6, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 7, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 5, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s7 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 6, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s8 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 7, 8, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s6); assertNotNull(s7); @@ -151,10 +153,10 @@ public void allocateSlotWithSharing() throws Exception { testingSlotProvider.addTaskManager(2); // schedule 4 tasks from the first vertex group - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1); assertNotNull(s2); @@ -165,7 +167,7 @@ public void allocateSlotWithSharing() throws Exception { // we cannot schedule another task from the first vertex group try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -176,10 +178,10 @@ public void allocateSlotWithSharing() throws Exception { } // schedule some tasks from the second ID group - LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_2); assertNotNull(s2_2); @@ -188,7 +190,7 @@ public void allocateSlotWithSharing() throws Exception { // we cannot schedule another task from the second vertex group try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -209,7 +211,7 @@ public void allocateSlotWithSharing() throws Exception { // we can still not schedule anything from the second group of vertices try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -220,7 +222,7 @@ public void allocateSlotWithSharing() throws Exception { } // we can schedule something from the first vertex group - LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s5); assertEquals(4, testingSlotProvider.getNumberOfSlots(sharingGroup)); @@ -230,7 +232,7 @@ public void allocateSlotWithSharing() throws Exception { // now we release a slot from the second vertex group and schedule another task from that group s2_2.releaseSlot(); - LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s5_2); // release all slots @@ -265,10 +267,10 @@ public void allocateSlotWithIntermediateTotallyEmptySharingGroup() { testingSlotProvider.addTaskManager(2); // schedule 4 tasks from the first vertex group - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertEquals(4, testingSlotProvider.getNumberOfSlots(sharingGroup)); assertEquals(0, testingSlotProvider.getNumberOfAvailableSlotsForGroup(sharingGroup, jid1)); @@ -284,10 +286,10 @@ public void allocateSlotWithIntermediateTotallyEmptySharingGroup() { assertEquals(0, testingSlotProvider.getNumberOfAvailableSlotsForGroup(sharingGroup, jid2)); // schedule some tasks from the second ID group - LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertEquals(4, testingSlotProvider.getNumberOfSlots(sharingGroup)); assertEquals(4, testingSlotProvider.getNumberOfAvailableSlotsForGroup(sharingGroup, jid1)); @@ -329,10 +331,10 @@ public void allocateSlotWithTemporarilyEmptyVertexGroup() { testingSlotProvider.addTaskManager(2); // schedule 4 tasks from the first vertex group - LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_1); assertNotNull(s2_1); @@ -342,10 +344,10 @@ public void allocateSlotWithTemporarilyEmptyVertexGroup() { assertTrue(areAllDistinct(s1_1, s2_1, s3_1, s4_1)); // schedule 4 tasks from the second vertex group - LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_2); assertNotNull(s2_2); @@ -355,10 +357,10 @@ public void allocateSlotWithTemporarilyEmptyVertexGroup() { assertTrue(areAllDistinct(s1_2, s2_2, s3_2, s4_2)); // schedule 4 tasks from the third vertex group - LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_3); assertNotNull(s2_3); @@ -370,7 +372,7 @@ public void allocateSlotWithTemporarilyEmptyVertexGroup() { // we cannot schedule another task from the second vertex group try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -386,9 +388,9 @@ public void allocateSlotWithTemporarilyEmptyVertexGroup() { s3_2.releaseSlot(); s4_2.releaseSlot(); - LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s6_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 6, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s7_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 7, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s5_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 5, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s6_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 6, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s7_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 7, 7, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s5_2); assertNotNull(s6_2); @@ -438,9 +440,9 @@ public void allocateSlotWithTemporarilyEmptyVertexGroup2() { testingSlotProvider.addTaskManager(2); // schedule 1 tasks from the first vertex group and 2 from the second - LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_1); assertNotNull(s2_1); @@ -456,7 +458,7 @@ public void allocateSlotWithTemporarilyEmptyVertexGroup2() { // this should free one slot so we can allocate one non-shared - LogicalSlot sx = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot sx = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(sx); assertEquals(1, testingSlotProvider.getNumberOfSlots(sharingGroup)); @@ -490,28 +492,28 @@ public void scheduleMixedSharingAndNonSharing() { testingSlotProvider.addTaskManager(2); // schedule some individual vertices - LogicalSlot sA2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot sA1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 0, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot sA2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 1, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot sA1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidA, 0, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(sA1); assertNotNull(sA2); // schedule some vertices in the sharing group - LogicalSlot s1_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_0); assertNotNull(s1_1); assertNotNull(s2_0); assertNotNull(s2_1); // schedule another isolated vertex - LogicalSlot sB1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 1, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot sB1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 1, 3, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(sB1); // should not be able to schedule more vertices try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -522,7 +524,7 @@ public void scheduleMixedSharingAndNonSharing() { } try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -533,7 +535,7 @@ public void scheduleMixedSharingAndNonSharing() { } try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -544,7 +546,7 @@ public void scheduleMixedSharingAndNonSharing() { } try { - testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 1, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 1, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("Scheduler accepted too many tasks at the same time"); } catch (ExecutionException e) { @@ -557,8 +559,8 @@ public void scheduleMixedSharingAndNonSharing() { // release some isolated task and check that the sharing group may grow sA1.releaseSlot(); - LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_2); assertNotNull(s2_2); @@ -570,19 +572,19 @@ public void scheduleMixedSharingAndNonSharing() { assertEquals(1, testingSlotProvider.getNumberOfAvailableSlots()); // schedule one more no-shared task - LogicalSlot sB0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot sB0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 0, 3, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(sB0); // release the last of the original shared slots and allocate one more non-shared slot s2_1.releaseSlot(); - LogicalSlot sB2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 2, 3, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot sB2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidB, 2, 3, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(sB2); // release on non-shared and add some shared slots sA2.releaseSlot(); - LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(s1_3); assertNotNull(s2_3); @@ -592,8 +594,8 @@ public void scheduleMixedSharingAndNonSharing() { s1_3.releaseSlot(); s2_3.releaseSlot(); - LogicalSlot sC0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 1, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot sC1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 2, null)), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot sC0 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 1, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot sC1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jidC, 0, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertNotNull(sC0); assertNotNull(sC1); @@ -633,8 +635,8 @@ public void testLocalizedAssignment1() { TaskManagerLocation loc2 = testingSlotProvider.addTaskManager(2); // schedule one to each instance - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); assertNotNull(s1); assertNotNull(s2); @@ -643,8 +645,8 @@ public void testLocalizedAssignment1() { assertEquals(loc2, s2.getTaskManagerLocation()); // schedule one from the other group to each instance - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); assertNotNull(s3); assertNotNull(s4); @@ -679,8 +681,8 @@ public void testLocalizedAssignment2() { TaskManagerLocation loc2 = testingSlotProvider.addTaskManager(2); // schedule one to each instance - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); assertNotNull(s1); assertNotNull(s2); @@ -689,8 +691,8 @@ public void testLocalizedAssignment2() { assertEquals(loc1, s2.getTaskManagerLocation()); // schedule one from the other group to each instance - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc2), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get(); assertNotNull(s3); assertNotNull(s4); @@ -724,14 +726,14 @@ public void testLocalizedAssignment3() { TaskManagerLocation loc2 = testingSlotProvider.addTaskManager(2); // schedule until the one instance is full - LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); - LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); - LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); // schedule two more with preference of same instance --> need to go to other instance - LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); - LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, Collections.singleton(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s5 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 3, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); + LogicalSlot s6 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertexWithLocation(jid2, 4, 4, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get(); assertNotNull(s1); assertNotNull(s2); @@ -775,19 +777,19 @@ public void testSequentialAllocateAndRelease() { testingSlotProvider.addTaskManager(4); // allocate something from group 1 and 2 interleaved with schedule for group 3 - LogicalSlot slot_1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_1_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_1_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_2_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_2_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_1_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_1_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_1_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_2_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_2_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_2_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // release groups 1 and 2 @@ -803,10 +805,10 @@ public void testSequentialAllocateAndRelease() { // allocate group 4 - LogicalSlot slot_4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot slot_4_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_4_1 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_4_2 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_4_3 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot_4_4 = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // release groups 3 and 4 @@ -855,7 +857,7 @@ public void testConcurrentAllocateAndRelease() { @Override public void run() { try { - LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, enumerator4.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid4, enumerator4.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); sleepUninterruptibly(rnd.nextInt(5)); slot.releaseSlot(); @@ -877,7 +879,7 @@ public void run() { public void run() { try { if (flag3.compareAndSet(false, true)) { - LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); sleepUninterruptibly(5); executor.execute(deploy4); @@ -905,7 +907,7 @@ public void run() { @Override public void run() { try { - LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, enumerator2.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, enumerator2.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // wait a bit till scheduling the successor sleepUninterruptibly(rnd.nextInt(5)); @@ -932,7 +934,7 @@ public void run() { @Override public void run() { try { - LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, enumerator1.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot slot = testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid1, enumerator1.getAndIncrement(), 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); // wait a bit till scheduling the successor sleepUninterruptibly(rnd.nextInt(5)); @@ -1005,27 +1007,27 @@ public void testDopIncreases() { scheduler.newInstanceAvailable(getRandomInstance(4)); // schedule one task for the first and second vertex - LogicalSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); assertEquals( s1.getTaskManagerLocation(), s2.getTaskManagerLocation() ); assertEquals(3, scheduler.getNumberOfAvailableSlots()); - LogicalSlot s3_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 1, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_0 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 0, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_1 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 1, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); s1.releaseSlot(); s2.releaseSlot(); - LogicalSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s3_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); - LogicalSlot s4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 2, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s3_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 3, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_2 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 2, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); + LogicalSlot s4_3 = scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid4, 3, 4, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); try { - scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, Collections.emptyList(), TestingUtils.infiniteTime()).get(); + scheduler.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 4, 5, sharingGroup), sharingGroup.getSlotSharingGroupId()), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get(); fail("should throw an exception"); } catch (ExecutionException e) { @@ -1050,4 +1052,8 @@ public void testDopIncreases() { fail(e.getMessage()); } } + + private static SlotProfile slotProfileForLocation(TaskManagerLocation location) { + return new SlotProfile(ResourceProfile.UNKNOWN, Collections.singletonList(location), Collections.emptyList()); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java index a28e8905222537..d9919acdaa4b28 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTest.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.jobmanager.scheduler; import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.executiongraph.Execution; import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; import org.apache.flink.runtime.executiongraph.ExecutionGraph; @@ -32,7 +33,6 @@ import org.hamcrest.Matchers; import org.junit.Test; -import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -123,7 +123,7 @@ public void testSlotAllocationTimeout() throws Exception { new ScheduledUnit( execution), true, - Collections.emptyList(), + SlotProfile.noRequirements(), Time.milliseconds(1L)); try { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java index 9cb9cffb633290..9738449e663432 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/SchedulerTestBase.java @@ -23,6 +23,7 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.Instance; import org.apache.flink.runtime.instance.SlotSharingGroupId; @@ -156,8 +157,13 @@ private TestingSchedulerSlotProvider(Scheduler scheduler) { } @Override - public CompletableFuture allocateSlot(SlotRequestId slotRequestId, ScheduledUnit task, boolean allowQueued, Collection preferredLocations, Time allocationTimeout) { - return scheduler.allocateSlot(task, allowQueued, preferredLocations, allocationTimeout); + public CompletableFuture allocateSlot( + SlotRequestId slotRequestId, + ScheduledUnit task, + boolean allowQueued, + SlotProfile slotProfile, + Time allocationTimeout) { + return scheduler.allocateSlot(task, allowQueued, slotProfile, allocationTimeout); } @Override @@ -349,8 +355,13 @@ public void shutdown() throws Exception { } @Override - public CompletableFuture allocateSlot(SlotRequestId slotRequestId, ScheduledUnit task, boolean allowQueued, Collection preferredLocations, Time allocationTimeout) { - return slotProvider.allocateSlot(task, allowQueued, preferredLocations, allocationTimeout).thenApply( + public CompletableFuture allocateSlot( + SlotRequestId slotRequestId, + ScheduledUnit task, + boolean allowQueued, + SlotProfile slotProfile, + Time allocationTimeout) { + return slotProvider.allocateSlot(task, allowQueued, slotProfile, allocationTimeout).thenApply( (LogicalSlot logicalSlot) -> { switch (logicalSlot.getLocality()) { case LOCAL: diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/AvailableSlotsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/AvailableSlotsTest.java index 2d18c6510baaac..c0074ed55ee397 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/AvailableSlotsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/AvailableSlotsTest.java @@ -21,6 +21,7 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.jobmanager.slots.SlotAndLocality; import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; @@ -98,9 +99,9 @@ public void testPollFreeSlot() { assertTrue(availableSlots.contains(slot1.getAllocationId())); assertTrue(availableSlots.containsTaskManager(resource1)); - assertNull(availableSlots.poll(DEFAULT_TESTING_BIG_PROFILE, null)); + assertNull(availableSlots.poll(SlotProfile.noLocality(DEFAULT_TESTING_BIG_PROFILE))); - SlotAndLocality slotAndLocality = availableSlots.poll(DEFAULT_TESTING_PROFILE, null); + SlotAndLocality slotAndLocality = availableSlots.poll(SlotProfile.noLocality(DEFAULT_TESTING_PROFILE)); assertEquals(slot1, slotAndLocality.getSlot()); assertEquals(0, availableSlots.size()); assertFalse(availableSlots.contains(slot1.getAllocationId())); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java index 9a0256cff051d3..03757607d9f3ff 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolCoLocationTest.java @@ -20,6 +20,7 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobgraph.JobVertexID; @@ -35,7 +36,6 @@ import org.junit.Test; -import java.util.Collections; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; @@ -79,7 +79,7 @@ public void testSimpleCoLocatedSlotScheduling() throws ExecutionException, Inter slotSharingGroupId, coLocationConstraint1), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture22 = slotProvider.allocateSlot( @@ -88,7 +88,7 @@ public void testSimpleCoLocatedSlotScheduling() throws ExecutionException, Inter slotSharingGroupId, coLocationConstraint2), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture12 = slotProvider.allocateSlot( @@ -97,7 +97,7 @@ public void testSimpleCoLocatedSlotScheduling() throws ExecutionException, Inter slotSharingGroupId, coLocationConstraint1), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture21 = slotProvider.allocateSlot( @@ -106,7 +106,7 @@ public void testSimpleCoLocatedSlotScheduling() throws ExecutionException, Inter slotSharingGroupId, coLocationConstraint2), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); final AllocationID allocationId1 = allocationIds.take(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java index b2be97ef29c859..cc837bc0496529 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java @@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobmanager.scheduler.DummyScheduledUnit; @@ -58,7 +59,6 @@ import javax.annotation.Nullable; -import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; @@ -121,8 +121,7 @@ public void testSlotAllocationNoResourceManager() throws Exception { CompletableFuture future = pool.allocateSlot( new SlotRequestId(), new ScheduledUnit(SchedulerTestUtils.getDummyTask()), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, fastTimeout); @@ -158,8 +157,7 @@ public void testCancelSlotAllocationWithoutResourceManager() throws Exception { CompletableFuture future = slotPoolGateway.allocateSlot( requestId, new ScheduledUnit(SchedulerTestUtils.getDummyTask()), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, fastTimeout); @@ -204,8 +202,7 @@ public void testSlotAllocationTimeout() throws Exception { CompletableFuture future = slotPoolGateway.allocateSlot( requestId, new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, fastTimeout); @@ -252,8 +249,7 @@ public void testExtraSlotsAreKept() throws Exception { CompletableFuture future = slotPoolGateway.allocateSlot( requestId, new ScheduledUnit(SchedulerTestUtils.getDummyTask()), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, fastTimeout); @@ -313,7 +309,7 @@ public void testProviderAndOwnerSlotAllocationTimeout() throws Exception { CompletableFuture future = pool.getSlotProvider().allocateSlot( new DummyScheduledUnit(), true, - Collections.emptyList(), + SlotProfile.noRequirements(), fastTimeout); try { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java index b7fa484132d26a..f33e1a103e0de9 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolSlotSharingTest.java @@ -20,6 +20,7 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobgraph.JobVertexID; @@ -35,7 +36,6 @@ import org.junit.Test; -import java.util.Collections; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; @@ -67,7 +67,7 @@ public void testSingleQueuedSharedSlotScheduling() throws Exception { slotSharingGroupId, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); assertFalse(logicalSlotFuture.isDone()); @@ -104,7 +104,7 @@ public void testFailingQueuedSharedSlotScheduling() throws ExecutionException, I new SlotSharingGroupId(), null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); final AllocationID allocationId = allocationIdFuture.get(); @@ -143,7 +143,7 @@ public void testQueuedSharedSlotScheduling() throws InterruptedException, Execut slotSharingGroupId, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture2 = slotProvider.allocateSlot( @@ -152,7 +152,7 @@ public void testQueuedSharedSlotScheduling() throws InterruptedException, Execut slotSharingGroupId, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); assertFalse(logicalSlotFuture1.isDone()); @@ -166,7 +166,7 @@ public void testQueuedSharedSlotScheduling() throws InterruptedException, Execut slotSharingGroupId, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture4 = slotProvider.allocateSlot( @@ -175,7 +175,7 @@ public void testQueuedSharedSlotScheduling() throws InterruptedException, Execut slotSharingGroupId, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); assertFalse(logicalSlotFuture3.isDone()); @@ -242,7 +242,7 @@ public void testQueuedMultipleSlotSharingGroups() throws ExecutionException, Int slotSharingGroupId1, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture2 = slotProvider.allocateSlot( @@ -251,7 +251,7 @@ public void testQueuedMultipleSlotSharingGroups() throws ExecutionException, Int slotSharingGroupId1, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture3 = slotProvider.allocateSlot( @@ -260,7 +260,7 @@ public void testQueuedMultipleSlotSharingGroups() throws ExecutionException, Int slotSharingGroupId2, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); CompletableFuture logicalSlotFuture4 = slotProvider.allocateSlot( @@ -269,7 +269,7 @@ public void testQueuedMultipleSlotSharingGroups() throws ExecutionException, Int slotSharingGroupId2, null), true, - Collections.emptyList(), + SlotProfile.noRequirements(), TestingUtils.infiniteTime()); assertFalse(logicalSlotFuture1.isDone()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java index d6e05217f47a99..c529ceb3c1a004 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobgraph.JobVertexID; @@ -122,8 +123,7 @@ public void testAllocateSimpleSlot() throws Exception { CompletableFuture future = slotPoolGateway.allocateSlot( requestId, new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); assertFalse(future.isDone()); @@ -165,15 +165,13 @@ public void testAllocationFulfilledByReturnedSlot() throws Exception { CompletableFuture future1 = slotPoolGateway.allocateSlot( new SlotRequestId(), new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); CompletableFuture future2 = slotPoolGateway.allocateSlot( new SlotRequestId(), new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); @@ -229,8 +227,7 @@ public void testAllocateWithFreeSlot() throws Exception { CompletableFuture future1 = slotPoolGateway.allocateSlot( new SlotRequestId(), new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); assertFalse(future1.isDone()); @@ -253,8 +250,7 @@ public void testAllocateWithFreeSlot() throws Exception { CompletableFuture future2 = slotPoolGateway.allocateSlot( new SlotRequestId(), new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); @@ -287,8 +283,7 @@ public void testOfferSlot() throws Exception { CompletableFuture future = slotPoolGateway.allocateSlot( new SlotRequestId(), new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); assertFalse(future.isDone()); @@ -362,8 +357,7 @@ public CompletableFuture releaseSlot( CompletableFuture future1 = slotPoolGateway.allocateSlot( new SlotRequestId(), new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); @@ -372,8 +366,7 @@ public CompletableFuture releaseSlot( CompletableFuture future2 = slotPoolGateway.allocateSlot( new SlotRequestId(), new DummyScheduledUnit(), - DEFAULT_TESTING_PROFILE, - Collections.emptyList(), + SlotProfile.noLocality(DEFAULT_TESTING_PROFILE), true, timeout); @@ -427,11 +420,15 @@ public void testSlotRequestCancellationUponFailingRequest() throws Exception { try { final SlotPoolGateway slotPoolGateway = setupSlotPool(slotPool, resourceManagerGateway); + SlotProfile slotProfile = new SlotProfile( + ResourceProfile.UNKNOWN, + Collections.emptyList(), + Collections.emptyList()); + CompletableFuture slotFuture = slotPoolGateway.allocateSlot( new SlotRequestId(), scheduledUnit, - ResourceProfile.UNKNOWN, - Collections.emptyList(), + slotProfile, true, timeout); @@ -485,8 +482,7 @@ public void testFulfillingSlotRequestsWithUnusedOfferedSlots() throws Exception CompletableFuture slotFuture1 = slotPoolGateway.allocateSlot( slotRequestId1, scheduledUnit, - ResourceProfile.UNKNOWN, - Collections.emptyList(), + SlotProfile.noRequirements(), true, timeout); @@ -496,8 +492,7 @@ public void testFulfillingSlotRequestsWithUnusedOfferedSlots() throws Exception CompletableFuture slotFuture2 = slotPoolGateway.allocateSlot( slotRequestId2, scheduledUnit, - ResourceProfile.UNKNOWN, - Collections.emptyList(), + SlotProfile.noRequirements(), true, timeout); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManagerTest.java index c18b3b3bb8d54a..ec6eae2e6ee2db 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotSharingManagerTest.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.jobmaster.slotpool; import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.SimpleSlotContext; import org.apache.flink.runtime.instance.SlotSharingGroupId; @@ -415,7 +416,8 @@ public void testGetResolvedSlot() { new SlotRequestId()); AbstractID groupId = new AbstractID(); - SlotSharingManager.MultiTaskSlotLocality resolvedRootSlotLocality = slotSharingManager.getResolvedRootSlot(groupId, Collections.emptyList()); + SlotSharingManager.MultiTaskSlotLocality resolvedRootSlotLocality = + slotSharingManager.getResolvedRootSlot(groupId, SlotProfile.noRequirements().matcher()); assertNotNull(resolvedRootSlotLocality); assertEquals(Locality.UNCONSTRAINED, resolvedRootSlotLocality.getLocality()); @@ -431,7 +433,7 @@ public void testGetResolvedSlot() { SlotSharingManager.MultiTaskSlotLocality resolvedRootSlot1 = slotSharingManager.getResolvedRootSlot( groupId, - Collections.emptyList()); + SlotProfile.noRequirements().matcher()); assertNull(resolvedRootSlot1); } @@ -470,7 +472,9 @@ public void testGetResolvedSlotWithLocationPreferences() { new SlotRequestId()); AbstractID groupId = new AbstractID(); - SlotSharingManager.MultiTaskSlotLocality resolvedRootSlot1 = slotSharingManager.getResolvedRootSlot(groupId, Collections.singleton(taskManagerLocation)); + SlotProfile.LocalityAwareRequirementsToSlotMatcher matcher = + new SlotProfile.LocalityAwareRequirementsToSlotMatcher(Collections.singleton(taskManagerLocation)); + SlotSharingManager.MultiTaskSlotLocality resolvedRootSlot1 = slotSharingManager.getResolvedRootSlot(groupId, matcher); assertNotNull(resolvedRootSlot1); assertEquals(Locality.LOCAL, resolvedRootSlot1.getLocality()); assertEquals(rootSlot2.getSlotRequestId(), resolvedRootSlot1.getMultiTaskSlot().getSlotRequestId()); @@ -481,7 +485,7 @@ public void testGetResolvedSlotWithLocationPreferences() { groupId, resolvedRootSlot1.getLocality()); - SlotSharingManager.MultiTaskSlotLocality resolvedRootSlot2 = slotSharingManager.getResolvedRootSlot(groupId, Collections.singleton(taskManagerLocation)); + SlotSharingManager.MultiTaskSlotLocality resolvedRootSlot2 = slotSharingManager.getResolvedRootSlot(groupId,matcher); assertNotNull(resolvedRootSlot2); assertNotSame(Locality.LOCAL, (resolvedRootSlot2.getLocality())); From 0479d6f254c7a2dc1b7612bd57f726a13926aeb2 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Fri, 23 Feb 2018 19:25:06 +0800 Subject: [PATCH 0023/2294] [FLINK-8741] [kafka] Fix incorrect user code classloader in FlinkKafkaConsumer This commit fixes incorrectly using the parent of the user code class loader. Since Kafka 010 / 011 versions directly reuse 09 code, this fix fixes the issue for all versions. This commit also extends the Kafka010Example, so that is uses a custom watermark assigner. This allows our end-to-end tests to have caught this bug. --- .../kafka/internal/Kafka09Fetcher.java | 2 +- .../examples/kafka/Kafka010Example.java | 49 ++++++++++++++++++- .../test_streaming_kafka010.sh | 4 +- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java index 58393600633cf1..dcc67d5b4fb140 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java @@ -95,7 +95,7 @@ public Kafka09Fetcher( watermarksPunctuated, processingTimeProvider, autoWatermarkInterval, - userCodeClassLoader.getParent(), + userCodeClassLoader, consumerMetricGroup, useMetrics); diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java index 3fbd2b462cceae..881aa6785f67d4 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java @@ -21,16 +21,26 @@ import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.serialization.SimpleStringSchema; import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; +import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor; +import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; +import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; +import javax.annotation.Nullable; /** * An example that shows how to read from and write to Kafka. This will read String messages * from the input topic, prefix them by a configured prefix and output to the output topic. * + *

This example also demonstrates using a watermark assigner to generate per-partition + * watermarks directly in the Flink Kafka consumer. For demonstration purposes, it is assumed that + * the String messages are of formatted as a (message,timestamp) tuple. + * *

Example usage: * --input-topic test-input --output-topic test-output --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer */ @@ -59,11 +69,15 @@ public static void main(String[] args) throws Exception { // make parameters available in the web interface env.getConfig().setGlobalJobParameters(parameterTool); + env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); + DataStream input = env - .addSource(new FlinkKafkaConsumer010<>( + .addSource( + new FlinkKafkaConsumer010<>( parameterTool.getRequired("input-topic"), new SimpleStringSchema(), - parameterTool.getProperties())) + parameterTool.getProperties()) + .assignTimestampsAndWatermarks(new CustomWatermarkExtractor())) .map(new PrefixingMapper(prefix)); input.addSink( @@ -76,6 +90,9 @@ public static void main(String[] args) throws Exception { } private static class PrefixingMapper implements MapFunction { + + private static final long serialVersionUID = 1180234853172462378L; + private final String prefix; public PrefixingMapper(String prefix) { @@ -87,4 +104,32 @@ public String map(String value) throws Exception { return prefix + value; } } + + /** + * A custom {@link AssignerWithPeriodicWatermarks}, that simply assumes that the input stream + * records are strictly ascending. + * + *

Flink also ships some built-in convenience assigners, such as the + * {@link BoundedOutOfOrdernessTimestampExtractor} and {@link AscendingTimestampExtractor} + */ + private static class CustomWatermarkExtractor implements AssignerWithPeriodicWatermarks { + + private static final long serialVersionUID = -742759155861320823L; + + private long currentTimestamp = Long.MIN_VALUE; + + @Override + public long extractTimestamp(String element, long previousElementTimestamp) { + // the inputs are assumed to be of format (message,timestamp) + long timestamp = Long.valueOf(element.substring(element.indexOf(",") + 1)); + this.currentTimestamp = timestamp; + return timestamp; + } + + @Nullable + @Override + public Watermark getCurrentWatermark() { + return new Watermark(currentTimestamp == Long.MIN_VALUE ? Long.MIN_VALUE : currentTimestamp - 1); + } + } } diff --git a/test-infra/end-to-end-test/test_streaming_kafka010.sh b/test-infra/end-to-end-test/test_streaming_kafka010.sh index dda2db566b853f..51a570ab181d59 100755 --- a/test-infra/end-to-end-test/test_streaming_kafka010.sh +++ b/test-infra/end-to-end-test/test_streaming_kafka010.sh @@ -70,12 +70,12 @@ $FLINK_DIR/bin/flink run -d build-target/examples/streaming/Kafka010Example.jar --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer --auto.offset.reset earliest # send some data to Kafka -echo -e "hello\nwhats\nup" | $KAFKA_DIR/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test-input +echo -e "hello,45218\nwhats,46213\nup,51348" | $KAFKA_DIR/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test-input DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --max-messages 3 2> /dev/null) # make sure we have actual newlines in the string, not "\n" -EXPECTED=$(printf "PREFIX:hello\nPREFIX:whats\nPREFIX:up") +EXPECTED=$(printf "PREFIX:hello,45218\nPREFIX:whats,46213\nPREFIX:up,51348") if [[ "$DATA_FROM_KAFKA" != "$EXPECTED" ]]; then echo "Output from Flink program does not match expected output." echo -e "EXPECTED: --$EXPECTED--" From 246507a7dd1f9fc8644dbdd6ca724b048f2b8c61 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Mon, 26 Feb 2018 16:11:04 +0800 Subject: [PATCH 0024/2294] [hotfix] [test] Make test-streaming-kafka010.sh more flexible for local execution --- test-infra/end-to-end-test/test_streaming_kafka010.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-infra/end-to-end-test/test_streaming_kafka010.sh b/test-infra/end-to-end-test/test_streaming_kafka010.sh index 51a570ab181d59..c7564eab53c95e 100755 --- a/test-infra/end-to-end-test/test_streaming_kafka010.sh +++ b/test-infra/end-to-end-test/test_streaming_kafka010.sh @@ -64,7 +64,7 @@ $KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication $KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-output # run the Flink job (detached mode) -$FLINK_DIR/bin/flink run -d build-target/examples/streaming/Kafka010Example.jar \ +$FLINK_DIR/bin/flink run -d $FLINK_DIR/examples/streaming/Kafka010Example.jar \ --input-topic test-input --output-topic test-output \ --prefix=PREFIX \ --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer --auto.offset.reset earliest From 2886a41728c0c13b3d01221c502a3e2a7014605d Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Mon, 26 Feb 2018 19:01:32 +0800 Subject: [PATCH 0025/2294] [hotfix] [test] Also trap INT signal in Kafka end-to-end test This allows the test to perform the cleanup procedure (as well as printing any error logs) if an interruption occurred while waiting for the test data to be written to Kafka, therefore increasing visibility of reasons to why the test was stalling. This closes #5568. --- test-infra/end-to-end-test/test_streaming_kafka010.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test-infra/end-to-end-test/test_streaming_kafka010.sh b/test-infra/end-to-end-test/test_streaming_kafka010.sh index c7564eab53c95e..a6a9a8e166b118 100755 --- a/test-infra/end-to-end-test/test_streaming_kafka010.sh +++ b/test-infra/end-to-end-test/test_streaming_kafka010.sh @@ -51,6 +51,7 @@ function kafka_cleanup { # make sure to run regular cleanup as well cleanup } +trap kafka_cleanup INT trap kafka_cleanup EXIT # zookeeper outputs the "Node does not exist" bit to stderr From 0ae7364bdd2a0ad1db1ec02dc6b1b730187f2b78 Mon Sep 17 00:00:00 2001 From: Matrix42 <934336389@qq.com> Date: Sat, 24 Feb 2018 21:52:44 +0800 Subject: [PATCH 0026/2294] [FLINK-8772] [kafka] Fix missing log parameter This closes #5574. --- .../streaming/connectors/kafka/FlinkKafkaConsumerBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index d126301c07dedf..df35de61f03c52 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -601,7 +601,7 @@ public void run() { while (running) { if (LOG.isDebugEnabled()) { - LOG.debug("Consumer subtask {} is trying to discover new partitions ..."); + LOG.debug("Consumer subtask {} is trying to discover new partitions ...", getRuntimeContext().getIndexOfThisSubtask()); } try { From 3e056b34f7be817e0d0eee612b6ae44891e33501 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 20 Feb 2018 18:02:32 +0100 Subject: [PATCH 0027/2294] [FLINK-8703][tests] Migrate tests to MiniClusterResource (batch #2) This closes #5542. --- .../apache/flink/ml/util/FlinkTestBase.scala | 25 ++-- ...alaStreamingMultipleProgramsTestBase.scala | 29 ++--- ...actEventTimeWindowCheckpointingITCase.java | 86 ++++++------- .../AbstractLocalRecoveryITCase.java | 100 +++++++++++++++ ...endEventTimeWindowCheckpointingITCase.java | 5 +- ...endEventTimeWindowCheckpointingITCase.java | 6 +- ...endEventTimeWindowCheckpointingITCase.java | 6 +- ...endEventTimeWindowCheckpointingITCase.java | 5 +- ...endEventTimeWindowCheckpointingITCase.java | 5 +- .../LocalRecoveryHeapITCase.java | 33 +++++ .../checkpointing/LocalRecoveryITCase.java | 120 ------------------ .../LocalRecoveryRocksDBFullITCase.java | 33 +++++ ...LocalRecoveryRocksDBIncrementalITCase.java | 33 +++++ ...endEventTimeWindowCheckpointingITCase.java | 5 +- ...endEventTimeWindowCheckpointingITCase.java | 5 +- .../operators/CustomDistributionITCase.java | 42 ++---- .../test/runtime/IPv6HostnamesITCase.java | 47 ++++--- 17 files changed, 311 insertions(+), 274 deletions(-) create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryHeapITCase.java delete mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBFullITCase.java create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBIncrementalITCase.java diff --git a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala index c27a2b574555b1..21651523a7bded 100644 --- a/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala +++ b/flink-libraries/flink-ml/src/test/scala/org/apache/flink/ml/util/FlinkTestBase.scala @@ -18,8 +18,9 @@ package org.apache.flink.ml.util -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster -import org.apache.flink.test.util.{TestBaseUtils, TestEnvironment} +import org.apache.flink.configuration.Configuration +import org.apache.flink.test.util.MiniClusterResource +import org.apache.flink.test.util.MiniClusterResource.MiniClusterResourceConfiguration import org.scalatest.{BeforeAndAfter, Suite} /** Mixin to start and stop a LocalFlinkMiniCluster automatically for Scala based tests. @@ -51,27 +52,21 @@ import org.scalatest.{BeforeAndAfter, Suite} trait FlinkTestBase extends BeforeAndAfter { that: Suite => - var cluster: Option[LocalFlinkMiniCluster] = None + var cluster: Option[MiniClusterResource] = None val parallelism = 4 before { - val cl = TestBaseUtils.startCluster( - 1, - parallelism, - false, - false, - true) - - val clusterEnvironment = new TestEnvironment(cl, parallelism, false) - clusterEnvironment.setAsContext() + val cl = new MiniClusterResource( + new MiniClusterResourceConfiguration(new Configuration(), 1, parallelism) + ) + + cl.before() cluster = Some(cl) } after { - cluster.foreach(c => TestBaseUtils.stopCluster(c, TestBaseUtils.DEFAULT_TIMEOUT)) - - TestEnvironment.unsetAsContext() + cluster.foreach(c => c.after()) } } diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ScalaStreamingMultipleProgramsTestBase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ScalaStreamingMultipleProgramsTestBase.scala index d9f727c2e4933b..e0c5b45a4fb526 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ScalaStreamingMultipleProgramsTestBase.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/ScalaStreamingMultipleProgramsTestBase.scala @@ -18,12 +18,10 @@ package org.apache.flink.streaming.api.scala -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster -import org.apache.flink.streaming.util.TestStreamEnvironment -import org.apache.flink.test.util.TestBaseUtils - +import org.apache.flink.configuration.Configuration +import org.apache.flink.test.util.MiniClusterResource.MiniClusterResourceConfiguration +import org.apache.flink.test.util.{MiniClusterResource, TestBaseUtils} import org.junit.{After, Before} - import org.scalatest.junit.JUnitSuiteLike trait ScalaStreamingMultipleProgramsTestBase @@ -31,28 +29,21 @@ trait ScalaStreamingMultipleProgramsTestBase with JUnitSuiteLike { val parallelism = 4 - var cluster: Option[LocalFlinkMiniCluster] = None + var cluster: Option[MiniClusterResource] = None @Before def beforeAll(): Unit = { - val cluster = Some( - TestBaseUtils.startCluster( - 1, - parallelism, - false, - false, - true - ) + val cl = new MiniClusterResource( + new MiniClusterResourceConfiguration(new Configuration(), 1, parallelism) ) - TestStreamEnvironment.setAsContext(cluster.get, parallelism) + cl.before() + + cluster = Some(cl) } @After def afterAll(): Unit = { - TestStreamEnvironment.unsetAsContext() - cluster.foreach { - TestBaseUtils.stopCluster(_, TestBaseUtils.DEFAULT_TIMEOUT) - } + cluster.foreach { c => c.after() } } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java index 557c097ffb782b..f37ba0d38053eb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java @@ -26,35 +26,32 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple4; import org.apache.flink.configuration.AkkaOptions; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.HighAvailabilityOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; import org.apache.flink.core.fs.Path; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.runtime.state.filesystem.FsStateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.checkpoint.ListCheckpointed; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.api.functions.source.RichSourceFunction; import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; -import org.apache.flink.streaming.util.TestStreamEnvironment; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.test.util.SuccessException; import org.apache.flink.util.Collector; import org.apache.flink.util.TestLogger; import org.apache.curator.test.TestingServer; import org.junit.After; -import org.junit.Before; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -65,9 +62,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum.ROCKSDB_INCREMENTAL_ZK; @@ -91,31 +85,42 @@ public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLog private static final int MAX_MEM_STATE_SIZE = 20 * 1024 * 1024; private static final int PARALLELISM = 4; - private static LocalFlinkMiniCluster cluster; + private TestingServer zkServer; - private static TestStreamEnvironment env; - - private static TestingServer zkServer; - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); + @ClassRule + public static TemporaryFolder tempFolder = new TemporaryFolder(); @Rule public TestName name = new TestName(); - private StateBackendEnum stateBackendEnum; - protected AbstractStateBackend stateBackend; + private AbstractStateBackend stateBackend; - AbstractEventTimeWindowCheckpointingITCase(StateBackendEnum stateBackendEnum) { - this.stateBackendEnum = stateBackendEnum; - } + @Rule + public final MiniClusterResource miniClusterResource = getMiniClusterResource(); enum StateBackendEnum { MEM, FILE, ROCKSDB_FULLY_ASYNC, ROCKSDB_INCREMENTAL, ROCKSDB_INCREMENTAL_ZK, MEM_ASYNC, FILE_ASYNC } - @Before - public void startTestCluster() throws Exception { + protected abstract StateBackendEnum getStateBackend(); + + protected final MiniClusterResource getMiniClusterResource() { + return new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfigurationSafe(), + 2, + PARALLELISM / 2)); + } + + private Configuration getConfigurationSafe() { + try { + return getConfiguration(); + } catch (Exception e) { + throw new AssertionError("Could not initialize test.", e); + } + } + + private Configuration getConfiguration() throws Exception { // print a message when starting a test method to avoid Travis' "Maven produced no // output for xxx seconds." messages @@ -123,6 +128,7 @@ public void startTestCluster() throws Exception { "Starting " + getClass().getCanonicalName() + "#" + name.getMethodName() + "."); // Testing HA Scenario / ZKCompletedCheckpointStore with incremental checkpoints + StateBackendEnum stateBackendEnum = getStateBackend(); if (ROCKSDB_INCREMENTAL_ZK.equals(stateBackendEnum)) { zkServer = new TestingServer(); zkServer.start(); @@ -130,23 +136,6 @@ public void startTestCluster() throws Exception { Configuration config = createClusterConfig(); - // purposefully delay in the executor to tease out races - final ScheduledExecutorService executor = Executors.newScheduledThreadPool(10); - HighAvailabilityServices haServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices( - config, - new Executor() { - @Override - public void execute(Runnable command) { - executor.schedule(command, 500, MILLISECONDS); - } - }); - - cluster = new LocalFlinkMiniCluster(config, haServices, false); - cluster.start(); - - env = new TestStreamEnvironment(cluster, PARALLELISM); - env.getConfig().setUseSnapshotCompression(true); - switch (stateBackendEnum) { case MEM: this.stateBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE, false); @@ -190,6 +179,7 @@ public void execute(Runnable command) { default: throw new IllegalStateException("No backend selected."); } + return config; } protected Configuration createClusterConfig() throws IOException { @@ -198,8 +188,6 @@ protected Configuration createClusterConfig() throws IOException { final File haDir = temporaryFolder.newFolder(); Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM / 2); config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 48L); // the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 80L << 20); // 80 MB @@ -215,11 +203,6 @@ protected Configuration createClusterConfig() throws IOException { @After public void stopTestCluster() throws IOException { - if (cluster != null) { - cluster.stop(); - cluster = null; - } - if (zkServer != null) { zkServer.stop(); zkServer = null; @@ -241,12 +224,14 @@ public void testTumblingTimeWindow() { FailingSource.reset(); try { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(PARALLELISM); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.enableCheckpointing(100); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); env.getConfig().disableSysoutLogging(); env.setStateBackend(this.stateBackend); + env.getConfig().setUseSnapshotCompression(true); env .addSource(new FailingSource(numKeys, numElementsPerKey, numElementsPerKey / 3)) @@ -310,6 +295,7 @@ public void doTestTumblingTimeWindowWithKVState(int maxParallelism) { FailingSource.reset(); try { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(PARALLELISM); env.setMaxParallelism(maxParallelism); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); @@ -317,6 +303,7 @@ public void doTestTumblingTimeWindowWithKVState(int maxParallelism) { env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); env.getConfig().disableSysoutLogging(); env.setStateBackend(this.stateBackend); + env.getConfig().setUseSnapshotCompression(true); env .addSource(new FailingSource(numKeys, numElementsPerKey, numElementsPerKey / 3)) @@ -376,6 +363,7 @@ public void testSlidingTimeWindow() { FailingSource.reset(); try { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setMaxParallelism(2 * PARALLELISM); env.setParallelism(PARALLELISM); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); @@ -438,12 +426,14 @@ public void testPreAggregatedTumblingTimeWindow() { FailingSource.reset(); try { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(PARALLELISM); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.enableCheckpointing(100); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); env.getConfig().disableSysoutLogging(); env.setStateBackend(this.stateBackend); + env.getConfig().setUseSnapshotCompression(true); env .addSource(new FailingSource(numKeys, numElementsPerKey, numElementsPerKey / 3)) @@ -507,12 +497,14 @@ public void testPreAggregatedSlidingTimeWindow() { FailingSource.reset(); try { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(PARALLELISM); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.enableCheckpointing(100); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0)); env.getConfig().disableSysoutLogging(); env.setStateBackend(this.stateBackend); + env.getConfig().setUseSnapshotCompression(true); env .addSource(new FailingSource(numKeys, numElementsPerKey, numElementsPerKey / 3)) diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java new file mode 100644 index 00000000000000..a02e902ab2bfd7 --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.checkpointing; + +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.TestLogger; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +import java.io.IOException; + +import static org.apache.flink.runtime.state.LocalRecoveryConfig.LocalRecoveryMode; +import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum; + +/** + * This test delegates to instances of {@link AbstractEventTimeWindowCheckpointingITCase} that have been reconfigured + * to use local recovery. + * + *

TODO: This class must be refactored to properly extend {@link AbstractEventTimeWindowCheckpointingITCase}. + */ +public abstract class AbstractLocalRecoveryITCase extends TestLogger { + + private final StateBackendEnum backendEnum; + private final LocalRecoveryMode recoveryMode; + + @Rule + public TestName testName = new TestName(); + + AbstractLocalRecoveryITCase(StateBackendEnum backendEnum, LocalRecoveryMode recoveryMode) { + this.backendEnum = backendEnum; + this.recoveryMode = recoveryMode; + } + + @Test + public final void executeTest() throws Exception { + AbstractEventTimeWindowCheckpointingITCase.tempFolder.create(); + AbstractEventTimeWindowCheckpointingITCase windowChkITCase = + new AbstractEventTimeWindowCheckpointingITCase() { + @Override + protected StateBackendEnum getStateBackend() { + return backendEnum; + } + + @Override + protected Configuration createClusterConfig() throws IOException { + Configuration config = super.createClusterConfig(); + + config.setString( + CheckpointingOptions.LOCAL_RECOVERY, + recoveryMode.toString()); + + return config; + } + }; + + executeTest(windowChkITCase); + } + + private void executeTest(AbstractEventTimeWindowCheckpointingITCase delegate) throws Exception { + delegate.name = testName; + try { + delegate.miniClusterResource.before(); + try { + delegate.testTumblingTimeWindow(); + delegate.miniClusterResource.after(); + } catch (Exception e) { + delegate.miniClusterResource.after(); + } + + delegate.miniClusterResource.before(); + try { + delegate.testSlidingTimeWindow(); + delegate.miniClusterResource.after(); + } catch (Exception e) { + delegate.miniClusterResource.after(); + } + } finally { + delegate.tempFolder.delete(); + } + } +} diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncFileBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncFileBackendEventTimeWindowCheckpointingITCase.java index f0db4d5ef01be1..c4b06d48089329 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncFileBackendEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncFileBackendEventTimeWindowCheckpointingITCase.java @@ -23,7 +23,8 @@ */ public class AsyncFileBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase { - public AsyncFileBackendEventTimeWindowCheckpointingITCase() { - super(StateBackendEnum.FILE_ASYNC); + @Override + protected StateBackendEnum getStateBackend() { + return StateBackendEnum.FILE_ASYNC; } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncMemBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncMemBackendEventTimeWindowCheckpointingITCase.java index 70ec757d8938bd..2cc5b01fc1fa7c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncMemBackendEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AsyncMemBackendEventTimeWindowCheckpointingITCase.java @@ -22,8 +22,8 @@ * Integration tests for asynchronous memory backend. */ public class AsyncMemBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase { - - public AsyncMemBackendEventTimeWindowCheckpointingITCase() { - super(StateBackendEnum.MEM_ASYNC); + @Override + protected StateBackendEnum getStateBackend() { + return StateBackendEnum.MEM_ASYNC; } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/FileBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/FileBackendEventTimeWindowCheckpointingITCase.java index 030c1a3902045e..eab615302cb15e 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/FileBackendEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/FileBackendEventTimeWindowCheckpointingITCase.java @@ -22,8 +22,8 @@ * Integration tests for file backend. */ public class FileBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase { - - public FileBackendEventTimeWindowCheckpointingITCase() { - super(StateBackendEnum.FILE); + @Override + protected StateBackendEnum getStateBackend() { + return StateBackendEnum.FILE; } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/HAIncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/HAIncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java index 394815f2ae61af..ed43ad60fd9290 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/HAIncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/HAIncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java @@ -23,8 +23,9 @@ */ public class HAIncrementalRocksDbBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase { - public HAIncrementalRocksDbBackendEventTimeWindowCheckpointingITCase() { - super(StateBackendEnum.ROCKSDB_INCREMENTAL_ZK); + @Override + protected StateBackendEnum getStateBackend() { + return StateBackendEnum.ROCKSDB_INCREMENTAL_ZK; } @Override diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java index dfb66cc240b8dd..1276a00f605b1c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase.java @@ -23,8 +23,9 @@ */ public class IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase { - public IncrementalRocksDbBackendEventTimeWindowCheckpointingITCase() { - super(StateBackendEnum.ROCKSDB_INCREMENTAL); + @Override + protected StateBackendEnum getStateBackend() { + return StateBackendEnum.ROCKSDB_INCREMENTAL; } @Override diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryHeapITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryHeapITCase.java new file mode 100644 index 00000000000000..2c0c2943c37749 --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryHeapITCase.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.checkpointing; + +import static org.apache.flink.runtime.state.LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED; +import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum.FILE_ASYNC; + +/** + * Tests file-based local recovery with the HeapBackend. + */ +public class LocalRecoveryHeapITCase extends AbstractLocalRecoveryITCase { + public LocalRecoveryHeapITCase() { + super( + FILE_ASYNC, + ENABLE_FILE_BASED); + } +} diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java deleted file mode 100644 index 51b3b8437eb9f4..00000000000000 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryITCase.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.test.checkpointing; - -import org.apache.flink.configuration.CheckpointingOptions; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.TestLogger; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; - -import java.io.IOException; - -import static org.apache.flink.runtime.state.LocalRecoveryConfig.LocalRecoveryMode; -import static org.apache.flink.runtime.state.LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED; -import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum; -import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum.FILE_ASYNC; -import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum.ROCKSDB_FULLY_ASYNC; -import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum.ROCKSDB_INCREMENTAL_ZK; - -/** - * This test delegates to instances of {@link AbstractEventTimeWindowCheckpointingITCase} that have been reconfigured - * to use local recovery. - */ -public class LocalRecoveryITCase extends TestLogger { - - @Rule - public TestName testName = new TestName(); - - @Test - public void testLocalRecoveryHeapBackendFileBased() throws Exception { - executeTest( - FILE_ASYNC, - ENABLE_FILE_BASED); - } - - @Test - public void testLocalRecoveryRocksIncrementalFileBased() throws Exception { - executeTest( - ROCKSDB_INCREMENTAL_ZK, - ENABLE_FILE_BASED); - } - - @Test - public void testLocalRecoveryRocksFullFileBased() throws Exception { - executeTest( - ROCKSDB_FULLY_ASYNC, - ENABLE_FILE_BASED); - } - - private void executeTest( - StateBackendEnum backendEnum, - LocalRecoveryMode recoveryMode) throws Exception { - - AbstractEventTimeWindowCheckpointingITCase windowChkITCase = - new AbstractEventTimeWindowCheckpointingITCaseWithLocalRecovery( - backendEnum, - recoveryMode); - - executeTest(windowChkITCase); - } - - private void executeTest(AbstractEventTimeWindowCheckpointingITCase delegate) throws Exception { - delegate.name = testName; - delegate.tempFolder.create(); - try { - delegate.startTestCluster(); - delegate.testTumblingTimeWindow(); - delegate.stopTestCluster(); - - delegate.startTestCluster(); - delegate.testSlidingTimeWindow(); - delegate.stopTestCluster(); - } finally { - delegate.tempFolder.delete(); - } - } - - private static class AbstractEventTimeWindowCheckpointingITCaseWithLocalRecovery - extends AbstractEventTimeWindowCheckpointingITCase { - - private final LocalRecoveryMode recoveryMode; - - AbstractEventTimeWindowCheckpointingITCaseWithLocalRecovery( - StateBackendEnum stateBackendEnum, - LocalRecoveryMode recoveryMode) { - - super(stateBackendEnum); - this.recoveryMode = recoveryMode; - } - - @Override - protected Configuration createClusterConfig() throws IOException { - Configuration config = super.createClusterConfig(); - - config.setString( - CheckpointingOptions.LOCAL_RECOVERY, - recoveryMode.toString()); - - return config; - } - } -} diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBFullITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBFullITCase.java new file mode 100644 index 00000000000000..16bbbfc19a6818 --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBFullITCase.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.checkpointing; + +import static org.apache.flink.runtime.state.LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED; +import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum.ROCKSDB_FULLY_ASYNC; + +/** + * Tests file-based local recovery with the RocksDB state-backend. + */ +public class LocalRecoveryRocksDBFullITCase extends AbstractLocalRecoveryITCase { + public LocalRecoveryRocksDBFullITCase() { + super( + ROCKSDB_FULLY_ASYNC, + ENABLE_FILE_BASED); + } +} diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBIncrementalITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBIncrementalITCase.java new file mode 100644 index 00000000000000..fa8e13971d8884 --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/LocalRecoveryRocksDBIncrementalITCase.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.checkpointing; + +import static org.apache.flink.runtime.state.LocalRecoveryConfig.LocalRecoveryMode.ENABLE_FILE_BASED; +import static org.apache.flink.test.checkpointing.AbstractEventTimeWindowCheckpointingITCase.StateBackendEnum.ROCKSDB_INCREMENTAL_ZK; + +/** + * Tests file-based local recovery with the RocksDB state-backend and incremental checkpointing enabled. + */ +public class LocalRecoveryRocksDBIncrementalITCase extends AbstractLocalRecoveryITCase { + public LocalRecoveryRocksDBIncrementalITCase() { + super( + ROCKSDB_INCREMENTAL_ZK, + ENABLE_FILE_BASED); + } +} diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/MemBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/MemBackendEventTimeWindowCheckpointingITCase.java index 54a29ed63b6c7f..e153b4b971c7df 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/MemBackendEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/MemBackendEventTimeWindowCheckpointingITCase.java @@ -23,7 +23,8 @@ */ public class MemBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase { - public MemBackendEventTimeWindowCheckpointingITCase() { - super(StateBackendEnum.MEM); + @Override + protected StateBackendEnum getStateBackend() { + return StateBackendEnum.MEM; } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RocksDbBackendEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RocksDbBackendEventTimeWindowCheckpointingITCase.java index 3873aff8fbe145..e6d5b9e103ceeb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RocksDbBackendEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RocksDbBackendEventTimeWindowCheckpointingITCase.java @@ -23,8 +23,9 @@ */ public class RocksDbBackendEventTimeWindowCheckpointingITCase extends AbstractEventTimeWindowCheckpointingITCase { - public RocksDbBackendEventTimeWindowCheckpointingITCase() { - super(StateBackendEnum.ROCKSDB_FULLY_ASYNC); + @Override + protected StateBackendEnum getStateBackend() { + return StateBackendEnum.ROCKSDB_FULLY_ASYNC; } @Override diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/CustomDistributionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/CustomDistributionITCase.java index 24524756cb9ec9..74b8cf74bc9263 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/operators/CustomDistributionITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/operators/CustomDistributionITCase.java @@ -27,19 +27,15 @@ import org.apache.flink.api.java.io.DiscardingOutputFormat; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.api.java.utils.DataSetUtils; +import org.apache.flink.configuration.Configuration; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.test.operators.util.CollectionDataSets; -import org.apache.flink.test.util.TestBaseUtils; -import org.apache.flink.test.util.TestEnvironment; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.util.Collector; import org.apache.flink.util.TestLogger; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; import java.io.IOException; @@ -52,32 +48,12 @@ @SuppressWarnings("serial") public class CustomDistributionITCase extends TestLogger { - // ------------------------------------------------------------------------ - // The mini cluster that is shared across tests - // ------------------------------------------------------------------------ - - private static LocalFlinkMiniCluster cluster; - - @BeforeClass - public static void setup() throws Exception { - cluster = TestBaseUtils.startCluster(1, 8, false, false, true); - } - - @AfterClass - public static void teardown() throws Exception { - TestBaseUtils.stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT); - } - - @Before - public void prepare() { - TestEnvironment clusterEnv = new TestEnvironment(cluster, 1, false); - clusterEnv.setAsContext(); - } - - @After - public void cleanup() { - TestEnvironment.unsetAsContext(); - } + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + new Configuration(), + 1, + 8)); // ------------------------------------------------------------------------ diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java index f24d21e43d2233..4a7b3459049eb8 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java @@ -27,14 +27,16 @@ import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.test.testdata.WordCountData; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.test.util.TestBaseUtils; import org.apache.flink.util.Collector; import org.apache.flink.util.NetUtils; import org.apache.flink.util.TestLogger; import akka.actor.ActorSystem; +import org.junit.AssumptionViolatedException; +import org.junit.Rule; import org.junit.Test; import java.io.IOException; @@ -56,31 +58,33 @@ @SuppressWarnings("serial") public class IPv6HostnamesITCase extends TestLogger { - @Test - public void testClusterWithIPv6host() { + @Rule + public final MiniClusterResource miniClusterResource = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + 2, + 2)); + private Configuration getConfiguration() { final Inet6Address ipv6address = getLocalIPv6Address(); if (ipv6address == null) { - System.err.println("--- Cannot find a non-loopback local IPv6 address that Akka/Netty can bind to; skipping IPv6HostnamesITCase"); - return; + throw new AssumptionViolatedException("--- Cannot find a non-loopback local IPv6 address that Akka/Netty can bind to; skipping IPv6HostnamesITCase"); } + final String addressString = ipv6address.getHostAddress(); + log.info("Test will use IPv6 address " + addressString + " for connection tests"); + + Configuration config = new Configuration(); + config.setString(JobManagerOptions.ADDRESS, addressString); + config.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, addressString); + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L); + return config; + } - LocalFlinkMiniCluster flink = null; + @Test + public void testClusterWithIPv6host() { try { - final String addressString = ipv6address.getHostAddress(); - log.info("Test will use IPv6 address " + addressString + " for connection tests"); - Configuration conf = new Configuration(); - conf.setString(JobManagerOptions.ADDRESS, addressString); - conf.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, addressString); - conf.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2); - conf.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2); - conf.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L); - - flink = new LocalFlinkMiniCluster(conf, false); - flink.start(); - - ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(addressString, flink.getLeaderRPCPort()); + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); env.getConfig().disableSysoutLogging(); @@ -108,11 +112,6 @@ public void flatMap(String value, Collector> out) throws e.printStackTrace(); fail(e.getMessage()); } - finally { - if (flink != null) { - flink.stop(); - } - } } private Inet6Address getLocalIPv6Address() { From f3bfd22b41acd4c16f71d6d69f7983abc63a1e71 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 20 Feb 2018 13:40:43 +0100 Subject: [PATCH 0028/2294] [FLINK-8596][CLI] Also catch NoClassDefFoundErrors This closes #5543. --- .../apache/flink/client/cli/CliFrontend.java | 2 +- test-infra/end-to-end-test/common.sh | 3 ++ .../end-to-end-test/test_hadoop_free.sh | 46 +++++++++++++++++++ tools/travis_mvn_watchdog.sh | 10 +++- 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100755 test-infra/end-to-end-test/test_hadoop_free.sh diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java index 92d2ccb8c3571e..06131dc6836b01 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java @@ -1158,7 +1158,7 @@ public static List> loadCustomCommandLines(Configuration co configurationDirectory, "y", "yarn")); - } catch (Exception e) { + } catch (NoClassDefFoundError | Exception e) { LOG.warn("Could not load CLI class {}.", flinkYarnSessionCLI, e); } diff --git a/test-infra/end-to-end-test/common.sh b/test-infra/end-to-end-test/common.sh index 67b7529ba6e59d..7492f365afde16 100644 --- a/test-infra/end-to-end-test/common.sh +++ b/test-infra/end-to-end-test/common.sh @@ -74,6 +74,7 @@ function stop_cluster { | grep -v '^INFO:.*AWSErrorCode=\[400 Bad Request\].*ServiceEndpoint=\[https://.*\.s3\.amazonaws\.com\].*RequestType=\[HeadBucketRequest\]' \ | grep -v "RejectedExecutionException" \ | grep -v "An exception was thrown by an exception handler" \ + | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/exceptions/YarnException" \ | grep -iq "error"; then echo "Found error in log files:" cat $FLINK_DIR/log/* @@ -90,6 +91,8 @@ function stop_cluster { | grep -v '^INFO:.*AWSErrorCode=\[400 Bad Request\].*ServiceEndpoint=\[https://.*\.s3\.amazonaws\.com\].*RequestType=\[HeadBucketRequest\]' \ | grep -v "RejectedExecutionException" \ | grep -v "An exception was thrown by an exception handler" \ + | grep -v "Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.exceptions.YarnException" \ + | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/exceptions/YarnException" \ | grep -iq "exception"; then echo "Found exception in log files:" cat $FLINK_DIR/log/* diff --git a/test-infra/end-to-end-test/test_hadoop_free.sh b/test-infra/end-to-end-test/test_hadoop_free.sh new file mode 100755 index 00000000000000..311357e939972d --- /dev/null +++ b/test-infra/end-to-end-test/test_hadoop_free.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh + +# move hadoop jar to /opt so it's not on the classpath +mv $FLINK_DIR/lib/flink-shaded-hadoop* $FLINK_DIR/opt +EXIT_CODE=$? +if [ $EXIT_CODE != 0 ]; then + echo "==============================================================================" + echo "Could not move hadoop jar out of /lib, aborting test." + echo "==============================================================================" +else + start_cluster + + $FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $TEST_INFRA_DIR/test-data/words --output $TEST_DATA_DIR/out/wc_out + check_result_hash "WordCount" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5" + EXIT_CODE=$? + + # move hadoop jar to /lib again to not have side-effects on subsequent tests + mv $FLINK_DIR/opt/flink-shaded-hadoop* $FLINK_DIR/lib + if [ $? != 0 ]; then + echo "==============================================================================" + echo "Could not move hadoop jar back to /lib, aborting tests." + echo "==============================================================================" + EXIT_CODE=1 + fi +fi + +exit $EXIT_CODE diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh index 0ef74dbb979b13..4b1c2e31ad5da5 100755 --- a/tools/travis_mvn_watchdog.sh +++ b/tools/travis_mvn_watchdog.sh @@ -616,7 +616,15 @@ case $TEST in EXIT_CODE=$? fi - if [ $EXIT_CODE == 0]; then + if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Hadoop-free Wordcount end-to-end test\n" + printf "==============================================================================\n" + FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_hadoop_free.sh + EXIT_CODE=$? + fi + + if [ $EXIT_CODE == 0 ]; then printf "\n==============================================================================\n" printf "Running Streaming Python Wordcount end-to-end test\n" printf "==============================================================================\n" From 50aea889bd88d5ce4a1569569176705f9973a08c Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 21 Feb 2018 14:55:22 +0100 Subject: [PATCH 0029/2294] [FLINK-8645][configuration] Split classloader.parent-first-patterns into "base" and "append This closes #5544. --- docs/ops/config.md | 12 +++-- .../flink/configuration/CoreOptions.java | 37 +++++++++++-- .../flink/configuration/CoreOptionsTest.java | 54 +++++++++++++++++++ .../ParentFirstPatternsTest.java | 2 +- .../avro/AvroKryoClassloadingTest.java | 2 +- .../jobmaster/JobManagerSharedServices.java | 4 +- .../TaskManagerConfiguration.java | 4 +- .../flink/runtime/jobmanager/JobManager.scala | 5 +- 8 files changed, 101 insertions(+), 19 deletions(-) create mode 100644 flink-core/src/test/java/org/apache/flink/configuration/CoreOptionsTest.java diff --git a/docs/ops/config.md b/docs/ops/config.md index efd3ee3af90370..ce5bc9b981beec 100644 --- a/docs/ops/config.md +++ b/docs/ops/config.md @@ -75,12 +75,16 @@ without explicit scheme definition, such as `/user/USERNAME/in.txt`, is going to - `classloader.resolve-order`: Whether Flink should use a child-first `ClassLoader` when loading user-code classes or a parent-first `ClassLoader`. Can be one of `parent-first` or `child-first`. (default: `child-first`) -- `classloader.parent-first-patterns`: A (semicolon-separated) list of patterns that specifies which +- `classloader.parent-first-patterns.default`: A (semicolon-separated) list of patterns that specifies which classes should always be resolved through the parent `ClassLoader` first. A pattern is a simple prefix that is checked against the fully qualified class name. By default, this is set to -`java.;org.apache.flink.;javax.annotation;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback`. -If you want to change this setting you have to make sure to also include the default patterns in -your list of patterns if you want to keep that default behaviour. +`"java.;scala.;org.apache.flink.;com.esotericsoftware.kryo;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback"`. +To extend this list beyond the default it is recommended to configure `classloader.parent-first-patterns.additional` instead of modifying this setting directly. + +- `classloader.parent-first-patterns.additional`: A (semicolon-separated) list of patterns that specifies which +classes should always be resolved through the parent `ClassLoader` first. A pattern is a simple +prefix that is checked against the fully qualified class name. +This list is appended to `classloader.parent-first-patterns.default`. ## Advanced Options diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java index 30c0cd64e30320..ccce0aba8db2dd 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java @@ -45,7 +45,7 @@ public class CoreOptions { * which means that user code jars can include and load different dependencies than * Flink uses (transitively). * - *

Exceptions to the rules are defined via {@link #ALWAYS_PARENT_FIRST_LOADER}. + *

Exceptions to the rules are defined via {@link #ALWAYS_PARENT_FIRST_LOADER_PATTERNS}. */ public static final ConfigOption CLASSLOADER_RESOLVE_ORDER = ConfigOptions .key("classloader.resolve-order") @@ -85,12 +85,41 @@ public class CoreOptions { * log bindings. * */ - public static final ConfigOption ALWAYS_PARENT_FIRST_LOADER = ConfigOptions - .key("classloader.parent-first-patterns") + public static final ConfigOption ALWAYS_PARENT_FIRST_LOADER_PATTERNS = ConfigOptions + .key("classloader.parent-first-patterns.default") .defaultValue("java.;scala.;org.apache.flink.;com.esotericsoftware.kryo;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback") + .withDeprecatedKeys("classloader.parent-first-patterns") .withDescription("A (semicolon-separated) list of patterns that specifies which classes should always be" + " resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against" + - " the fully qualified class name."); + " the fully qualified class name. This setting should generally not be modified. To add another pattern we" + + " recommend to use \"classloader.parent-first-patterns.additional\" instead."); + + public static final ConfigOption ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL = ConfigOptions + .key("classloader.parent-first-patterns.additional") + .defaultValue("") + .withDescription("A (semicolon-separated) list of patterns that specifies which classes should always be" + + " resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against" + + " the fully qualified class name. These patterns are appended to \"" + ALWAYS_PARENT_FIRST_LOADER_PATTERNS.key() + "\"."); + + public static String[] getParentFirstLoaderPatterns(Configuration config) { + String base = config.getString(ALWAYS_PARENT_FIRST_LOADER_PATTERNS); + String append = config.getString(ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL); + + String[] basePatterns = base.isEmpty() + ? new String[0] + : base.split(";"); + + if (append.isEmpty()) { + return basePatterns; + } else { + String[] appendPatterns = append.split(";"); + + String[] joinedPatterns = new String[basePatterns.length + appendPatterns.length]; + System.arraycopy(basePatterns, 0, joinedPatterns, 0, basePatterns.length); + System.arraycopy(appendPatterns, 0, joinedPatterns, basePatterns.length, appendPatterns.length); + return joinedPatterns; + } + } // ------------------------------------------------------------------------ // process parameters diff --git a/flink-core/src/test/java/org/apache/flink/configuration/CoreOptionsTest.java b/flink-core/src/test/java/org/apache/flink/configuration/CoreOptionsTest.java new file mode 100644 index 00000000000000..0bc59104e57343 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/configuration/CoreOptionsTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.configuration; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for {@link CoreOptions}. + */ +public class CoreOptionsTest { + @Test + public void testGetParentFirstLoaderPatterns() { + Configuration config = new Configuration(); + + Assert.assertArrayEquals( + CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"), + CoreOptions.getParentFirstLoaderPatterns(config)); + + config.setString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS, "hello;world"); + + Assert.assertArrayEquals( + "hello;world".split(";"), + CoreOptions.getParentFirstLoaderPatterns(config)); + + config.setString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL, "how;are;you"); + + Assert.assertArrayEquals( + "hello;world;how;are;you".split(";"), + CoreOptions.getParentFirstLoaderPatterns(config)); + + config.setString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS, ""); + + Assert.assertArrayEquals( + "how;are;you".split(";"), + CoreOptions.getParentFirstLoaderPatterns(config)); + } +} diff --git a/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java b/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java index 784d0998768b4e..ca4b511f456367 100644 --- a/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java +++ b/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java @@ -34,7 +34,7 @@ public class ParentFirstPatternsTest extends TestLogger { private static final HashSet PARENT_FIRST_PACKAGES = new HashSet<>( - Arrays.asList(CoreOptions.ALWAYS_PARENT_FIRST_LOADER.defaultValue().split(";"))); + Arrays.asList(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"))); /** * All java and Flink classes must be loaded parent first. diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroKryoClassloadingTest.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroKryoClassloadingTest.java index 6eaca15240efd2..8f0916cfd66985 100644 --- a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroKryoClassloadingTest.java +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroKryoClassloadingTest.java @@ -73,7 +73,7 @@ public void testKryoInChildClasspath() throws Exception { final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst( new URL[] { avroLocation, kryoLocation }, parentClassLoader, - CoreOptions.ALWAYS_PARENT_FIRST_LOADER.defaultValue().split(";")); + CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";")); final Class userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader); assertNotEquals(avroClass, userLoadedAvroClass); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java index 34b338e8973758..c1e910cbeb84b0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java @@ -131,9 +131,7 @@ public static JobManagerSharedServices fromConfiguration( final String classLoaderResolveOrder = config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER); - final String alwaysParentFirstLoaderString = - config.getString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER); - final String[] alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderString.split(";"); + final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config); final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager( diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java index aebefd65b0520c..cb6fe51b0b9002 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java @@ -241,9 +241,7 @@ public static TaskManagerConfiguration fromConfiguration(Configuration configura final String classLoaderResolveOrder = configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER); - final String alwaysParentFirstLoaderString = - configuration.getString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER); - final String[] alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderString.split(";"); + final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration); final String taskManagerLogPath = configuration.getString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, System.getProperty("log.file")); final String taskManagerStdoutPath; diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala index 1dfaa5d344bf7a..b9529de226bcdc 100644 --- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala +++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala @@ -2430,9 +2430,8 @@ object JobManager { val timeout: FiniteDuration = AkkaUtils.getTimeout(configuration) val classLoaderResolveOrder = configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER) - val alwaysParentFirstLoaderString = - configuration.getString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER) - val alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderString.split(';') + + val alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration) val restartStrategy = RestartStrategyFactory.createRestartStrategyFactory(configuration) From a2d1d084b90f0f47b91aa372525f01a382c892e7 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 14 Feb 2018 12:45:17 +0100 Subject: [PATCH 0030/2294] [FLINK-8593][metrics] Update latency metric docs This closes #5484. --- docs/monitoring/metrics.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md index 62adeb184a543f..1e70dd725029f5 100644 --- a/docs/monitoring/metrics.md +++ b/docs/monitoring/metrics.md @@ -1189,6 +1189,12 @@ Thus, in order to infer the metric identifier: + + Job (only available on TaskManager) + <source_id>.<source_subtask_index>.<operator_id>.<operator_subtask_index>.latency + The latency distributions from a given source subtask to an operator subtask (in milliseconds). + Histogram + Task numBytesInLocal @@ -1247,7 +1253,7 @@ Thus, in order to infer the metric identifier: Counter - Operator + Operator currentInputWatermark The last watermark this operator has received (in milliseconds). @@ -1278,11 +1284,6 @@ Thus, in order to infer the metric identifier: Gauge - - latency - The latency distributions from all incoming sources (in milliseconds). - Histogram - numSplitsProcessed The total number of InputSplits this data source has processed (if the operator is a data source). From 915213c7afaf3f9d04c240f43d88710280d844e3 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Thu, 22 Feb 2018 17:24:33 +0100 Subject: [PATCH 0031/2294] [FLINK-8543] Don't call super.close() in AvroKeyValueSinkWriter The call to keyValueWriter.close() in AvroKeyValueSinkWriter.close() will eventually call flush() on the wrapped stream which fails if we close it before(). Now we call flush ourselves before closing the KeyValyeWriter, which internally closes the wrapped stream eventually. --- .../connectors/fs/AvroKeyValueSinkWriter.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java index e9316333519b16..6b2f7d625a1105 100644 --- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java +++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/AvroKeyValueSinkWriter.java @@ -150,17 +150,29 @@ private CodecFactory getCompressionCodec(Map conf) { public void open(FileSystem fs, Path path) throws IOException { super.open(fs, path); - CodecFactory compressionCodec = getCompressionCodec(properties); - Schema keySchema = Schema.parse(properties.get(CONF_OUTPUT_KEY_SCHEMA)); - Schema valueSchema = Schema.parse(properties.get(CONF_OUTPUT_VALUE_SCHEMA)); - keyValueWriter = new AvroKeyValueWriter(keySchema, valueSchema, compressionCodec, getStream()); + try { + CodecFactory compressionCodec = getCompressionCodec(properties); + Schema keySchema = Schema.parse(properties.get(CONF_OUTPUT_KEY_SCHEMA)); + Schema valueSchema = Schema.parse(properties.get(CONF_OUTPUT_VALUE_SCHEMA)); + keyValueWriter = new AvroKeyValueWriter( + keySchema, + valueSchema, + compressionCodec, + getStream()); + } finally { + if (keyValueWriter == null) { + close(); + } + } } @Override public void close() throws IOException { - super.close(); //the order is important since super.close flushes inside if (keyValueWriter != null) { keyValueWriter.close(); + } else { + // need to make sure we close this if we never created the Key/Value Writer. + super.close(); } } From 4bf76ae69e3f22e25c2dad3e802be094554b5d43 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 20 Feb 2018 18:04:12 +0100 Subject: [PATCH 0032/2294] [FLINK-8733][network] fix SpillableSubpartition#spillFinishedBufferConsumers() not counting spilled bytes This closes #5549. --- .../partition/SpillableSubpartition.java | 5 ++++- .../partition/SpillableSubpartitionTest.java | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java index 8758b34ef552c4..6ac493e7e2752f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.io.network.partition; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.io.disk.iomanager.BufferFileWriter; import org.apache.flink.runtime.io.disk.iomanager.IOManager; @@ -240,13 +241,15 @@ public int releaseMemory() throws IOException { return 0; } - private long spillFinishedBufferConsumers() throws IOException { + @VisibleForTesting + protected long spillFinishedBufferConsumers() throws IOException { long spilledBytes = 0; while (!buffers.isEmpty()) { BufferConsumer bufferConsumer = buffers.peek(); Buffer buffer = bufferConsumer.build(); updateStatistics(buffer); + spilledBytes += buffer.getSize(); spillWriter.writeBlock(buffer); if (bufferConsumer.isFinished()) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java index 9dc7bed21ec2f0..65d98e69de25fe 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java @@ -26,6 +26,7 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsyncWithNoOpBufferFileWriter; import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent; import org.apache.flink.runtime.io.network.api.serialization.EventSerializer; +import org.apache.flink.runtime.io.network.buffer.BufferBuilder; import org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils; import org.apache.flink.runtime.io.network.buffer.BufferConsumer; import org.apache.flink.runtime.io.network.buffer.BufferProvider; @@ -40,12 +41,14 @@ import org.mockito.stubbing.Answer; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createBufferBuilder; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -700,6 +703,24 @@ private void testCleanupReleasedPartition(boolean spilled, boolean createView) t // assertEquals((createView ? 4 : 0) + 2 * BUFFER_DATA_SIZE, partition.getTotalNumberOfBytes()); } + /** + * Tests {@link SpillableSubpartition#spillFinishedBufferConsumers()} spilled bytes counting. + */ + @Test + public void testSpillFinishedBufferConsumers() throws Exception { + SpillableSubpartition partition = createSubpartition(); + BufferBuilder bufferBuilder = createBufferBuilder(BUFFER_DATA_SIZE); + + try (BufferConsumer buffer = bufferBuilder.createBufferConsumer()) { + partition.add(buffer); + assertEquals(0, partition.releaseMemory()); + // finally fill the buffer with some bytes + bufferBuilder.appendAndCommit(ByteBuffer.allocate(BUFFER_DATA_SIZE)); + bufferBuilder.finish(); // so that this buffer can be removed from the queue + assertEquals(BUFFER_DATA_SIZE, partition.spillFinishedBufferConsumers()); + } + } + /** * An {@link IOManagerAsync} that creates closed {@link BufferFileWriter} instances in its * {@link #createBufferFileWriter(FileIOChannel.ID)} method. From f9daf9cc4243a80b38a1f81bf2b9b37565fe2d61 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 20 Feb 2018 18:05:54 +0100 Subject: [PATCH 0033/2294] [FLINK-8734][network] fix partition bytes counting and re-enable in tests This closes #5550. --- .../partition/SpillableSubpartitionView.java | 7 +++- .../partition/PipelinedSubpartitionTest.java | 16 ++++---- .../partition/SpillableSubpartitionTest.java | 41 +++++++++++-------- .../partition/SubpartitionTestBase.java | 3 ++ 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java index 789b3d0f0afdae..b821dcf6afe8de 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java @@ -18,15 +18,17 @@ package org.apache.flink.runtime.io.network.partition; -import org.apache.flink.runtime.io.network.buffer.BufferConsumer; -import org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog; import org.apache.flink.runtime.io.disk.iomanager.BufferFileWriter; import org.apache.flink.runtime.io.disk.iomanager.IOManager; import org.apache.flink.runtime.io.network.buffer.Buffer; +import org.apache.flink.runtime.io.network.buffer.BufferConsumer; +import org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; + import java.io.IOException; import java.util.ArrayDeque; import java.util.concurrent.atomic.AtomicBoolean; @@ -115,6 +117,7 @@ int releaseMemory() throws IOException { checkState(bufferConsumer.isFinished(), "BufferConsumer must be finished before " + "spilling. Otherwise we would not be able to simply remove it from the queue. This should " + "be guaranteed by creating ResultSubpartitionView only once Subpartition isFinished."); + parent.updateStatistics(buffer); spilledBytes += buffer.getSize(); spillWriter.writeBlock(buffer); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java index 2ca01c8f695955..528f0e296d341d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java @@ -207,8 +207,7 @@ public void testBasicPipelinedProduceConsumeLogic() throws Exception { assertEquals(1, subpartition.getTotalNumberOfBuffers()); assertEquals(1, subpartition.getBuffersInBacklog()); - // TODO: re-enable? -// assertEquals(BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); + assertEquals(0, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer // ...should have resulted in a notification verify(listener, times(1)).notifyDataAvailable(); @@ -218,6 +217,7 @@ public void testBasicPipelinedProduceConsumeLogic() throws Exception { BufferAndBacklog read = view.getNextBuffer(); assertNotNull(read); assertTrue(read.buffer().isBuffer()); + assertEquals(BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer assertEquals(0, subpartition.getBuffersInBacklog()); assertEquals(subpartition.getBuffersInBacklog(), read.buffersInBacklog()); assertFalse(read.nextBufferIsEvent()); @@ -231,14 +231,14 @@ public void testBasicPipelinedProduceConsumeLogic() throws Exception { assertEquals(2, subpartition.getTotalNumberOfBuffers()); assertEquals(1, subpartition.getBuffersInBacklog()); - // TODO: re-enable? -// assertEquals(2 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); + assertEquals(BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer verify(listener, times(2)).notifyDataAvailable(); assertFalse(view.nextBufferIsEvent()); read = view.getNextBuffer(); assertNotNull(read); assertTrue(read.buffer().isBuffer()); + assertEquals(2 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer assertEquals(0, subpartition.getBuffersInBacklog()); assertEquals(subpartition.getBuffersInBacklog(), read.buffersInBacklog()); assertFalse(read.nextBufferIsEvent()); @@ -258,14 +258,14 @@ public void testBasicPipelinedProduceConsumeLogic() throws Exception { assertEquals(5, subpartition.getTotalNumberOfBuffers()); assertEquals(2, subpartition.getBuffersInBacklog()); // two buffers (events don't count) - // TODO: re-enable? -// assertEquals(5 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); + assertEquals(2 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer verify(listener, times(4)).notifyDataAvailable(); assertFalse(view.nextBufferIsEvent()); // the first buffer read = view.getNextBuffer(); assertNotNull(read); assertTrue(read.buffer().isBuffer()); + assertEquals(3 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer assertEquals(1, subpartition.getBuffersInBacklog()); assertEquals(subpartition.getBuffersInBacklog(), read.buffersInBacklog()); assertTrue(read.nextBufferIsEvent()); @@ -274,6 +274,7 @@ public void testBasicPipelinedProduceConsumeLogic() throws Exception { read = view.getNextBuffer(); assertNotNull(read); assertFalse(read.buffer().isBuffer()); + assertEquals(4 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer assertEquals(1, subpartition.getBuffersInBacklog()); assertEquals(subpartition.getBuffersInBacklog(), read.buffersInBacklog()); assertFalse(read.nextBufferIsEvent()); @@ -282,6 +283,7 @@ public void testBasicPipelinedProduceConsumeLogic() throws Exception { read = view.getNextBuffer(); assertNotNull(read); assertTrue(read.buffer().isBuffer()); + assertEquals(5 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer assertEquals(0, subpartition.getBuffersInBacklog()); assertEquals(subpartition.getBuffersInBacklog(), read.buffersInBacklog()); assertFalse(read.nextBufferIsEvent()); @@ -473,6 +475,6 @@ private void testCleanupReleasedPartition(boolean createView) throws Exception { Assert.fail("buffer 2 not recycled"); } assertEquals(2, partition.getTotalNumberOfBuffers()); - //assertEquals(2 * 4096, partition.getTotalNumberOfBytes()); + assertEquals(0, partition.getTotalNumberOfBytes()); // buffer data is never consumed } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java index 65d98e69de25fe..43bcd31b3f9964 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java @@ -199,8 +199,7 @@ public void testConsumeSpilledPartition() throws Exception { assertEquals(4, partition.getTotalNumberOfBuffers()); assertEquals(3, partition.getBuffersInBacklog()); - //TODO: re-enable this? -// assertEquals(BUFFER_DATA_SIZE * 4, partition.getTotalNumberOfBytes()); + assertEquals(0, partition.getTotalNumberOfBytes()); // only updated when getting/releasing the buffers assertFalse(bufferConsumer.isRecycled()); assertEquals(4, partition.releaseMemory()); @@ -305,8 +304,7 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception assertEquals(5, partition.getTotalNumberOfBuffers()); assertEquals(3, partition.getBuffersInBacklog()); - //TODO: re-enable this? -// assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); + assertEquals(0, partition.getTotalNumberOfBytes()); // only updated when getting/spilling the buffers AwaitableBufferAvailablityListener listener = new AwaitableBufferAvailablityListener(); SpillableSubpartitionView reader = (SpillableSubpartitionView) partition.createReadView(listener); @@ -319,6 +317,7 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception BufferAndBacklog read = reader.getNextBuffer(); // first buffer (non-spilled) assertNotNull(read); assertTrue(read.buffer().isBuffer()); + assertEquals(BUFFER_DATA_SIZE, partition.getTotalNumberOfBytes()); // only updated when getting/spilling the buffers assertEquals(2, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); read.buffer().recycleBuffer(); @@ -332,8 +331,8 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception // still same statistics: assertEquals(5, partition.getTotalNumberOfBuffers()); assertEquals(2, partition.getBuffersInBacklog()); - //TODO: re-enable this? -// assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); + // only updated when getting/spilling the buffers but without the nextBuffer (kept in memory) + assertEquals(BUFFER_DATA_SIZE * 3 + 4, partition.getTotalNumberOfBytes()); listener.awaitNotifications(3, 30_000); assertEquals(3, listener.getNumNotifications()); @@ -342,6 +341,7 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception read = reader.getNextBuffer(); assertNotNull(read); assertTrue(read.buffer().isBuffer()); + assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // finally integrates the nextBuffer statistics assertEquals(1, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); read.buffer().recycleBuffer(); @@ -353,6 +353,7 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception read = reader.getNextBuffer(); assertNotNull(read); assertFalse(read.buffer().isBuffer()); + assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling assertEquals(1, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); read.buffer().recycleBuffer(); @@ -362,6 +363,7 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception read = reader.getNextBuffer(); assertNotNull(read); assertTrue(read.buffer().isBuffer()); + assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling assertEquals(0, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); assertFalse(read.buffer().isRecycled()); @@ -373,6 +375,7 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception assertTrue(reader.nextBufferIsEvent()); read = reader.getNextBuffer(); assertNotNull(read); + assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling assertEquals(0, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); assertEquals(EndOfPartitionEvent.class, @@ -421,8 +424,8 @@ private void testAddOnFinishedPartition(boolean spilled) throws Exception { partition.finish(); // finish adds an EndOfPartitionEvent assertEquals(1, partition.getTotalNumberOfBuffers()); - //TODO: re-enable this? -// assertEquals(4, partition.getTotalNumberOfBytes()); + // if not spilled, statistics are only updated when consuming the buffers + assertEquals(spilled ? 4 : 0, partition.getTotalNumberOfBytes()); BufferConsumer buffer = createFilledBufferConsumer(BUFFER_DATA_SIZE, BUFFER_DATA_SIZE); try { @@ -435,8 +438,8 @@ private void testAddOnFinishedPartition(boolean spilled) throws Exception { } // still same statistics assertEquals(1, partition.getTotalNumberOfBuffers()); - //TODO: re-enable this? -// assertEquals(4, partition.getTotalNumberOfBytes()); + // if not spilled, statistics are only updated when consuming the buffers + assertEquals(spilled ? 4 : 0, partition.getTotalNumberOfBytes()); } @Test @@ -546,13 +549,13 @@ private void testReleaseOnSpillablePartitionWithSlowWriter(boolean createView) t assertFalse("buffer1 should not be recycled (still in the queue)", buffer1.isRecycled()); assertFalse("buffer2 should not be recycled (still in the queue)", buffer2.isRecycled()); assertEquals(2, partition.getTotalNumberOfBuffers()); - //TODO: re-enable this? -// assertEquals(BUFFER_DATA_SIZE * 2, partition.getTotalNumberOfBytes()); + assertEquals(0, partition.getTotalNumberOfBytes()); // only updated when buffers are consumed or spilled if (createView) { // Create a read view partition.finish(); partition.createReadView(new NoOpBufferAvailablityListener()); + assertEquals(0, partition.getTotalNumberOfBytes()); // only updated when buffers are consumed or spilled } // one instance of the buffers is placed in the view's nextBuffer and not released @@ -571,8 +574,8 @@ private void testReleaseOnSpillablePartitionWithSlowWriter(boolean createView) t } // note: a view requires a finished partition which has an additional EndOfPartitionEvent assertEquals(2 + (createView ? 1 : 0), partition.getTotalNumberOfBuffers()); - //TODO: re-enable this? -// assertEquals(BUFFER_DATA_SIZE * 2 + (createView ? 4 : 0), partition.getTotalNumberOfBytes()); + // with a view, one buffer remains in nextBuffer and is not counted yet + assertEquals(BUFFER_DATA_SIZE + (createView ? 4 : BUFFER_DATA_SIZE), partition.getTotalNumberOfBytes()); } /** @@ -699,8 +702,14 @@ private void testCleanupReleasedPartition(boolean spilled, boolean createView) t } // note: in case we create a view, there will be an additional EndOfPartitionEvent assertEquals(createView ? 3 : 2, partition.getTotalNumberOfBuffers()); - //TODO: re-enable this? -// assertEquals((createView ? 4 : 0) + 2 * BUFFER_DATA_SIZE, partition.getTotalNumberOfBytes()); + if (spilled) { + // with a view, one buffer remains in nextBuffer and is not counted yet + assertEquals(BUFFER_DATA_SIZE + (createView ? 4 : BUFFER_DATA_SIZE), + partition.getTotalNumberOfBytes()); + } else { + // non-spilled byte statistics are only updated when buffers are consumed + assertEquals(0, partition.getTotalNumberOfBytes()); + } } /** diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java index 48846b6394390b..1b861dfab51db2 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java @@ -52,15 +52,18 @@ public void testAddAfterFinish() throws Exception { try { subpartition.finish(); assertEquals(1, subpartition.getTotalNumberOfBuffers()); + assertEquals(0, subpartition.getTotalNumberOfBytes()); // only updated after consuming the buffers assertEquals(1, subpartition.getTotalNumberOfBuffers()); assertEquals(0, subpartition.getBuffersInBacklog()); + assertEquals(0, subpartition.getTotalNumberOfBytes()); // only updated after consuming the buffers BufferConsumer bufferConsumer = createFilledBufferConsumer(4096, 4096); assertFalse(subpartition.add(bufferConsumer)); assertEquals(1, subpartition.getTotalNumberOfBuffers()); assertEquals(0, subpartition.getBuffersInBacklog()); + assertEquals(0, subpartition.getTotalNumberOfBytes()); // only updated after consuming the buffers } finally { if (subpartition != null) { subpartition.release(); From 81259ad28b58b82ee836452903a37e4172d8468b Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 20 Feb 2018 18:06:41 +0100 Subject: [PATCH 0034/2294] [hotfix][network] remove PowerMockRunner from RecordWriterTest --- .../runtime/io/network/api/writer/RecordWriterTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java index c7ef4f1ce79b50..ec0dfe27169707 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java @@ -41,11 +41,8 @@ import org.apache.flink.util.XORShiftRandom; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; import java.util.ArrayDeque; @@ -71,8 +68,6 @@ /** * Tests for the {@link RecordWriter}. */ -@PrepareForTest({EventSerializer.class}) -@RunWith(PowerMockRunner.class) public class RecordWriterTest { // --------------------------------------------------------------------------------------------- From d30df346b43180a5a5e90b84b0f19b7f379985e2 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 20 Feb 2018 18:07:02 +0100 Subject: [PATCH 0035/2294] [hotfix][network] various minor improvements --- .../runtime/io/network/partition/PipelinedSubpartition.java | 2 ++ .../api/writer/AbstractCollectingResultPartitionWriter.java | 2 +- .../runtime/io/network/partition/SpillableSubpartitionTest.java | 2 -- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java index dcaa3608fb8067..a9c6e57b96597d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java @@ -265,6 +265,8 @@ private void notifyDataAvailable() { } private int getNumberOfFinishedBuffers() { + assert Thread.holdsLock(buffers); + if (buffers.size() == 1 && buffers.peekLast().isFinished()) { return 1; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/AbstractCollectingResultPartitionWriter.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/AbstractCollectingResultPartitionWriter.java index 03243752d6ce20..981ca5665a98db 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/AbstractCollectingResultPartitionWriter.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/AbstractCollectingResultPartitionWriter.java @@ -76,7 +76,7 @@ private void processBufferConsumers() throws IOException { Buffer buffer = bufferConsumer.build(); try { deserializeBuffer(buffer); - if (!bufferConsumers.peek().isFinished()) { + if (!bufferConsumer.isFinished()) { break; } bufferConsumers.pop().close(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java index 43bcd31b3f9964..a6be748194ba7a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java @@ -227,7 +227,6 @@ public void testConsumeSpilledPartition() throws Exception { assertTrue(read.buffer().isBuffer()); assertEquals(2, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertNotSame(bufferConsumer, read); assertFalse(read.buffer().isRecycled()); read.buffer().recycleBuffer(); assertTrue(read.buffer().isRecycled()); @@ -239,7 +238,6 @@ public void testConsumeSpilledPartition() throws Exception { assertTrue(read.buffer().isBuffer()); assertEquals(1, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertNotSame(bufferConsumer, read); assertFalse(read.buffer().isRecycled()); read.buffer().recycleBuffer(); assertTrue(read.buffer().isRecycled()); From 6597e6747804b9c8cb9029bab28e2514917c64ff Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Wed, 21 Feb 2018 16:30:53 +0100 Subject: [PATCH 0036/2294] [hotfix][network] initialize SingleInputGate#enqueuedInputChannelsWithData with the right size --- .../runtime/io/network/partition/consumer/SingleInputGate.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java index 04b8ee6c1585ee..a1f3cdcc5b4b7c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java @@ -144,7 +144,7 @@ public class SingleInputGate implements InputGate { * Field guaranteeing uniqueness for inputChannelsWithData queue. Both of those fields should be unified * onto one. */ - private final BitSet enqueuedInputChannelsWithData = new BitSet(); + private final BitSet enqueuedInputChannelsWithData; private final BitSet channelsWithEndOfPartitionEvents; @@ -205,6 +205,7 @@ public SingleInputGate( this.inputChannels = new HashMap<>(numberOfInputChannels); this.channelsWithEndOfPartitionEvents = new BitSet(numberOfInputChannels); + this.enqueuedInputChannelsWithData = new BitSet(numberOfInputChannels); this.taskActions = checkNotNull(taskActions); } From 9fb1c23aaead71bd7e81a6a73be1b4206dac405f Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Wed, 21 Feb 2018 17:09:31 +0100 Subject: [PATCH 0037/2294] [FLINK-8736][network] fix memory segment offsets for slices of slices being wrong This closes #5551. --- .../buffer/ReadOnlySlicedNetworkBuffer.java | 13 ++-- .../buffer/ReadOnlySlicedBufferTest.java | 59 ++++++++++++++++++- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedNetworkBuffer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedNetworkBuffer.java index e4b81131ed4d70..52fb57a4d8e5ad 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedNetworkBuffer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedNetworkBuffer.java @@ -38,7 +38,7 @@ */ public final class ReadOnlySlicedNetworkBuffer extends ReadOnlyByteBuf implements Buffer { - private final int index; + private final int memorySegmentOffset; /** * Creates a buffer which shares the memory segment of the given buffer and exposed the given @@ -53,7 +53,7 @@ public final class ReadOnlySlicedNetworkBuffer extends ReadOnlyByteBuf implement */ ReadOnlySlicedNetworkBuffer(NetworkBuffer buffer, int index, int length) { super(new SlicedByteBuf(buffer, index, length)); - this.index = index; + this.memorySegmentOffset = buffer.getMemorySegmentOffset() + index; } /** @@ -66,10 +66,11 @@ public final class ReadOnlySlicedNetworkBuffer extends ReadOnlyByteBuf implement * @param buffer the buffer to derive from * @param index the index to start from * @param length the length of the slice + * @param memorySegmentOffset buffer's absolute offset in the backing {@link MemorySegment} */ - private ReadOnlySlicedNetworkBuffer(ByteBuf buffer, int index, int length) { + private ReadOnlySlicedNetworkBuffer(ByteBuf buffer, int index, int length, int memorySegmentOffset) { super(new SlicedByteBuf(buffer, index, length)); - this.index = index; + this.memorySegmentOffset = memorySegmentOffset + index; } @Override @@ -102,7 +103,7 @@ public MemorySegment getMemorySegment() { @Override public int getMemorySegmentOffset() { - return ((Buffer) unwrap()).getMemorySegmentOffset() + index; + return memorySegmentOffset; } @Override @@ -133,7 +134,7 @@ public ReadOnlySlicedNetworkBuffer readOnlySlice() { @Override public ReadOnlySlicedNetworkBuffer readOnlySlice(int index, int length) { - return new ReadOnlySlicedNetworkBuffer(super.unwrap(), index, length); + return new ReadOnlySlicedNetworkBuffer(super.unwrap(), index, length, memorySegmentOffset); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedBufferTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedBufferTest.java index 834ec74180feee..529b0f45f9a41a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedBufferTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/ReadOnlySlicedBufferTest.java @@ -33,6 +33,7 @@ import java.nio.ByteBuffer; import java.nio.ReadOnlyBufferException; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -50,8 +51,10 @@ public class ReadOnlySlicedBufferTest { @Before public void setUp() throws Exception { final MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(BUFFER_SIZE); - buffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, true, DATA_SIZE); - buffer.setSize(DATA_SIZE); + buffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, true, 0); + for (int i = 0; i < DATA_SIZE; ++i) { + buffer.writeByte(i); + } } @Test @@ -137,34 +140,64 @@ public void testForwardsRetainBuffer2() { @Test public void testCreateSlice1() { + buffer.readByte(); // so that we do not start at position 0 ReadOnlySlicedNetworkBuffer slice1 = buffer.readOnlySlice(); + buffer.readByte(); // should not influence the second slice at all ReadOnlySlicedNetworkBuffer slice2 = slice1.readOnlySlice(); ByteBuf unwrap = slice2.unwrap(); assertSame(buffer, unwrap); + assertSame(slice1.getMemorySegment(), slice2.getMemorySegment()); + assertEquals(1, slice1.getMemorySegmentOffset()); + assertEquals(slice1.getMemorySegmentOffset(), slice2.getMemorySegmentOffset()); + + assertReadableBytes(slice1, 1, 2, 3, 4, 5, 6, 7, 8, 9); + assertReadableBytes(slice2, 1, 2, 3, 4, 5, 6, 7, 8, 9); } @Test public void testCreateSlice2() { + buffer.readByte(); // so that we do not start at position 0 ReadOnlySlicedNetworkBuffer slice1 = buffer.readOnlySlice(); + buffer.readByte(); // should not influence the second slice at all ReadOnlySlicedNetworkBuffer slice2 = slice1.readOnlySlice(1, 2); ByteBuf unwrap = slice2.unwrap(); assertSame(buffer, unwrap); + assertSame(slice1.getMemorySegment(), slice2.getMemorySegment()); + assertEquals(1, slice1.getMemorySegmentOffset()); + assertEquals(2, slice2.getMemorySegmentOffset()); + + assertReadableBytes(slice1, 1, 2, 3, 4, 5, 6, 7, 8, 9); + assertReadableBytes(slice2, 2, 3); } @Test public void testCreateSlice3() { ReadOnlySlicedNetworkBuffer slice1 = buffer.readOnlySlice(1, 2); + buffer.readByte(); // should not influence the second slice at all ReadOnlySlicedNetworkBuffer slice2 = slice1.readOnlySlice(); ByteBuf unwrap = slice2.unwrap(); assertSame(buffer, unwrap); + assertSame(slice1.getMemorySegment(), slice2.getMemorySegment()); + assertEquals(1, slice1.getMemorySegmentOffset()); + assertEquals(1, slice2.getMemorySegmentOffset()); + + assertReadableBytes(slice1, 1, 2); + assertReadableBytes(slice2, 1, 2); } @Test public void testCreateSlice4() { ReadOnlySlicedNetworkBuffer slice1 = buffer.readOnlySlice(1, 5); + buffer.readByte(); // should not influence the second slice at all ReadOnlySlicedNetworkBuffer slice2 = slice1.readOnlySlice(1, 2); ByteBuf unwrap = slice2.unwrap(); assertSame(buffer, unwrap); + assertSame(slice1.getMemorySegment(), slice2.getMemorySegment()); + assertEquals(1, slice1.getMemorySegmentOffset()); + assertEquals(2, slice2.getMemorySegmentOffset()); + + assertReadableBytes(slice1, 1, 2, 3, 4, 5); + assertReadableBytes(slice2, 2, 3); } @Test @@ -323,4 +356,26 @@ private void testForwardsSetAllocator(ReadOnlySlicedNetworkBuffer slice) { assertSame(buffer.alloc(), slice.alloc()); assertSame(allocator, slice.alloc()); } + + private static void assertReadableBytes(Buffer actualBuffer, int... expectedBytes) { + ByteBuffer actualBytesBuffer = actualBuffer.getNioBufferReadable(); + int[] actual = new int[actualBytesBuffer.limit()]; + for (int i = 0; i < actual.length; ++i) { + actual[i] = actualBytesBuffer.get(); + } + assertArrayEquals(expectedBytes, actual); + + // verify absolutely positioned read method: + ByteBuf buffer = (ByteBuf) actualBuffer; + for (int i = 0; i < buffer.readableBytes(); ++i) { + actual[i] = buffer.getByte(buffer.readerIndex() + i); + } + assertArrayEquals(expectedBytes, actual); + + // verify relatively positioned read method: + for (int i = 0; i < buffer.readableBytes(); ++i) { + actual[i] = buffer.readByte(); + } + assertArrayEquals(expectedBytes, actual); + } } From cf7d6024248555877b8b0463bdd0fdba7ed69365 Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Tue, 20 Feb 2018 11:04:43 -0800 Subject: [PATCH 0038/2294] [FLINK-8719] Add module description for flink-contrib to clarify its purpose This closes #5537. --- flink-contrib/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 flink-contrib/README.md diff --git a/flink-contrib/README.md b/flink-contrib/README.md new file mode 100644 index 00000000000000..861d8a19e32493 --- /dev/null +++ b/flink-contrib/README.md @@ -0,0 +1,4 @@ +## Purpose of this module + +`flink-contrib` is a staging/incubating area for new modules while the community evaluates +how they are received by users and whether there will be commitment to maintain them in long term. From a13f7b23440d865e451f68b1e5e0febb18e0e46f Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 27 Feb 2018 12:20:51 +0100 Subject: [PATCH 0039/2294] [hotfix][tests] Remove unused variable --- .../test/misc/SuccessAfterNetworkBuffersFailureITCase.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java index 02fdaf99981155..dc19ad1d64712e 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java @@ -32,7 +32,6 @@ import org.apache.flink.examples.java.graph.ConnectedComponents; import org.apache.flink.examples.java.graph.util.ConnectedComponentsData; import org.apache.flink.runtime.client.JobExecutionException; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.util.TestLogger; @@ -67,8 +66,6 @@ private static Configuration getConfiguration() { @Test public void testSuccessfulProgramAfterFailure() { - LocalFlinkMiniCluster cluster = null; - try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); From 8d180d5fa69a68a7023b052f83e51ca6a3d35256 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 27 Feb 2018 14:44:35 +0100 Subject: [PATCH 0040/2294] Update version to 1.6-SNAPSHOT --- docs/_config.yml | 6 +++--- flink-annotations/pom.xml | 2 +- flink-clients/pom.xml | 2 +- flink-connectors/flink-connector-cassandra/pom.xml | 2 +- flink-connectors/flink-connector-elasticsearch-base/pom.xml | 2 +- flink-connectors/flink-connector-elasticsearch/pom.xml | 2 +- flink-connectors/flink-connector-elasticsearch2/pom.xml | 2 +- flink-connectors/flink-connector-elasticsearch5/pom.xml | 2 +- flink-connectors/flink-connector-filesystem/pom.xml | 2 +- flink-connectors/flink-connector-kafka-0.10/pom.xml | 2 +- flink-connectors/flink-connector-kafka-0.11/pom.xml | 2 +- flink-connectors/flink-connector-kafka-0.8/pom.xml | 2 +- flink-connectors/flink-connector-kafka-0.9/pom.xml | 2 +- flink-connectors/flink-connector-kafka-base/pom.xml | 2 +- flink-connectors/flink-connector-kinesis/pom.xml | 2 +- flink-connectors/flink-connector-nifi/pom.xml | 2 +- flink-connectors/flink-connector-rabbitmq/pom.xml | 2 +- flink-connectors/flink-connector-twitter/pom.xml | 2 +- flink-connectors/flink-hadoop-compatibility/pom.xml | 2 +- flink-connectors/flink-hbase/pom.xml | 2 +- flink-connectors/flink-hcatalog/pom.xml | 2 +- flink-connectors/flink-jdbc/pom.xml | 2 +- flink-connectors/flink-orc/pom.xml | 2 +- flink-connectors/pom.xml | 2 +- flink-contrib/flink-connector-wikiedits/pom.xml | 2 +- flink-contrib/flink-storm-examples/pom.xml | 2 +- flink-contrib/flink-storm/pom.xml | 2 +- flink-contrib/pom.xml | 2 +- flink-core/pom.xml | 2 +- flink-dist/pom.xml | 2 +- flink-docs/pom.xml | 2 +- flink-end-to-end-tests/pom.xml | 2 +- flink-examples/flink-examples-batch/pom.xml | 2 +- flink-examples/flink-examples-streaming/pom.xml | 2 +- flink-examples/flink-examples-table/pom.xml | 2 +- flink-examples/pom.xml | 2 +- flink-filesystems/flink-hadoop-fs/pom.xml | 2 +- flink-filesystems/flink-mapr-fs/pom.xml | 2 +- flink-filesystems/flink-s3-fs-hadoop/pom.xml | 2 +- flink-filesystems/flink-s3-fs-presto/pom.xml | 2 +- flink-filesystems/flink-swift-fs-hadoop/pom.xml | 2 +- flink-filesystems/pom.xml | 2 +- flink-formats/flink-avro/pom.xml | 2 +- flink-formats/flink-json/pom.xml | 2 +- flink-formats/pom.xml | 2 +- flink-fs-tests/pom.xml | 2 +- flink-java/pom.xml | 2 +- flink-java8/pom.xml | 2 +- flink-libraries/flink-cep-scala/pom.xml | 2 +- flink-libraries/flink-cep/pom.xml | 2 +- flink-libraries/flink-gelly-examples/pom.xml | 2 +- flink-libraries/flink-gelly-scala/pom.xml | 2 +- flink-libraries/flink-gelly/pom.xml | 2 +- flink-libraries/flink-ml/pom.xml | 2 +- flink-libraries/flink-python/pom.xml | 2 +- flink-libraries/flink-sql-client/pom.xml | 2 +- flink-libraries/flink-streaming-python/pom.xml | 2 +- flink-libraries/flink-table/pom.xml | 2 +- flink-libraries/pom.xml | 2 +- flink-mesos/pom.xml | 2 +- flink-metrics/flink-metrics-core/pom.xml | 2 +- flink-metrics/flink-metrics-datadog/pom.xml | 2 +- flink-metrics/flink-metrics-dropwizard/pom.xml | 2 +- flink-metrics/flink-metrics-ganglia/pom.xml | 2 +- flink-metrics/flink-metrics-graphite/pom.xml | 2 +- flink-metrics/flink-metrics-jmx/pom.xml | 2 +- flink-metrics/flink-metrics-prometheus/pom.xml | 2 +- flink-metrics/flink-metrics-slf4j/pom.xml | 2 +- flink-metrics/flink-metrics-statsd/pom.xml | 2 +- flink-metrics/pom.xml | 2 +- flink-optimizer/pom.xml | 2 +- .../flink-queryable-state-client-java/pom.xml | 2 +- flink-queryable-state/flink-queryable-state-runtime/pom.xml | 2 +- flink-queryable-state/pom.xml | 2 +- flink-quickstart/flink-quickstart-java/pom.xml | 2 +- flink-quickstart/flink-quickstart-scala/pom.xml | 2 +- flink-quickstart/pom.xml | 2 +- flink-runtime-web/pom.xml | 2 +- flink-runtime/pom.xml | 2 +- flink-scala-shell/pom.xml | 2 +- flink-scala/pom.xml | 2 +- flink-shaded-curator/pom.xml | 2 +- flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml | 2 +- flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml | 2 +- flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml | 2 +- flink-shaded-hadoop/pom.xml | 2 +- flink-state-backends/flink-statebackend-rocksdb/pom.xml | 2 +- flink-state-backends/pom.xml | 2 +- flink-streaming-java/pom.xml | 2 +- flink-streaming-scala/pom.xml | 2 +- flink-test-utils-parent/flink-test-utils-junit/pom.xml | 2 +- flink-test-utils-parent/flink-test-utils/pom.xml | 2 +- flink-test-utils-parent/pom.xml | 2 +- flink-tests/pom.xml | 2 +- flink-yarn-tests/pom.xml | 2 +- flink-yarn/pom.xml | 2 +- pom.xml | 4 ++-- tools/force-shading/pom.xml | 2 +- 98 files changed, 101 insertions(+), 101 deletions(-) diff --git a/docs/_config.yml b/docs/_config.yml index 2937499f81334a..605740878ca5c5 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -27,11 +27,11 @@ # we change the version for the complete docs when forking of a release branch # etc. # The full version string as referenced in Maven (e.g. 1.2.1) -version: "1.5-SNAPSHOT" +version: "1.6-SNAPSHOT" # For stable releases, leave the bugfix version out (e.g. 1.2). For snapshot # release this should be the same as the regular version -version_title: "1.5-SNAPSHOT" -version_javadocs: "1.5" +version_title: "1.6-SNAPSHOT" +version_javadocs: "1.6-SNAPSHOT" # This suffix is appended to the Scala-dependent Maven artifact names scala_version_suffix: "_2.11" diff --git a/flink-annotations/pom.xml b/flink-annotations/pom.xml index 7433471a981935..f28b32c5ee4ba8 100644 --- a/flink-annotations/pom.xml +++ b/flink-annotations/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml index d175b8d031878c..50379c7fb320a6 100644 --- a/flink-clients/pom.xml +++ b/flink-clients/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-cassandra/pom.xml b/flink-connectors/flink-connector-cassandra/pom.xml index 0fbd968038aa3b..d1126233420c67 100644 --- a/flink-connectors/flink-connector-cassandra/pom.xml +++ b/flink-connectors/flink-connector-cassandra/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-elasticsearch-base/pom.xml b/flink-connectors/flink-connector-elasticsearch-base/pom.xml index 9694786f9213bc..4b4c93c68ec00a 100644 --- a/flink-connectors/flink-connector-elasticsearch-base/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch-base/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-elasticsearch/pom.xml b/flink-connectors/flink-connector-elasticsearch/pom.xml index 5187b14143f05a..1714ec3752012d 100644 --- a/flink-connectors/flink-connector-elasticsearch/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml index 5660219d331f3d..1068c1b4826a3d 100644 --- a/flink-connectors/flink-connector-elasticsearch2/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml index 16b7cff89b278e..fc97294a8db1aa 100644 --- a/flink-connectors/flink-connector-elasticsearch5/pom.xml +++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-filesystem/pom.xml b/flink-connectors/flink-connector-filesystem/pom.xml index f1e7473161a650..7907242f469c6a 100644 --- a/flink-connectors/flink-connector-filesystem/pom.xml +++ b/flink-connectors/flink-connector-filesystem/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml index 8b4ff38ab73f35..1ae1cd89206324 100644 --- a/flink-connectors/flink-connector-kafka-0.10/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-kafka-0.11/pom.xml b/flink-connectors/flink-connector-kafka-0.11/pom.xml index 1e935f658a8a34..8a4e339431f6dd 100644 --- a/flink-connectors/flink-connector-kafka-0.11/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.11/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-kafka-0.8/pom.xml b/flink-connectors/flink-connector-kafka-0.8/pom.xml index a58591e5c6d80e..02d36f5fceb409 100644 --- a/flink-connectors/flink-connector-kafka-0.8/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.8/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml index d07cb5a83ea2a0..ee0c458171096d 100644 --- a/flink-connectors/flink-connector-kafka-0.9/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml index e7412cf1d5222e..4620b8fca00f6b 100644 --- a/flink-connectors/flink-connector-kafka-base/pom.xml +++ b/flink-connectors/flink-connector-kafka-base/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-kinesis/pom.xml b/flink-connectors/flink-connector-kinesis/pom.xml index 99629bfec955a1..43046cc4021d03 100644 --- a/flink-connectors/flink-connector-kinesis/pom.xml +++ b/flink-connectors/flink-connector-kinesis/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-nifi/pom.xml b/flink-connectors/flink-connector-nifi/pom.xml index 3fe6f236c5e420..dfc73f940a9ee6 100644 --- a/flink-connectors/flink-connector-nifi/pom.xml +++ b/flink-connectors/flink-connector-nifi/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-rabbitmq/pom.xml b/flink-connectors/flink-connector-rabbitmq/pom.xml index cc80c95fb42541..0a9549818590ad 100644 --- a/flink-connectors/flink-connector-rabbitmq/pom.xml +++ b/flink-connectors/flink-connector-rabbitmq/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-connector-twitter/pom.xml b/flink-connectors/flink-connector-twitter/pom.xml index 8aa8af202bde73..6faa86b629699b 100644 --- a/flink-connectors/flink-connector-twitter/pom.xml +++ b/flink-connectors/flink-connector-twitter/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml index 468dc0a4e370dc..0d23e470b81816 100644 --- a/flink-connectors/flink-hadoop-compatibility/pom.xml +++ b/flink-connectors/flink-hadoop-compatibility/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml index f70249934b3355..9b2d8e97b8d92e 100644 --- a/flink-connectors/flink-hbase/pom.xml +++ b/flink-connectors/flink-hbase/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml index e613dde7299921..32a42ca4a26d7d 100644 --- a/flink-connectors/flink-hcatalog/pom.xml +++ b/flink-connectors/flink-hcatalog/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-jdbc/pom.xml b/flink-connectors/flink-jdbc/pom.xml index 168d173cbafa6a..ec84afb8dc04a6 100644 --- a/flink-connectors/flink-jdbc/pom.xml +++ b/flink-connectors/flink-jdbc/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/flink-orc/pom.xml b/flink-connectors/flink-orc/pom.xml index 3ee5e493318782..689cb08b9f0d70 100644 --- a/flink-connectors/flink-orc/pom.xml +++ b/flink-connectors/flink-orc/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-connectors - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml index 1b77833a836191..bb98403ff2d6fd 100644 --- a/flink-connectors/pom.xml +++ b/flink-connectors/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-contrib/flink-connector-wikiedits/pom.xml b/flink-contrib/flink-connector-wikiedits/pom.xml index 39086589d3f5db..57f56268f85820 100644 --- a/flink-contrib/flink-connector-wikiedits/pom.xml +++ b/flink-contrib/flink-connector-wikiedits/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-contrib - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-contrib/flink-storm-examples/pom.xml b/flink-contrib/flink-storm-examples/pom.xml index fcd4f4e492202c..effdd979f0842e 100644 --- a/flink-contrib/flink-storm-examples/pom.xml +++ b/flink-contrib/flink-storm-examples/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-contrib - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml index 7abe054c1f5d0b..496aecd34ae3ea 100644 --- a/flink-contrib/flink-storm/pom.xml +++ b/flink-contrib/flink-storm/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-contrib - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-contrib/pom.xml b/flink-contrib/pom.xml index 644f2990d6a6a2..5d935719eaebab 100644 --- a/flink-contrib/pom.xml +++ b/flink-contrib/pom.xml @@ -27,7 +27,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-core/pom.xml b/flink-core/pom.xml index 567de5c25b2ab7..aaebeb0152327e 100644 --- a/flink-core/pom.xml +++ b/flink-core/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index c555f666bed20c..019047c56b5a6a 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml index 6abf52b1db0e6a..8b41229865f4af 100644 --- a/flink-docs/pom.xml +++ b/flink-docs/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml index 6e9dee67757474..a5bbc52ff462b3 100644 --- a/flink-end-to-end-tests/pom.xml +++ b/flink-end-to-end-tests/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-examples/flink-examples-batch/pom.xml b/flink-examples/flink-examples-batch/pom.xml index e8820eb075aa10..a8e455c26b9c14 100644 --- a/flink-examples/flink-examples-batch/pom.xml +++ b/flink-examples/flink-examples-batch/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.flink flink-examples - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml index c9367b75d32154..ea253d8ea97710 100644 --- a/flink-examples/flink-examples-streaming/pom.xml +++ b/flink-examples/flink-examples-streaming/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-examples - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml index 91c6a9670c2d28..3eb53e6c500318 100644 --- a/flink-examples/flink-examples-table/pom.xml +++ b/flink-examples/flink-examples-table/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-examples - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml index 81584b91544dbe..eec41b0a1b2599 100644 --- a/flink-examples/pom.xml +++ b/flink-examples/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-filesystems/flink-hadoop-fs/pom.xml b/flink-filesystems/flink-hadoop-fs/pom.xml index 425aa2c4a86ac8..6137036b56bdd8 100644 --- a/flink-filesystems/flink-hadoop-fs/pom.xml +++ b/flink-filesystems/flink-hadoop-fs/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-filesystems - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-filesystems/flink-mapr-fs/pom.xml b/flink-filesystems/flink-mapr-fs/pom.xml index d59baafdf82740..d67b9021165c8d 100644 --- a/flink-filesystems/flink-mapr-fs/pom.xml +++ b/flink-filesystems/flink-mapr-fs/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-filesystems - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml index ca5790940a1aed..5dd911e5ed9f3f 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-s3-fs-hadoop/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-filesystems - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml index 8ff756183a1c18..b49c8149587bde 100644 --- a/flink-filesystems/flink-s3-fs-presto/pom.xml +++ b/flink-filesystems/flink-s3-fs-presto/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-filesystems - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-filesystems/flink-swift-fs-hadoop/pom.xml b/flink-filesystems/flink-swift-fs-hadoop/pom.xml index 66ee6d77492941..5d6bce8c369cc9 100644 --- a/flink-filesystems/flink-swift-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-swift-fs-hadoop/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-filesystems - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-filesystems/pom.xml b/flink-filesystems/pom.xml index 341348c0cf910b..25e0bbb0ce1c41 100644 --- a/flink-filesystems/pom.xml +++ b/flink-filesystems/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-formats/flink-avro/pom.xml b/flink-formats/flink-avro/pom.xml index 56768134c6987a..3458760b2f74da 100644 --- a/flink-formats/flink-avro/pom.xml +++ b/flink-formats/flink-avro/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-formats - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml index ccc48f4e41bc6e..d0f55ab43efa99 100644 --- a/flink-formats/flink-json/pom.xml +++ b/flink-formats/flink-json/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-formats - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-formats/pom.xml b/flink-formats/pom.xml index ad776cffdc818e..7cb67e8bd42606 100644 --- a/flink-formats/pom.xml +++ b/flink-formats/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml index 71a69a33dff027..ceb78daa36d491 100644 --- a/flink-fs-tests/pom.xml +++ b/flink-fs-tests/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-java/pom.xml b/flink-java/pom.xml index e8dd33b27426cb..c1e324107c1136 100644 --- a/flink-java/pom.xml +++ b/flink-java/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-java8/pom.xml b/flink-java8/pom.xml index 273146d5d47495..a8750288f99c7d 100644 --- a/flink-java8/pom.xml +++ b/flink-java8/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-cep-scala/pom.xml b/flink-libraries/flink-cep-scala/pom.xml index 0ea9deae4bbd4f..f00fbf4b714cdb 100644 --- a/flink-libraries/flink-cep-scala/pom.xml +++ b/flink-libraries/flink-cep-scala/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-cep/pom.xml b/flink-libraries/flink-cep/pom.xml index 4cbdc7f881797e..04c4c04d4fd4f3 100644 --- a/flink-libraries/flink-cep/pom.xml +++ b/flink-libraries/flink-cep/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-gelly-examples/pom.xml b/flink-libraries/flink-gelly-examples/pom.xml index 70e38201d6dc68..fc2e4b98e76588 100644 --- a/flink-libraries/flink-gelly-examples/pom.xml +++ b/flink-libraries/flink-gelly-examples/pom.xml @@ -23,7 +23,7 @@ org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-gelly-scala/pom.xml b/flink-libraries/flink-gelly-scala/pom.xml index 3d14db97fc6e69..50ac01e778c176 100644 --- a/flink-libraries/flink-gelly-scala/pom.xml +++ b/flink-libraries/flink-gelly-scala/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. 4.0.0 diff --git a/flink-libraries/flink-gelly/pom.xml b/flink-libraries/flink-gelly/pom.xml index 7dd3083f9fcaed..52730c5c8ede16 100644 --- a/flink-libraries/flink-gelly/pom.xml +++ b/flink-libraries/flink-gelly/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-ml/pom.xml b/flink-libraries/flink-ml/pom.xml index b78e41e148cf3d..f05c7e7df3e8e0 100644 --- a/flink-libraries/flink-ml/pom.xml +++ b/flink-libraries/flink-ml/pom.xml @@ -24,7 +24,7 @@ org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-python/pom.xml b/flink-libraries/flink-python/pom.xml index a8a1d0b9cf08c8..30ba11aeb72a11 100644 --- a/flink-libraries/flink-python/pom.xml +++ b/flink-libraries/flink-python/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-sql-client/pom.xml b/flink-libraries/flink-sql-client/pom.xml index 743f58308b7cea..03fca24c917c43 100644 --- a/flink-libraries/flink-sql-client/pom.xml +++ b/flink-libraries/flink-sql-client/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-streaming-python/pom.xml b/flink-libraries/flink-streaming-python/pom.xml index a3ae8b78434f92..2ad76806e08475 100644 --- a/flink-libraries/flink-streaming-python/pom.xml +++ b/flink-libraries/flink-streaming-python/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml index bf6585a0be29dd..b4b0eee0050736 100644 --- a/flink-libraries/flink-table/pom.xml +++ b/flink-libraries/flink-table/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-libraries - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-libraries/pom.xml b/flink-libraries/pom.xml index c8bfa93c4f0b8c..c0ac1241a47d93 100644 --- a/flink-libraries/pom.xml +++ b/flink-libraries/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-mesos/pom.xml b/flink-mesos/pom.xml index 4e1ec76bd1ceac..e838d62e426b8d 100644 --- a/flink-mesos/pom.xml +++ b/flink-mesos/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-core/pom.xml b/flink-metrics/flink-metrics-core/pom.xml index cad9c3605f8f0c..6c757c0170e72b 100644 --- a/flink-metrics/flink-metrics-core/pom.xml +++ b/flink-metrics/flink-metrics-core/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-datadog/pom.xml b/flink-metrics/flink-metrics-datadog/pom.xml index f62afd1f36067d..995dafc91fa02f 100644 --- a/flink-metrics/flink-metrics-datadog/pom.xml +++ b/flink-metrics/flink-metrics-datadog/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-dropwizard/pom.xml b/flink-metrics/flink-metrics-dropwizard/pom.xml index 532cbf5b31c442..11ca3becce3575 100644 --- a/flink-metrics/flink-metrics-dropwizard/pom.xml +++ b/flink-metrics/flink-metrics-dropwizard/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-ganglia/pom.xml b/flink-metrics/flink-metrics-ganglia/pom.xml index 427d069abd06e4..b6f147d807bbf4 100644 --- a/flink-metrics/flink-metrics-ganglia/pom.xml +++ b/flink-metrics/flink-metrics-ganglia/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-graphite/pom.xml b/flink-metrics/flink-metrics-graphite/pom.xml index 65d7047a9271b0..9e7bbac6299a05 100644 --- a/flink-metrics/flink-metrics-graphite/pom.xml +++ b/flink-metrics/flink-metrics-graphite/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-jmx/pom.xml b/flink-metrics/flink-metrics-jmx/pom.xml index 66de1b3f90b8f6..d738a7e9abeab2 100644 --- a/flink-metrics/flink-metrics-jmx/pom.xml +++ b/flink-metrics/flink-metrics-jmx/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-prometheus/pom.xml b/flink-metrics/flink-metrics-prometheus/pom.xml index 544843c3cf5671..cb983edc1312ee 100644 --- a/flink-metrics/flink-metrics-prometheus/pom.xml +++ b/flink-metrics/flink-metrics-prometheus/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-slf4j/pom.xml b/flink-metrics/flink-metrics-slf4j/pom.xml index d610af23224d20..c33a3a17acbb6c 100644 --- a/flink-metrics/flink-metrics-slf4j/pom.xml +++ b/flink-metrics/flink-metrics-slf4j/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/flink-metrics-statsd/pom.xml b/flink-metrics/flink-metrics-statsd/pom.xml index 9341987a70fe42..d99865b9cd72fe 100644 --- a/flink-metrics/flink-metrics-statsd/pom.xml +++ b/flink-metrics/flink-metrics-statsd/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-metrics - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-metrics/pom.xml b/flink-metrics/pom.xml index b98cc34a6e6f83..a0aba30e97f0d2 100644 --- a/flink-metrics/pom.xml +++ b/flink-metrics/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-optimizer/pom.xml b/flink-optimizer/pom.xml index 33ab35bf121f70..903621a537c498 100644 --- a/flink-optimizer/pom.xml +++ b/flink-optimizer/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-queryable-state/flink-queryable-state-client-java/pom.xml b/flink-queryable-state/flink-queryable-state-client-java/pom.xml index 3616d0e5e20521..d377292ead8a02 100644 --- a/flink-queryable-state/flink-queryable-state-client-java/pom.xml +++ b/flink-queryable-state/flink-queryable-state-client-java/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-queryable-state - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-queryable-state/flink-queryable-state-runtime/pom.xml b/flink-queryable-state/flink-queryable-state-runtime/pom.xml index 7c003be46a23af..294a8ddaefb9bb 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/pom.xml +++ b/flink-queryable-state/flink-queryable-state-runtime/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-queryable-state - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-queryable-state/pom.xml b/flink-queryable-state/pom.xml index 651b1dcd5d7d39..2503a9381bc750 100644 --- a/flink-queryable-state/pom.xml +++ b/flink-queryable-state/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-quickstart/flink-quickstart-java/pom.xml b/flink-quickstart/flink-quickstart-java/pom.xml index ab41e471f317d6..45d44b04dfb8a8 100644 --- a/flink-quickstart/flink-quickstart-java/pom.xml +++ b/flink-quickstart/flink-quickstart-java/pom.xml @@ -27,7 +27,7 @@ under the License. org.apache.flink flink-quickstart - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-quickstart/flink-quickstart-scala/pom.xml b/flink-quickstart/flink-quickstart-scala/pom.xml index 734a5f9ecff8a0..6026f2518a8f16 100644 --- a/flink-quickstart/flink-quickstart-scala/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/pom.xml @@ -27,7 +27,7 @@ under the License. org.apache.flink flink-quickstart - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-quickstart/pom.xml b/flink-quickstart/pom.xml index 7878969a86781c..deb891eff09e14 100644 --- a/flink-quickstart/pom.xml +++ b/flink-quickstart/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-runtime-web/pom.xml b/flink-runtime-web/pom.xml index fc45ddb13a2ff3..837aadb4b25131 100644 --- a/flink-runtime-web/pom.xml +++ b/flink-runtime-web/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index 2fa922eaf76795..5ed096a6fcdddf 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-scala-shell/pom.xml b/flink-scala-shell/pom.xml index 8e0e751b49a985..f5621bcb6ba666 100644 --- a/flink-scala-shell/pom.xml +++ b/flink-scala-shell/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml index 05d0f6776f7fa5..6fce155864f9cc 100644 --- a/flink-scala/pom.xml +++ b/flink-scala/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-shaded-curator/pom.xml b/flink-shaded-curator/pom.xml index 587e4754d79130..f623a1e99dc9b4 100644 --- a/flink-shaded-curator/pom.xml +++ b/flink-shaded-curator/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml index 2af2dba54d2495..6699347a9f808c 100644 --- a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml +++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-shaded-hadoop - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml index f842c1cdeb2489..1c195d4e1ef625 100644 --- a/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml +++ b/flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-shaded-hadoop - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml b/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml index 65ac699cb99cc6..d469a95cc9ab00 100644 --- a/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml +++ b/flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-shaded-hadoop - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml index 5e147d775dc21a..242087b326635c 100644 --- a/flink-shaded-hadoop/pom.xml +++ b/flink-shaded-hadoop/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-state-backends/flink-statebackend-rocksdb/pom.xml b/flink-state-backends/flink-statebackend-rocksdb/pom.xml index c6a9f92081a2a8..32c6168a07d22b 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/pom.xml +++ b/flink-state-backends/flink-statebackend-rocksdb/pom.xml @@ -27,7 +27,7 @@ under the License. org.apache.flink flink-state-backends - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-state-backends/pom.xml b/flink-state-backends/pom.xml index c02ad84fb2206a..d87cecad1fbf62 100644 --- a/flink-state-backends/pom.xml +++ b/flink-state-backends/pom.xml @@ -27,7 +27,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml index ef8f1622b4b08e..300a2926c96d7b 100644 --- a/flink-streaming-java/pom.xml +++ b/flink-streaming-java/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-streaming-scala/pom.xml b/flink-streaming-scala/pom.xml index 16a5a21d408560..7bcf9d7a590218 100644 --- a/flink-streaming-scala/pom.xml +++ b/flink-streaming-scala/pom.xml @@ -24,7 +24,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-test-utils-parent/flink-test-utils-junit/pom.xml b/flink-test-utils-parent/flink-test-utils-junit/pom.xml index 970fbde7589083..4310c2e9eb9540 100644 --- a/flink-test-utils-parent/flink-test-utils-junit/pom.xml +++ b/flink-test-utils-parent/flink-test-utils-junit/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-test-utils-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-test-utils-parent/flink-test-utils/pom.xml b/flink-test-utils-parent/flink-test-utils/pom.xml index d13febc9fbd8db..2e92cfb3c32065 100644 --- a/flink-test-utils-parent/flink-test-utils/pom.xml +++ b/flink-test-utils-parent/flink-test-utils/pom.xml @@ -25,7 +25,7 @@ under the License. org.apache.flink flink-test-utils-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-test-utils-parent/pom.xml b/flink-test-utils-parent/pom.xml index fd5a730c72bda3..5fbc6685c01920 100644 --- a/flink-test-utils-parent/pom.xml +++ b/flink-test-utils-parent/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml index 9d767a2e2a7eea..8dd131b6317d11 100644 --- a/flink-tests/pom.xml +++ b/flink-tests/pom.xml @@ -26,7 +26,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml index 6af7126c64bb04..b5a86b4b8f2f72 100644 --- a/flink-yarn-tests/pom.xml +++ b/flink-yarn-tests/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/flink-yarn/pom.xml b/flink-yarn/pom.xml index aa5a00a215085e..729358d69b3926 100644 --- a/flink-yarn/pom.xml +++ b/flink-yarn/pom.xml @@ -23,7 +23,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 5f0ebb4a00c95d..a140e6e3687a1a 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ under the License. org.apache.flink flink-parent - 1.5-SNAPSHOT + 1.6-SNAPSHOT flink pom @@ -146,7 +146,7 @@ under the License. org.apache.flink force-shading - 1.5-SNAPSHOT + 1.6-SNAPSHOT diff --git a/tools/force-shading/pom.xml b/tools/force-shading/pom.xml index 9089c07edd4b85..eefed0c8a98960 100644 --- a/tools/force-shading/pom.xml +++ b/tools/force-shading/pom.xml @@ -38,7 +38,7 @@ under the License. org.apache.flink force-shading - 1.5-SNAPSHOT + 1.6-SNAPSHOT jar From 56c756040fc4b3f224c4e7c12d208a3ccf5a7c5e Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Mon, 26 Feb 2018 18:03:14 +0100 Subject: [PATCH 0041/2294] [hotfix] Improved logging for task local recovery --- .../PrioritizedOperatorSubtaskState.java | 31 ++++----- .../TaskExecutorLocalStateStoresManager.java | 68 +++++++++++++------ .../state/TaskLocalStateStoreImpl.java | 63 +++++++++++------ .../runtime/state/TaskStateManagerImpl.java | 27 +++++--- .../PrioritizedOperatorSubtaskStateTest.java | 4 +- .../state/TaskStateManagerImplTest.java | 4 +- .../operators/BackendRestorerProcedure.java | 61 +++++++++++------ .../StreamTaskStateInitializerImpl.java | 20 ++++-- .../BackendRestorerProcedureTest.java | 18 ++--- 9 files changed, 193 insertions(+), 103 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java index f48d3110c53c07..512f912620bc5d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.function.BiFunction; @@ -79,39 +78,39 @@ public class PrioritizedOperatorSubtaskState { // ----------------------------------------------------------------------------------------------------------------- /** - * Returns an iterator over all alternative snapshots to restore the managed operator state, in the order in which - * we should attempt to restore. + * Returns an immutable list with all alternative snapshots to restore the managed operator state, in the order in + * which we should attempt to restore. */ @Nonnull - public Iterator> getPrioritizedManagedOperatorState() { - return prioritizedManagedOperatorState.iterator(); + public List> getPrioritizedManagedOperatorState() { + return prioritizedManagedOperatorState; } /** - * Returns an iterator over all alternative snapshots to restore the raw operator state, in the order in which we - * should attempt to restore. + * Returns an immutable list with all alternative snapshots to restore the raw operator state, in the order in + * which we should attempt to restore. */ @Nonnull - public Iterator> getPrioritizedRawOperatorState() { - return prioritizedRawOperatorState.iterator(); + public List> getPrioritizedRawOperatorState() { + return prioritizedRawOperatorState; } /** - * Returns an iterator over all alternative snapshots to restore the managed keyed state, in the order in which we - * should attempt to restore. + * Returns an immutable list with all alternative snapshots to restore the managed keyed state, in the order in + * which we should attempt to restore. */ @Nonnull - public Iterator> getPrioritizedManagedKeyedState() { - return prioritizedManagedKeyedState.iterator(); + public List> getPrioritizedManagedKeyedState() { + return prioritizedManagedKeyedState; } /** - * Returns an iterator over all alternative snapshots to restore the raw keyed state, in the order in which we + * Returns an immutable list with all alternative snapshots to restore the raw keyed state, in the order in which we * should attempt to restore. */ @Nonnull - public Iterator> getPrioritizedRawKeyedState() { - return prioritizedRawKeyedState.iterator(); + public List> getPrioritizedRawKeyedState() { + return prioritizedRawKeyedState; } // ----------------------------------------------------------------------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java index a940aefcc7f6d0..e7a7d8fe35c784 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java @@ -110,38 +110,68 @@ public TaskLocalStateStore localStateStoreForSubtask( "register a new TaskLocalStateStore."); } - final Map taskStateManagers = - this.taskStateStoresByAllocationID.computeIfAbsent(allocationID, k -> new HashMap<>()); + Map taskStateManagers = + this.taskStateStoresByAllocationID.get(allocationID); + + if (taskStateManagers == null) { + taskStateManagers = new HashMap<>(); + this.taskStateStoresByAllocationID.put(allocationID, taskStateManagers); + + if (LOG.isDebugEnabled()) { + LOG.debug("Registered new allocation id {} for local state stores for job {}.", + allocationID, jobId); + } + } final JobVertexSubtaskKey taskKey = new JobVertexSubtaskKey(jobVertexID, subtaskIndex); - // create the allocation base dirs, one inside each root dir. - File[] allocationBaseDirectories = allocationBaseDirectories(allocationID); + TaskLocalStateStoreImpl taskLocalStateStore = taskStateManagers.get(taskKey); - LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl( - allocationBaseDirectories, - jobId, - jobVertexID, - subtaskIndex); + if (taskLocalStateStore == null) { + + // create the allocation base dirs, one inside each root dir. + File[] allocationBaseDirectories = allocationBaseDirectories(allocationID); + + LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl( + allocationBaseDirectories, + jobId, + jobVertexID, + subtaskIndex); - LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig( - localRecoveryMode, - directoryProvider); + LocalRecoveryConfig localRecoveryConfig = + new LocalRecoveryConfig(localRecoveryMode, directoryProvider); - return taskStateManagers.computeIfAbsent( - taskKey, - k -> new TaskLocalStateStoreImpl( + taskLocalStateStore = new TaskLocalStateStoreImpl( jobId, allocationID, jobVertexID, subtaskIndex, localRecoveryConfig, - discardExecutor)); + discardExecutor); + + taskStateManagers.put(taskKey, taskLocalStateStore); + + if (LOG.isTraceEnabled()) { + LOG.trace("Registered new local state store with configuration {} for {} - {} - {} under allocation id {}.", + localRecoveryConfig, jobId, jobVertexID, subtaskIndex, allocationID); + } + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("Found existing local state store for {} - {} - {} under allocation id {}.", + jobId, jobVertexID, subtaskIndex, allocationID); + } + } + + return taskLocalStateStore; } } public void releaseLocalStateForAllocationId(@Nonnull AllocationID allocationID) { + if (LOG.isDebugEnabled()) { + LOG.debug("Releasing local state under allocation id {}.", allocationID); + } + Map cleanupLocalStores; synchronized (lock) { @@ -175,7 +205,7 @@ public void shutdown() { ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG); - LOG.debug("Shutting down TaskExecutorLocalStateStoresManager."); + LOG.info("Shutting down TaskExecutorLocalStateStoresManager."); for (Map.Entry> entry : toRelease.entrySet()) { @@ -217,7 +247,7 @@ private void doRelease(Iterable toRelease) { try { stateStore.dispose(); } catch (Exception disposeEx) { - LOG.warn("Exception while disposing local state store " + stateStore, disposeEx); + LOG.warn("Exception while disposing local state store {}.", stateStore, disposeEx); } } } @@ -233,7 +263,7 @@ private void cleanupAllocationBaseDirs(AllocationID allocationID) { try { FileUtils.deleteFileOrDirectory(directory); } catch (IOException e) { - LOG.warn("Exception while deleting local state directory for allocation " + allocationID, e); + LOG.warn("Exception while deleting local state directory for allocation id {}.", allocationID, e); } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java index 191c1096aa4d55..bb4f0116dffe5b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java @@ -103,15 +103,15 @@ public TaskLocalStateStoreImpl( @Nonnull LocalRecoveryConfig localRecoveryConfig, @Nonnull Executor discardExecutor) { + this.lock = new Object(); + this.storedTaskStateByCheckpointID = new TreeMap<>(); this.jobID = jobID; this.allocationID = allocationID; this.jobVertexID = jobVertexID; this.subtaskIndex = subtaskIndex; this.discardExecutor = discardExecutor; - this.lock = new Object(); - this.storedTaskStateByCheckpointID = new TreeMap<>(); - this.disposed = false; this.localRecoveryConfig = localRecoveryConfig; + this.disposed = false; } @Override @@ -123,8 +123,15 @@ public void storeLocalState( localState = NULL_DUMMY; } - LOG.info("Storing local state for checkpoint {}.", checkpointId); - LOG.debug("Local state for checkpoint {} is {}.", checkpointId, localState); + if (LOG.isTraceEnabled()) { + LOG.debug( + "Stored local state for checkpoint {} in subtask ({} - {} - {}) : {}.", + checkpointId, jobID, jobVertexID, subtaskIndex, localState); + } else if (LOG.isDebugEnabled()) { + LOG.debug( + "Stored local state for checkpoint {} in subtask ({} - {} - {})", + checkpointId, jobID, jobVertexID, subtaskIndex); + } Map toDiscard = new HashMap<>(16); @@ -148,10 +155,21 @@ public void storeLocalState( @Override @Nullable public TaskStateSnapshot retrieveLocalState(long checkpointID) { + + TaskStateSnapshot snapshot; synchronized (lock) { - TaskStateSnapshot snapshot = storedTaskStateByCheckpointID.get(checkpointID); - return snapshot != NULL_DUMMY ? snapshot : null; + snapshot = storedTaskStateByCheckpointID.get(checkpointID); } + + if (LOG.isTraceEnabled()) { + LOG.trace("Found entry for local state for checkpoint {} in subtask ({} - {} - {}) : {}", + checkpointID, jobID, jobVertexID, subtaskIndex, snapshot); + } else if (LOG.isDebugEnabled()) { + LOG.debug("Found entry for local state for checkpoint {} in subtask ({} - {} - {})", + checkpointID, jobID, jobVertexID, subtaskIndex); + } + + return snapshot != NULL_DUMMY ? snapshot : null; } @Override @@ -163,7 +181,8 @@ public LocalRecoveryConfig getLocalRecoveryConfig() { @Override public void confirmCheckpoint(long confirmedCheckpointId) { - LOG.debug("Received confirmation for checkpoint {}. Starting to prune history.", confirmedCheckpointId); + LOG.debug("Received confirmation for checkpoint {} in subtask ({} - {} - {}). Starting to prune history.", + confirmedCheckpointId, jobID, jobVertexID, subtaskIndex); final List> toRemove = new ArrayList<>(); @@ -216,7 +235,8 @@ public CompletableFuture dispose() { try { deleteDirectory(subtaskBaseDirectory); } catch (IOException e) { - LOG.warn("Exception when deleting local recovery subtask base dir: " + subtaskBaseDirectory, e); + LOG.warn("Exception when deleting local recovery subtask base directory {} in subtask ({} - {} - {})", + subtaskBaseDirectory, jobID, jobVertexID, subtaskIndex, e); } } }, @@ -240,27 +260,32 @@ private void syncDiscardLocalStateForCollection(CollectionReported state is tagged by clients so that this class can properly forward to the right receiver for the * checkpointed state. - * - * TODO: all interaction with local state store must still be implemented! It is currently just a placeholder. */ public class TaskStateManagerImpl implements TaskStateManager { + /** The logger for this class. */ + private static final Logger LOG = LoggerFactory.getLogger(TaskStateManagerImpl.class); + /** The id of the job for which this manager was created, can report, and recover. */ private final JobID jobId; @@ -117,21 +122,27 @@ public PrioritizedOperatorSubtaskState prioritizedOperatorState(OperatorID opera TaskStateSnapshot localStateSnapshot = localStateStore.retrieveLocalState(jobManagerTaskRestore.getRestoreCheckpointId()); + List alternativesByPriority = Collections.emptyList(); + if (localStateSnapshot != null) { OperatorSubtaskState localSubtaskState = localStateSnapshot.getSubtaskStateByOperatorID(operatorID); if (localSubtaskState != null) { - PrioritizedOperatorSubtaskState.Builder builder = new PrioritizedOperatorSubtaskState.Builder( - jobManagerSubtaskState, - Collections.singletonList(localSubtaskState)); - return builder.build(); + alternativesByPriority = Collections.singletonList(localSubtaskState); } } + if (LOG.isTraceEnabled()) { + LOG.trace("Operator {} has remote state {} from job manager and local state alternatives {} from local " + + "state store {}.", + operatorID, jobManagerSubtaskState, alternativesByPriority, localStateStore); + } + PrioritizedOperatorSubtaskState.Builder builder = new PrioritizedOperatorSubtaskState.Builder( jobManagerSubtaskState, - Collections.emptyList(), + alternativesByPriority, true); + return builder.build(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java index 09c9efb6959940..82082e0f8f0321 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java @@ -216,7 +216,7 @@ private OperatorSubtaskState createAlternativeSubtaskState(OperatorSubtaskState private boolean checkResultAsExpected( Function> extractor, - Function>> extractor2, + Function>> extractor2, PrioritizedOperatorSubtaskState prioritizedResult, OperatorSubtaskState... expectedOrdered) { @@ -226,7 +226,7 @@ private boolean checkResultAsExpected( } return checkRepresentSameOrder( - extractor2.apply(prioritizedResult), + extractor2.apply(prioritizedResult).iterator(), collector.toArray(new StateObjectCollection[collector.size()])); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java index 926c1961c6652e..f58f3f442ab258 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java @@ -142,7 +142,7 @@ public void testStateReportingAndRetrieving() { // checks for operator 1. Iterator> prioritizedManagedKeyedState_1 = - prioritized_1.getPrioritizedManagedKeyedState(); + prioritized_1.getPrioritizedManagedKeyedState().iterator(); Assert.assertTrue(prioritizedManagedKeyedState_1.hasNext()); StateObjectCollection current = prioritizedManagedKeyedState_1.next(); @@ -158,7 +158,7 @@ public void testStateReportingAndRetrieving() { // checks for operator 2. Iterator> prioritizedRawKeyedState_2 = - prioritized_2.getPrioritizedRawKeyedState(); + prioritized_2.getPrioritizedRawKeyedState().iterator(); Assert.assertTrue(prioritizedRawKeyedState_2.hasNext()); current = prioritizedRawKeyedState_2.next(); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java index ba27a0a27dd4d0..dd75fb298ca0c5 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; +import java.util.List; /** * This class implements the logic that creates (and potentially restores) a state backend. The restore logic @@ -62,6 +62,9 @@ public class BackendRestorerProcedure< /** This registry is used so that recovery can participate in the task lifecycle, i.e. can be canceled. */ private final CloseableRegistry backendCloseableRegistry; + /** Description of this instance for logging. */ + private final String logDescription; + /** * Creates a new backend restorer using the given backend supplier and the closeable registry. * @@ -70,43 +73,63 @@ public class BackendRestorerProcedure< */ public BackendRestorerProcedure( @Nonnull SupplierWithException instanceSupplier, - @Nonnull CloseableRegistry backendCloseableRegistry) { + @Nonnull CloseableRegistry backendCloseableRegistry, + @Nonnull String logDescription) { this.instanceSupplier = Preconditions.checkNotNull(instanceSupplier); this.backendCloseableRegistry = Preconditions.checkNotNull(backendCloseableRegistry); + this.logDescription = logDescription; } /** * Creates a new state backend and restores it from the provided set of state snapshot alternatives. * - * @param restoreOptions iterator over a prioritized set of state snapshot alternatives for recovery. + * @param restoreOptions list of prioritized state snapshot alternatives for recovery. * @return the created (and restored) state backend. * @throws Exception if the backend could not be created or restored. */ - public @Nonnull - T createAndRestore(@Nonnull Iterator> restoreOptions) throws Exception { + @Nonnull + public T createAndRestore(@Nonnull List> restoreOptions) throws Exception { + + if (restoreOptions.isEmpty()) { + restoreOptions = Collections.singletonList(Collections.emptyList()); + } + + int alternativeIdx = 0; + + Exception collectedException = null; + + while (alternativeIdx < restoreOptions.size()) { + + Collection restoreState = restoreOptions.get(alternativeIdx); - // This ensures that we always call the restore method even if there is no previous state - // (required by some backends). - Collection attemptState = restoreOptions.hasNext() ? - restoreOptions.next() : - Collections.emptyList(); + ++alternativeIdx; + + if (restoreState.isEmpty()) { + LOG.debug("Creating {} with empty state.", logDescription); + } else { + if (LOG.isTraceEnabled()) { + LOG.trace("Creating {} and restoring with state {} from alternative ({}/{}).", + logDescription, restoreState, alternativeIdx, restoreOptions.size()); + } else { + LOG.debug("Creating {} and restoring with state from alternative ({}/{}).", + logDescription, alternativeIdx, restoreOptions.size()); + } + } - while (true) { try { - return attemptCreateAndRestore(attemptState); + return attemptCreateAndRestore(restoreState); } catch (Exception ex) { - // more attempts? - if (restoreOptions.hasNext()) { - attemptState = restoreOptions.next(); - LOG.warn("Exception while restoring backend, will retry with another snapshot replica.", ex); - } else { + collectedException = ExceptionUtils.firstOrSuppressed(ex, collectedException); - throw new FlinkException("Could not restore from any of the provided restore options.", ex); - } + LOG.warn("Exception while restoring {} from alternative ({}/{}), will retry while more " + + "alternatives are available.", logDescription, alternativeIdx, restoreOptions.size(), ex); } } + + throw new FlinkException("Could not restore " + logDescription + " from any of the " + restoreOptions.size() + + " provided restore options.", collectedException); } private T attemptCreateAndRestore(Collection restoreState) throws Exception { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java index 11e2dda82ea729..acbc2f8bc9ed5d 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java @@ -143,11 +143,11 @@ public StreamOperatorStateContext streamOperatorStateContext( // -------------- Raw State Streams -------------- rawKeyedStateInputs = rawKeyedStateInputs( - prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState()); + prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState().iterator()); streamTaskCloseableRegistry.registerCloseable(rawKeyedStateInputs); rawOperatorStateInputs = rawOperatorStateInputs( - prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState()); + prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState().iterator()); streamTaskCloseableRegistry.registerCloseable(rawOperatorStateInputs); // -------------- Internal Timer Service Manager -------------- @@ -226,12 +226,16 @@ protected OperatorStateBackend operatorStateBackend( PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates, CloseableRegistry backendCloseableRegistry) throws Exception { + String logDescription = "operator state backend for " + operatorIdentifierText; + BackendRestorerProcedure backendRestorer = new BackendRestorerProcedure<>( () -> stateBackend.createOperatorStateBackend(environment, operatorIdentifierText), - backendCloseableRegistry); + backendCloseableRegistry, + logDescription); - return backendRestorer.createAndRestore(prioritizedOperatorSubtaskStates.getPrioritizedManagedOperatorState()); + return backendRestorer.createAndRestore( + prioritizedOperatorSubtaskStates.getPrioritizedManagedOperatorState()); } protected AbstractKeyedStateBackend keyedStatedBackend( @@ -244,6 +248,8 @@ protected AbstractKeyedStateBackend keyedStatedBackend( return null; } + String logDescription = "keyed state backend for " + operatorIdentifierText; + TaskInfo taskInfo = environment.getTaskInfo(); final KeyGroupRange keyGroupRange = KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex( @@ -261,9 +267,11 @@ protected AbstractKeyedStateBackend keyedStatedBackend( taskInfo.getMaxNumberOfParallelSubtasks(), keyGroupRange, environment.getTaskKvStateRegistry()), - backendCloseableRegistry); + backendCloseableRegistry, + logDescription); - return backendRestorer.createAndRestore(prioritizedOperatorSubtaskStates.getPrioritizedManagedKeyedState()); + return backendRestorer.createAndRestore( + prioritizedOperatorSubtaskStates.getPrioritizedManagedKeyedState()); } protected CloseableIterable rawOperatorStateInputs( diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java index 2126f707a621e5..0f15d110280a9f 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java @@ -104,21 +104,17 @@ public void testRestoreProcedureOrderAndFailure() throws Exception { new StateObjectCollection<>(Collections.singletonList(firstFailHandle)), new StateObjectCollection<>(Collections.singletonList(secondSuccessHandle)), new StateObjectCollection<>(Collections.singletonList(thirdNotUsedHandle))); - Iterator> iterator = sortedRestoreOptions.iterator(); BackendRestorerProcedure restorerProcedure = - new BackendRestorerProcedure<>(backendSupplier, closeableRegistry); + new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend"); - OperatorStateBackend restoredBackend = restorerProcedure.createAndRestore(iterator); + OperatorStateBackend restoredBackend = restorerProcedure.createAndRestore(sortedRestoreOptions); Assert.assertNotNull(restoredBackend); try { - Assert.assertTrue(iterator.hasNext()); - Assert.assertTrue(thirdNotUsedHandle == iterator.next().iterator().next()); verify(firstFailHandle).openInputStream(); verify(secondSuccessHandle).openInputStream(); verifyZeroInteractions(thirdNotUsedHandle); - Assert.assertFalse(iterator.hasNext()); ListState listState = restoredBackend.getListState(stateDescriptor); @@ -151,13 +147,12 @@ public void testExceptionThrownIfAllRestoresFailed() throws Exception { new StateObjectCollection<>(Collections.singletonList(firstFailHandle)), new StateObjectCollection<>(Collections.singletonList(secondFailHandle)), new StateObjectCollection<>(Collections.singletonList(thirdFailHandle))); - Iterator> iterator = sortedRestoreOptions.iterator(); BackendRestorerProcedure restorerProcedure = - new BackendRestorerProcedure<>(backendSupplier, closeableRegistry); + new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend"); try { - restorerProcedure.createAndRestore(iterator); + restorerProcedure.createAndRestore(sortedRestoreOptions); Assert.fail(); } catch (Exception ignore) { } @@ -165,7 +160,6 @@ public void testExceptionThrownIfAllRestoresFailed() throws Exception { verify(firstFailHandle).openInputStream(); verify(secondFailHandle).openInputStream(); verify(thirdFailHandle).openInputStream(); - Assert.assertFalse(iterator.hasNext()); } /** @@ -183,12 +177,12 @@ public void testCanBeCanceledViaRegistry() throws Exception { Collections.singletonList(new StateObjectCollection<>(Collections.singletonList(blockingRestoreHandle))); BackendRestorerProcedure restorerProcedure = - new BackendRestorerProcedure<>(backendSupplier, closeableRegistry); + new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend"); AtomicReference exceptionReference = new AtomicReference<>(null); Thread restoreThread = new Thread(() -> { try { - restorerProcedure.createAndRestore(sortedRestoreOptions.iterator()); + restorerProcedure.createAndRestore(sortedRestoreOptions); } catch (Exception e) { exceptionReference.set(e); } From 70de6da4660f09fe7e71afc7b05da15ee85da5ae Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 27 Feb 2018 16:53:03 +0100 Subject: [PATCH 0042/2294] [hotfix] [core] Suppress unused warning config options only used in shell scripts and doc generation. --- .../apache/flink/configuration/CoreOptions.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java index ccce0aba8db2dd..dc544e0b433d1c 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java @@ -137,17 +137,32 @@ public static String[] getParentFirstLoaderPatterns(Configuration config) { .key("env.java.opts.taskmanager") .defaultValue(""); + /** + * This options is here only for documentation generation, it is only + * evaluated in the shell scripts. + */ + @SuppressWarnings("unused") public static final ConfigOption FLINK_LOG_DIR = ConfigOptions .key("env.log.dir") .noDefaultValue() .withDescription("Defines the directory where the Flink logs are saved. It has to be an absolute path." + " (Defaults to the log directory under Flink’s home)"); + /** + * This options is here only for documentation generation, it is only + * evaluated in the shell scripts. + */ + @SuppressWarnings("unused") public static final ConfigOption FLINK_LOG_MAX = ConfigOptions .key("env.log.max") .defaultValue(5) .withDescription("The maximum number of old log files to keep."); + /** + * This options is here only for documentation generation, it is only + * evaluated in the shell scripts. + */ + @SuppressWarnings("unused") public static final ConfigOption FLINK_SSH_OPTIONS = ConfigOptions .key("env.ssh.opts") .noDefaultValue() From 59fb56bc8378645b82ee31e1b3bd07e5045a3698 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 27 Feb 2018 17:04:29 +0100 Subject: [PATCH 0043/2294] [FLINK-8798] [core] Make force 'commons-logging' to be parent-first loaded. --- .../main/java/org/apache/flink/configuration/CoreOptions.java | 4 ++-- .../apache/flink/configuration/ParentFirstPatternsTest.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java index dc544e0b433d1c..8ac729d94f7f47 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java @@ -80,14 +80,14 @@ public class CoreOptions { * and formats (flink-avro, etc) are loaded parent-first as well if they are in the * core classpath. *

  • Java annotations and loggers, defined by the following list: - * javax.annotation;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback. + * javax.annotation;org.slf4j;org.apache.log4j;org.apache.logging;org.apache.commons.logging;ch.qos.logback. * This is done for convenience, to avoid duplication of annotations and multiple * log bindings.
    • * */ public static final ConfigOption ALWAYS_PARENT_FIRST_LOADER_PATTERNS = ConfigOptions .key("classloader.parent-first-patterns.default") - .defaultValue("java.;scala.;org.apache.flink.;com.esotericsoftware.kryo;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback") + .defaultValue("java.;scala.;org.apache.flink.;com.esotericsoftware.kryo;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging;org.apache.commons.logging;ch.qos.logback") .withDeprecatedKeys("classloader.parent-first-patterns") .withDescription("A (semicolon-separated) list of patterns that specifies which classes should always be" + " resolved through the parent ClassLoader first. A pattern is a simple prefix that is checked against" + diff --git a/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java b/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java index ca4b511f456367..b373a6dde2a976 100644 --- a/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java +++ b/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java @@ -54,7 +54,8 @@ public void testAllCorePatterns() { public void testLoggersParentFirst() { assertTrue(PARENT_FIRST_PACKAGES.contains("org.slf4j")); assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.log4j")); - assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.logging.log4j")); + assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.logging")); + assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.commons.logging")); assertTrue(PARENT_FIRST_PACKAGES.contains("ch.qos.logback")); } From 2450d2b24006a4db846b9b688f9b598e3fdf7c6e Mon Sep 17 00:00:00 2001 From: Xingcan Cui Date: Mon, 12 Feb 2018 18:11:36 +0800 Subject: [PATCH 0044/2294] [FLINK-8538][table]Add a Kafka table source factory with JSON format support --- .../kafka/Kafka010JsonTableSourceFactory.java | 36 +++ ...che.flink.table.sources.TableSourceFactory | 16 ++ .../resources/tableSourceConverter.properties | 29 +++ .../kafka/Kafka010TableSourceFactoryTest.java | 41 ++++ .../kafka/Kafka011JsonTableSourceFactory.java | 36 +++ ...che.flink.table.sources.TableSourceFactory | 16 ++ .../resources/tableSourceConverter.properties | 29 +++ .../kafka/Kafka011TableSourceFactoryTest.java | 41 ++++ .../kafka/Kafka08JsonTableSourceFactory.java | 36 +++ ...che.flink.table.sources.TableSourceFactory | 16 ++ .../resources/tableSourceConverter.properties | 29 +++ .../kafka/Kafka08TableSourceFactoryTest.java | 42 ++++ .../kafka/Kafka09JsonTableSourceFactory.java | 36 +++ ...che.flink.table.sources.TableSourceFactory | 16 ++ .../resources/tableSourceConverter.properties | 29 +++ .../kafka/Kafka09TableSourceFactoryTest.java | 41 ++++ .../flink-connector-kafka-base/pom.xml | 14 ++ .../kafka/KafkaJsonTableSource.java | 17 ++ .../kafka/KafkaJsonTableSourceFactory.java | 227 ++++++++++++++++++ .../connectors/kafka/KafkaTableSource.java | 30 +++ .../apache/flink/table/descriptors/Kafka.java | 199 +++++++++++++++ .../table/descriptors/KafkaValidator.java | 193 +++++++++++++++ .../KafkaJsonTableFromDescriptorTestBase.java | 127 ++++++++++ .../src/test/resources/kafka-json-schema.json | 35 +++ .../apache/flink/table/api/TableSchema.scala | 5 + 25 files changed, 1336 insertions(+) create mode 100644 flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java create mode 100644 flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory create mode 100644 flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties create mode 100644 flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java create mode 100644 flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java create mode 100644 flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory create mode 100644 flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties create mode 100644 flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java create mode 100644 flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java create mode 100644 flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory create mode 100644 flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties create mode 100644 flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java create mode 100644 flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java create mode 100644 flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory create mode 100644 flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties create mode 100644 flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java create mode 100644 flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java create mode 100644 flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java create mode 100644 flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java new file mode 100644 index 00000000000000..1d03f6c4e8c031 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010; + +/** + * Factory for creating configured instances of {@link Kafka010JsonTableSource}. + */ +public class Kafka010JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override + protected KafkaJsonTableSource.Builder createBuilder() { + return new Kafka010JsonTableSource.Builder(); + } + + @Override + protected String kafkaVersion() { + return KAFKA_VERSION_VALUE_010; + } +} diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory new file mode 100644 index 00000000000000..9ef54fcb045fa9 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.streaming.connectors.kafka.Kafka010JsonTableSourceFactory diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties new file mode 100644 index 00000000000000..5409b49703088b --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +################################################################################ +# The config file is used to specify the packages of current module where +# to find TableSourceConverter implementation class annotated with TableType. +# If there are multiple packages to scan, put those packages together into a +# string separated with ',', for example, org.package1,org.package2. +# Please notice: +# It's better to have a tableSourceConverter.properties in each connector Module +# which offers converters instead of put all information into the +# tableSourceConverter.properties of flink-table module. +################################################################################ +scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java new file mode 100644 index 00000000000000..15b89e8e2f85a4 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import org.apache.flink.table.descriptors.Kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010; + +/** + * Tests for {@link Kafka010JsonTableSourceFactory}. + */ +public class Kafka010TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { + protected String versionForTest() { + return KAFKA_VERSION_VALUE_010; + } + + protected KafkaJsonTableSource.Builder builderForTest() { + return Kafka010JsonTableSource.builder(); + } + + @Override + protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { + // no extra settings + } +} diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java new file mode 100644 index 00000000000000..ca4d6ce01374d3 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011; + +/** + * Factory for creating configured instances of {@link Kafka011JsonTableSource}. + */ +public class Kafka011JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override + protected KafkaJsonTableSource.Builder createBuilder() { + return new Kafka011JsonTableSource.Builder(); + } + + @Override + protected String kafkaVersion() { + return KAFKA_VERSION_VALUE_011; + } +} diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory new file mode 100644 index 00000000000000..75135e57d746cc --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.streaming.connectors.kafka.Kafka011JsonTableSourceFactory diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties new file mode 100644 index 00000000000000..5409b49703088b --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +################################################################################ +# The config file is used to specify the packages of current module where +# to find TableSourceConverter implementation class annotated with TableType. +# If there are multiple packages to scan, put those packages together into a +# string separated with ',', for example, org.package1,org.package2. +# Please notice: +# It's better to have a tableSourceConverter.properties in each connector Module +# which offers converters instead of put all information into the +# tableSourceConverter.properties of flink-table module. +################################################################################ +scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java new file mode 100644 index 00000000000000..84ac39b6f47e8b --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import org.apache.flink.table.descriptors.Kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011; + +/** + * Tests for {@link Kafka011JsonTableSourceFactory}. + */ +public class Kafka011TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { + protected String versionForTest() { + return KAFKA_VERSION_VALUE_011; + } + + protected KafkaJsonTableSource.Builder builderForTest() { + return Kafka011JsonTableSource.builder(); + } + + @Override + protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { + // no extra settings + } +} diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java new file mode 100644 index 00000000000000..e4e50960b627c5 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08; + +/** + * Factory for creating configured instances of {@link Kafka08JsonTableSource}. + */ +public class Kafka08JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override + protected KafkaJsonTableSource.Builder createBuilder() { + return new Kafka08JsonTableSource.Builder(); + } + + @Override + protected String kafkaVersion() { + return KAFKA_VERSION_VALUE_08; + } +} diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory new file mode 100644 index 00000000000000..9092955842e7a4 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.streaming.connectors.kafka.Kafka08JsonTableSourceFactory diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties new file mode 100644 index 00000000000000..5409b49703088b --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +################################################################################ +# The config file is used to specify the packages of current module where +# to find TableSourceConverter implementation class annotated with TableType. +# If there are multiple packages to scan, put those packages together into a +# string separated with ',', for example, org.package1,org.package2. +# Please notice: +# It's better to have a tableSourceConverter.properties in each connector Module +# which offers converters instead of put all information into the +# tableSourceConverter.properties of flink-table module. +################################################################################ +scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java new file mode 100644 index 00000000000000..a2edc09ec125f9 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import org.apache.flink.table.descriptors.Kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08; + +/** + * Tests for {@link Kafka08JsonTableSourceFactory}. + */ +public class Kafka08TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { + protected String versionForTest() { + return KAFKA_VERSION_VALUE_08; + } + + protected KafkaJsonTableSource.Builder builderForTest() { + return Kafka08JsonTableSource.builder(); + } + + @Override + protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { + builder.getKafkaProps().put("zookeeper.connect", "localhost:1111"); + kafka.zookeeperConnect("localhost:1111"); + } +} diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java new file mode 100644 index 00000000000000..bbda4ae66d6367 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09; + +/** + * Factory for creating configured instances of {@link Kafka09JsonTableSource}. + */ +public class Kafka09JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override + protected KafkaJsonTableSource.Builder createBuilder() { + return new Kafka09JsonTableSource.Builder(); + } + + @Override + protected String kafkaVersion() { + return KAFKA_VERSION_VALUE_09; + } +} diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory new file mode 100644 index 00000000000000..2f38bd0e9746df --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.streaming.connectors.kafka.Kafka09JsonTableSourceFactory diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties new file mode 100644 index 00000000000000..5409b49703088b --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +################################################################################ +# The config file is used to specify the packages of current module where +# to find TableSourceConverter implementation class annotated with TableType. +# If there are multiple packages to scan, put those packages together into a +# string separated with ',', for example, org.package1,org.package2. +# Please notice: +# It's better to have a tableSourceConverter.properties in each connector Module +# which offers converters instead of put all information into the +# tableSourceConverter.properties of flink-table module. +################################################################################ +scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java new file mode 100644 index 00000000000000..fc85ea77a07cd6 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import org.apache.flink.table.descriptors.Kafka; + +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09; + +/** + * Factory for creating configured instances of {@link Kafka09JsonTableSource}. + */ +public class Kafka09TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { + protected String versionForTest() { + return KAFKA_VERSION_VALUE_09; + } + + protected KafkaJsonTableSource.Builder builderForTest() { + return Kafka09JsonTableSource.builder(); + } + + @Override + protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { + // no extra settings + } +} diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml index 4620b8fca00f6b..212a86b9746cb5 100644 --- a/flink-connectors/flink-connector-kafka-base/pom.xml +++ b/flink-connectors/flink-connector-kafka-base/pom.xml @@ -205,6 +205,20 @@ under the License. test + + org.apache.flink + flink-scala_${scala.binary.version} + ${project.version} + test + + + + org.apache.flink + flink-streaming-scala_${scala.binary.version} + ${project.version} + test + +     diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java index f581e89830bf62..d2dafe72b34951 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java @@ -27,6 +27,7 @@ import org.apache.flink.table.sources.StreamTableSource; import java.util.Map; +import java.util.Objects; import java.util.Properties; /** @@ -86,6 +87,22 @@ public String explainSource() { return "KafkaJSONTableSource"; } + @Override + public boolean equals(Object other) { + if (super.equals(other)) { + KafkaJsonTableSource otherSource = (KafkaJsonTableSource) other; + return Objects.equals(failOnMissingField, otherSource.failOnMissingField) + && Objects.equals(jsonSchema, otherSource.jsonSchema) + && Objects.equals(fieldMapping, otherSource.fieldMapping); + } + return false; + } + + @Override + public int hashCode() { + return 31 * super.hashCode() + Objects.hash(failOnMissingField, jsonSchema, fieldMapping); + } + //////// SETTERS FOR OPTIONAL PARAMETERS /** diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java new file mode 100644 index 00000000000000..918b83357ff569 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.formats.json.JsonSchemaConverter; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.descriptors.DescriptorProperties; +import org.apache.flink.table.descriptors.JsonValidator; +import org.apache.flink.table.descriptors.KafkaValidator; +import org.apache.flink.table.descriptors.SchemaValidator; +import org.apache.flink.table.sources.TableSource; +import org.apache.flink.table.sources.TableSourceFactory; +import org.apache.flink.types.Row; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE; +import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION; +import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_TYPE; +import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_VERSION; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA_STRING; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_TYPE_VALUE; +import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE; +import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID; +import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD; +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION; +import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET; +import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION; +import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS; +import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE; +import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_EARLIEST; +import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_GROUP_OFFSETS; +import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_LATEST; +import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_SPECIFIC_OFFSETS; +import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD; +import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING; +import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC; +import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT; +import static org.apache.flink.table.descriptors.SchemaValidator.PROCTIME; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_VERSION; + +import scala.Option; +import scala.collection.JavaConversions; + +/** + * Factory for creating configured instances of {@link KafkaJsonTableSource}. + */ +public abstract class KafkaJsonTableSourceFactory implements TableSourceFactory { + @Override + public Map requiredContext() { + Map context = new HashMap<>(); + context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE); // kafka connector + context.put(FORMAT_TYPE(), FORMAT_TYPE_VALUE()); // Json format + context.put(KAFKA_VERSION, kafkaVersion()); // for different implementations + context.put(CONNECTOR_VERSION(), "1"); + context.put(FORMAT_VERSION(), "1"); + context.put(SCHEMA_VERSION(), "1"); + return context; + } + + @Override + public List supportedProperties() { + List properties = new ArrayList<>(); + + // kafka + properties.add(KAFKA_VERSION); + properties.add(BOOTSTRAP_SERVERS); + properties.add(GROUP_ID); + properties.add(ZOOKEEPER_CONNECT); + properties.add(TOPIC); + properties.add(STARTUP_MODE); + properties.add(SPECIFIC_OFFSETS + ".#." + PARTITION); + properties.add(SPECIFIC_OFFSETS + ".#." + OFFSET); + + // json format + properties.add(FORMAT_SCHEMA_STRING()); + properties.add(FORMAT_FAIL_ON_MISSING_FIELD()); + + // table json mapping + properties.add(TABLE_JSON_MAPPING + ".#." + TABLE_FIELD); + properties.add(TABLE_JSON_MAPPING + ".#." + JSON_FIELD); + + // schema + properties.add(SCHEMA() + ".#." + DescriptorProperties.TYPE()); + properties.add(SCHEMA() + ".#." + DescriptorProperties.NAME()); + + // time attributes + properties.add(SCHEMA() + ".#." + PROCTIME()); +// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_CLASS()); +// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_TYPE()); + + return properties; + } + + @Override + public TableSource create(Map properties) { + DescriptorProperties params = new DescriptorProperties(true); + params.putProperties(properties); + + // validate + new KafkaValidator().validate(params); + new JsonValidator().validate(params); + new SchemaValidator(true).validate(params); + + // build + KafkaJsonTableSource.Builder builder = createBuilder(); + Properties kafkaProps = new Properties(); + + // Set the required parameters. + String topic = params.getString(TOPIC).get(); + TableSchema tableSchema = params.getTableSchema(SCHEMA()).get(); + + kafkaProps.put(BOOTSTRAP_SERVERS, params.getString(BOOTSTRAP_SERVERS).get()); + kafkaProps.put(GROUP_ID, params.getString(GROUP_ID).get()); + + // Set the zookeeper connect for kafka 0.8. + Option zkConnect = params.getString(ZOOKEEPER_CONNECT); + if (zkConnect.isDefined()) { + kafkaProps.put(ZOOKEEPER_CONNECT, zkConnect.get()); + } + + builder.withKafkaProperties(kafkaProps).forTopic(topic).withSchema(tableSchema); + + // Set the startup mode. + String startupMode = params.getString(STARTUP_MODE).get(); + if (null != startupMode) { + switch (startupMode) { + case STARTUP_MODE_VALUE_EARLIEST: + builder.fromEarliest(); + break; + case STARTUP_MODE_VALUE_LATEST: + builder.fromLatest(); + break; + case STARTUP_MODE_VALUE_GROUP_OFFSETS: + builder.fromGroupOffsets(); + break; + case STARTUP_MODE_VALUE_SPECIFIC_OFFSETS: + Map partitions = JavaConversions. + mapAsJavaMap(params.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION)); + Map offsetMap = new HashMap<>(); + for (int i = 0; i < partitions.size(); i++) { + offsetMap.put( + new KafkaTopicPartition( + topic, + Integer.valueOf(params.getString( + SPECIFIC_OFFSETS + "" + "." + i + "." + PARTITION).get())), + Long.valueOf(params.getString( + SPECIFIC_OFFSETS + "" + "." + i + "." + OFFSET).get())); + } + builder.fromSpecificOffsets(offsetMap); + break; + } + } + + // Set whether fail on missing JSON field. + Option failOnMissing = params.getString(FORMAT_FAIL_ON_MISSING_FIELD()); + if (failOnMissing.isDefined()) { + builder.failOnMissingField(Boolean.valueOf(failOnMissing.get())); + } + + // Set the JSON schema. + Option jsonSchema = params.getString(FORMAT_SCHEMA_STRING()); + if (jsonSchema.isDefined()) { + TypeInformation jsonSchemaType = JsonSchemaConverter.convert(jsonSchema.get()); + builder.forJsonSchema(TableSchema.fromTypeInfo(jsonSchemaType)); + } + + // Set the table => JSON fields mapping. + Map mappingTableFields = JavaConversions. + mapAsJavaMap(params.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD)); + + if (!mappingTableFields.isEmpty()) { + Map tableJsonMapping = new HashMap<>(); + for (int i = 0; i < mappingTableFields.size(); i++) { + tableJsonMapping.put(params.getString(TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD).get(), + params.getString(TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD).get() + ); + } + builder.withTableToJsonMapping(tableJsonMapping); + } + + // Set the time attributes. + setTimeAttributes(tableSchema, params, builder); + + return builder.build(); + } + + protected abstract KafkaJsonTableSource.Builder createBuilder(); + + protected abstract String kafkaVersion(); + + private void setTimeAttributes(TableSchema schema, DescriptorProperties params, KafkaJsonTableSource.Builder builder) { + // TODO to deal with rowtime fields + Option proctimeField; + for (int i = 0; i < schema.getColumnNum(); i++) { + proctimeField = params.getString(SCHEMA() + "." + i + "." + PROCTIME()); + if (proctimeField.isDefined()) { + builder.withProctimeAttribute(schema.getColumnName(i).get()); + } + } + } +} diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java index d5cda4ab8daa76..9ce3b8ed5a17c0 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import scala.Option; @@ -138,6 +139,35 @@ public String explainSource() { return TableConnectorUtil.generateRuntimeName(this.getClass(), schema.getColumnNames()); } + @Override + public boolean equals(Object o) { + if (!o.getClass().equals(this.getClass())) { + return false; + } + KafkaTableSource other = (KafkaTableSource) o; + return Objects.equals(topic, other.topic) + && Objects.equals(schema, other.schema) + && Objects.equals(properties, other.properties) + && Objects.equals(proctimeAttribute, other.proctimeAttribute) + && Objects.equals(returnType, other.returnType) + && Objects.equals(rowtimeAttributeDescriptors, other.rowtimeAttributeDescriptors) + && Objects.equals(specificStartupOffsets, other.specificStartupOffsets) + && Objects.equals(startupMode, other.startupMode); + } + + @Override + public int hashCode() { + return Objects.hash( + topic, + schema, + properties, + proctimeAttribute, + returnType, + rowtimeAttributeDescriptors, + specificStartupOffsets, + startupMode); + } + /** * Returns a version-specific Kafka consumer with the start position configured. * diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java new file mode 100644 index 00000000000000..4733f6e002db84 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors; + +import org.apache.flink.streaming.connectors.kafka.config.StartupMode; + +import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE; +import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID; +import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD; +import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION; +import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET; +import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION; +import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS; +import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD; +import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING; +import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC; +import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import scala.collection.JavaConversions; +import scala.collection.Seq; + +/** + * Connector descriptor for the kafka message queue. + */ +public class Kafka extends ConnectorDescriptor { + + private Optional version = Optional.empty(); + private Optional bootstrapServers = Optional.empty(); + private Optional groupId = Optional.empty(); + private Optional topic = Optional.empty(); + private Optional zookeeperConnect = Optional.empty(); + private Optional> tableJsonMapping = Optional.empty(); + + private Optional startupMode = Optional.empty(); + private Optional> specificOffsets = Optional.empty(); + + public Kafka() { + super(CONNECTOR_TYPE_VALUE, 1); + } + + /** + * Sets the kafka version. + * + * @param version + * Could be {@link KafkaValidator#KAFKA_VERSION_VALUE_011}, + * {@link KafkaValidator#KAFKA_VERSION_VALUE_010}, + * {@link KafkaValidator#KAFKA_VERSION_VALUE_09}, + * or {@link KafkaValidator#KAFKA_VERSION_VALUE_08}. + */ + public Kafka version(String version) { + this.version = Optional.of(version); + return this; + } + + /** + * Sets the bootstrap servers for kafka. + */ + public Kafka bootstrapServers(String bootstrapServers) { + this.bootstrapServers = Optional.of(bootstrapServers); + return this; + } + + /** + * Sets the consumer group id. + */ + public Kafka groupId(String groupId) { + this.groupId = Optional.of(groupId); + return this; + } + + /** + * Sets the topic to consume. + */ + public Kafka topic(String topic) { + this.topic = Optional.of(topic); + return this; + } + + /** + * Sets the startup mode. + */ + public Kafka startupMode(StartupMode startupMode) { + this.startupMode = Optional.of(startupMode); + return this; + } + + /** + * Sets the zookeeper hosts. Only required by kafka 0.8. + */ + public Kafka zookeeperConnect(String zookeeperConnect) { + this.zookeeperConnect = Optional.of(zookeeperConnect); + return this; + } + + /** + * Sets the consume offsets for the topic set with {@link Kafka#topic(String)}. + * Only works in {@link StartupMode#SPECIFIC_OFFSETS} mode. + */ + public Kafka specificOffsets(Map specificOffsets) { + this.specificOffsets = Optional.of(specificOffsets); + return this; + } + + /** + * Sets the mapping from logical table schema to json schema. + */ + public Kafka tableJsonMapping(Map jsonTableMapping) { + this.tableJsonMapping = Optional.of(jsonTableMapping); + return this; + } + + @Override + public void addConnectorProperties(DescriptorProperties properties) { + if (version.isPresent()) { + properties.putString(KAFKA_VERSION, version.get()); + } + if (bootstrapServers.isPresent()) { + properties.putString(BOOTSTRAP_SERVERS, bootstrapServers.get()); + } + if (groupId.isPresent()) { + properties.putString(GROUP_ID, groupId.get()); + } + if (topic.isPresent()) { + properties.putString(TOPIC, topic.get()); + } + if (zookeeperConnect.isPresent()) { + properties.putString(ZOOKEEPER_CONNECT, zookeeperConnect.get()); + } + if (startupMode.isPresent()) { + Map map = KafkaValidator.normalizeStartupMode(startupMode.get()); + for (Map.Entry entry : map.entrySet()) { + properties.putString(entry.getKey(), entry.getValue()); + } + } + if (specificOffsets.isPresent()) { + List propertyKeys = new ArrayList<>(); + propertyKeys.add(PARTITION); + propertyKeys.add(OFFSET); + + List> propertyValues = new ArrayList<>(specificOffsets.get().size()); + for (Map.Entry entry : specificOffsets.get().entrySet()) { + List partitionOffset = new ArrayList<>(2); + partitionOffset.add(entry.getKey().toString()); + partitionOffset.add(entry.getValue().toString()); + propertyValues.add(JavaConversions.asScalaBuffer(partitionOffset).toSeq()); + } + properties.putIndexedFixedProperties( + SPECIFIC_OFFSETS, + JavaConversions.asScalaBuffer(propertyKeys).toSeq(), + JavaConversions.asScalaBuffer(propertyValues).toSeq() + ); + } + if (tableJsonMapping.isPresent()) { + List propertyKeys = new ArrayList<>(); + propertyKeys.add(TABLE_FIELD); + propertyKeys.add(JSON_FIELD); + + List> mappingFields = new ArrayList<>(tableJsonMapping.get().size()); + for (Map.Entry entry : tableJsonMapping.get().entrySet()) { + List singleMapping = new ArrayList<>(2); + singleMapping.add(entry.getKey()); + singleMapping.add(entry.getValue()); + mappingFields.add(JavaConversions.asScalaBuffer(singleMapping).toSeq()); + } + properties.putIndexedFixedProperties( + TABLE_JSON_MAPPING, + JavaConversions.asScalaBuffer(propertyKeys).toSeq(), + JavaConversions.asScalaBuffer(mappingFields).toSeq() + ); + } + } + + @Override + public boolean needsFormat() { + return true; + } +} diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java new file mode 100644 index 00000000000000..a3ca22f90b912e --- /dev/null +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors; + +import org.apache.flink.streaming.connectors.kafka.config.StartupMode; +import org.apache.flink.table.api.ValidationException; + +import java.util.HashMap; +import java.util.Map; + +import scala.Function0; +import scala.Tuple2; +import scala.collection.JavaConversions; +import scala.runtime.AbstractFunction0; +import scala.runtime.BoxedUnit; + + +/** + * The validator for {@link Kafka}. + */ +public class KafkaValidator extends ConnectorDescriptorValidator { + // fields + public static final String CONNECTOR_TYPE_VALUE = "kafka"; + public static final String KAFKA_VERSION = "kafka.version"; + public static final String BOOTSTRAP_SERVERS = "bootstrap.servers"; + public static final String GROUP_ID = "group.id"; + public static final String TOPIC = "topic"; + public static final String STARTUP_MODE = "startup.mode"; + public static final String SPECIFIC_OFFSETS = "specific.offsets"; + public static final String TABLE_JSON_MAPPING = "table.json.mapping"; + + public static final String PARTITION = "partition"; + public static final String OFFSET = "offset"; + + public static final String TABLE_FIELD = "table.field"; + public static final String JSON_FIELD = "json.field"; + + public static final String ZOOKEEPER_CONNECT = "zookeeper.connect"; // only required for 0.8 + + // values + public static final String KAFKA_VERSION_VALUE_08 = "0.8"; + public static final String KAFKA_VERSION_VALUE_09 = "0.9"; + public static final String KAFKA_VERSION_VALUE_010 = "0.10"; + public static final String KAFKA_VERSION_VALUE_011 = "0.11"; + + public static final String STARTUP_MODE_VALUE_EARLIEST = "earliest-offset"; + public static final String STARTUP_MODE_VALUE_LATEST = "latest-offset"; + public static final String STARTUP_MODE_VALUE_GROUP_OFFSETS = "group-offsets"; + public static final String STARTUP_MODE_VALUE_SPECIFIC_OFFSETS = "specific-offsets"; + + // utils + public static Map normalizeStartupMode(StartupMode startupMode) { + Map mapPair = new HashMap<>(); + switch (startupMode) { + case EARLIEST: + mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_EARLIEST); + break; + case LATEST: + mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_LATEST); + break; + case GROUP_OFFSETS: + mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_GROUP_OFFSETS); + break; + case SPECIFIC_OFFSETS: + mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_SPECIFIC_OFFSETS); + break; + } + return mapPair; + } + + @Override + public void validate(DescriptorProperties properties) { + super.validate(properties); + + AbstractFunction0 emptyValidator = new AbstractFunction0() { + @Override + public BoxedUnit apply() { + return BoxedUnit.UNIT; + } + }; + + properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE, false); + + AbstractFunction0 version08Validator = new AbstractFunction0() { + @Override + public BoxedUnit apply() { + properties.validateString(ZOOKEEPER_CONNECT, false, 0, Integer.MAX_VALUE); + return BoxedUnit.UNIT; + } + }; + + Map> versionValidatorMap = new HashMap<>(); + versionValidatorMap.put(KAFKA_VERSION_VALUE_08, version08Validator); + versionValidatorMap.put(KAFKA_VERSION_VALUE_09, emptyValidator); + versionValidatorMap.put(KAFKA_VERSION_VALUE_010, emptyValidator); + versionValidatorMap.put(KAFKA_VERSION_VALUE_011, emptyValidator); + properties.validateEnum( + KAFKA_VERSION, + false, + toScalaImmutableMap(versionValidatorMap) + ); + + properties.validateString(BOOTSTRAP_SERVERS, false, 1, Integer.MAX_VALUE); + properties.validateString(GROUP_ID, false, 1, Integer.MAX_VALUE); + properties.validateString(TOPIC, false, 1, Integer.MAX_VALUE); + + AbstractFunction0 specificOffsetsValidator = new AbstractFunction0() { + @Override + public BoxedUnit apply() { + Map partitions = JavaConversions.mapAsJavaMap( + properties.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION)); + + Map offsets = JavaConversions.mapAsJavaMap( + properties.getIndexedProperty(SPECIFIC_OFFSETS, OFFSET)); + if (partitions.isEmpty() || offsets.isEmpty()) { + throw new ValidationException("Offsets must be set for SPECIFIC_OFFSETS mode."); + } + for (int i = 0; i < partitions.size(); ++i) { + properties.validateInt( + SPECIFIC_OFFSETS + "." + i + "." + PARTITION, + false, + 0, + Integer.MAX_VALUE); + properties.validateLong( + SPECIFIC_OFFSETS + "." + i + "." + OFFSET, + false, + 0, + Long.MAX_VALUE); + } + return BoxedUnit.UNIT; + } + }; + Map> startupModeValidatorMap = new HashMap<>(); + startupModeValidatorMap.put(STARTUP_MODE_VALUE_GROUP_OFFSETS, emptyValidator); + startupModeValidatorMap.put(STARTUP_MODE_VALUE_EARLIEST, emptyValidator); + startupModeValidatorMap.put(STARTUP_MODE_VALUE_LATEST, emptyValidator); + startupModeValidatorMap.put(STARTUP_MODE_VALUE_SPECIFIC_OFFSETS, specificOffsetsValidator); + + properties.validateEnum(STARTUP_MODE, true, toScalaImmutableMap(startupModeValidatorMap)); + validateTableJsonMapping(properties); + } + + private void validateTableJsonMapping(DescriptorProperties properties) { + Map mappingTableField = JavaConversions.mapAsJavaMap( + properties.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD)); + Map mappingJsonField = JavaConversions.mapAsJavaMap( + properties.getIndexedProperty(TABLE_JSON_MAPPING, JSON_FIELD)); + + if (mappingJsonField.size() != mappingJsonField.size()) { + throw new ValidationException("Table JSON mapping must be one to one."); + } + + for (int i = 0; i < mappingTableField.size(); i++) { + properties.validateString( + TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD, + false, + 1, + Integer.MAX_VALUE); + properties.validateString( + TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD, + false, + 1, + Integer.MAX_VALUE); + } + } + + @SuppressWarnings("unchecked") + private scala.collection.immutable.Map toScalaImmutableMap(Map javaMap) { + final java.util.List> list = new java.util.ArrayList<>(javaMap.size()); + for (final java.util.Map.Entry entry : javaMap.entrySet()) { + list.add(scala.Tuple2.apply(entry.getKey(), entry.getValue())); + } + final scala.collection.Seq> seq = + scala.collection.JavaConverters.asScalaBufferConverter(list).asScala().toSeq(); + return (scala.collection.immutable.Map) scala.collection.immutable.Map$.MODULE$.apply(seq); + } +} diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java new file mode 100644 index 00000000000000..964a62425481e3 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.descriptors.Kafka; + +import org.mockito.Mockito; + +/** + * Tests for {@link KafkaJsonTableSourceFactory}. + */ +public abstract class KafkaJsonTableFromDescriptorTestBase { + private static final String GROUP_ID = "test-group"; + private static final String BOOTSTRAP_SERVERS = "localhost:1234"; + private static final String TOPIC = "test-topic"; + + protected abstract String versionForTest(); + + protected abstract KafkaJsonTableSource.Builder builderForTest(); + + protected abstract void extraSettings(KafkaTableSource.Builder builder, Kafka kafka); + + private static StreamExecutionEnvironment env = Mockito.mock(StreamExecutionEnvironment.class); + private static StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env); + +// @Test +// public void buildJsonTableSourceTest() throws Exception { +// final URL url = getClass().getClassLoader().getResource("kafka-json-schema.json"); +// Objects.requireNonNull(url); +// final String schema = FileUtils.readFileUtf8(new File(url.getFile())); +// +// Map tableJsonMapping = new HashMap<>(); +// tableJsonMapping.put("fruit-name", "name"); +// tableJsonMapping.put("fruit-count", "count"); +// tableJsonMapping.put("event-time", "time"); +// +// // Construct with the builder. +// Properties props = new Properties(); +// props.put("group.id", GROUP_ID); +// props.put("bootstrap.servers", BOOTSTRAP_SERVERS); +// +// Map specificOffsets = new HashMap<>(); +// specificOffsets.put(new KafkaTopicPartition(TOPIC, 0), 100L); +// specificOffsets.put(new KafkaTopicPartition(TOPIC, 1), 123L); +// +// KafkaTableSource.Builder builder = builderForTest() +// .forJsonSchema(TableSchema.fromTypeInfo(JsonSchemaConverter.convert(schema))) +// .failOnMissingField(true) +// .withTableToJsonMapping(tableJsonMapping) +// .withKafkaProperties(props) +// .forTopic(TOPIC) +// .fromSpecificOffsets(specificOffsets) +// .withSchema( +// TableSchema.builder() +// .field("fruit-name", Types.STRING) +// .field("fruit-count", Types.INT) +// .field("event-time", Types.LONG) +// .field("proc-time", Types.SQL_TIMESTAMP) +// .build()) +// .withProctimeAttribute("proc-time"); +// +// // Construct with the descriptor. +// Map offsets = new HashMap<>(); +// offsets.put(0, 100L); +// offsets.put(1, 123L); +// Kafka kafka = new Kafka() +// .version(versionForTest()) +// .groupId(GROUP_ID) +// .bootstrapServers(BOOTSTRAP_SERVERS) +// .topic(TOPIC) +// .startupMode(StartupMode.SPECIFIC_OFFSETS) +// .specificOffsets(offsets) +// .tableJsonMapping(tableJsonMapping); +// extraSettings(builder, kafka); +// +// TableSource source = tEnv +// .from(kafka) +// .withFormat( +// new Json() +// .schema(schema) +// .failOnMissingField(true)) +// .withSchema(new Schema() +// .field("fruit-name", Types.STRING) +// .field("fruit-count", Types.INT) +// .field("event-time", Types.LONG) +// .field("proc-time", Types.SQL_TIMESTAMP).proctime()) +// .toTableSource(); +// +// Assert.assertEquals(builder.build(), source); +// } + +// @Test(expected = TableException.class) +// public void buildJsonTableSourceFailTest() { +// tEnv.from( +// new Kafka() +// .version(versionForTest()) +// .groupId(GROUP_ID) +// .bootstrapServers(BOOTSTRAP_SERVERS) +// .topic(TOPIC) +// .startupMode(StartupMode.SPECIFIC_OFFSETS) +// .specificOffsets(new HashMap<>())) +// .withFormat( +// new Json() +// .schema("") +// .failOnMissingField(true)) +// .toTableSource(); +// } +} diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json new file mode 100644 index 00000000000000..5167e5e8724057 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{ + "title": "Fruit", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "time": { + "description": "Age in years", + "type": "number" + } + }, + "required": ["name", "count", "time"] +} diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala index 534ef394f27bd5..1e88d932ed6f99 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala @@ -91,6 +91,11 @@ class TableSchema( } } + /** + * Returns the number of columns. + */ + def getColumnNum: Int = columnNames.length + /** * Returns all column names as an array. */ From d9f2f2f83c8a099e687e158705375db1b56dca01 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 19 Feb 2018 13:35:45 +0100 Subject: [PATCH 0045/2294] [FLINK-8538] [table] Improve unified table sources This closes #5564. --- .../flink-connector-kafka-0.10/pom.xml | 8 + .../kafka/Kafka010JsonTableSourceFactory.java | 5 +- .../resources/tableSourceConverter.properties | 29 - ...> Kafka010JsonTableSourceFactoryTest.java} | 18 +- .../flink-connector-kafka-0.11/pom.xml | 8 + .../kafka/Kafka011JsonTableSourceFactory.java | 5 +- .../resources/tableSourceConverter.properties | 29 - ...> Kafka011JsonTableSourceFactoryTest.java} | 18 +- .../flink-connector-kafka-0.8/pom.xml | 8 + .../kafka/Kafka08JsonTableSourceFactory.java | 5 +- .../resources/tableSourceConverter.properties | 29 - ...=> Kafka08JsonTableSourceFactoryTest.java} | 19 +- .../flink-connector-kafka-0.9/pom.xml | 8 + .../kafka/Kafka09JsonTableSourceFactory.java | 5 +- .../resources/tableSourceConverter.properties | 29 - ...=> Kafka09JsonTableSourceFactoryTest.java} | 18 +- .../flink-connector-kafka-base/pom.xml | 16 +- .../kafka/KafkaJsonTableSource.java | 24 +- .../kafka/KafkaJsonTableSourceFactory.java | 267 +++--- .../connectors/kafka/KafkaTableSource.java | 42 +- .../apache/flink/table/descriptors/Kafka.java | 239 ++--- .../table/descriptors/KafkaValidator.java | 226 ++--- .../KafkaJsonTableFromDescriptorTestBase.java | 127 --- .../KafkaJsonTableSourceFactoryTestBase.java | 145 +++ .../flink/table/descriptors/KafkaTest.java | 113 +++ .../src/test/resources/kafka-json-schema.json | 35 - flink-dist/pom.xml | 16 - flink-formats/flink-json/pom.xml | 29 +- .../apache/flink/table/descriptors/Json.java | 129 +++ .../table/descriptors/JsonValidator.java | 55 ++ .../flink/table/descriptors/JsonTest.java | 124 +++ .../client/gateway/local/LocalExecutor.java | 2 +- flink-libraries/flink-table/pom.xml | 12 + .../resources/tableSourceConverter.properties | 7 + .../apache/flink/table/api/TableSchema.scala | 6 +- .../table/catalog/ExternalCatalogTable.scala | 19 +- .../catalog/ExternalTableSourceUtil.scala | 2 +- .../BatchTableSourceDescriptor.scala | 2 +- .../descriptors/ConnectorDescriptor.scala | 9 +- .../ConnectorDescriptorValidator.scala | 18 +- .../apache/flink/table/descriptors/Csv.scala | 13 +- .../descriptors/DescriptorProperties.scala | 833 +++++++++++++++--- .../flink/table/descriptors/FileSystem.scala | 5 +- .../table/descriptors/FormatDescriptor.scala | 4 +- .../FormatDescriptorValidator.scala | 23 +- .../apache/flink/table/descriptors/Json.scala | 78 -- .../table/descriptors/JsonValidator.scala | 41 - .../table/descriptors/MetadataValidator.scala | 6 +- .../flink/table/descriptors/Rowtime.scala | 8 +- .../table/descriptors/RowtimeValidator.scala | 179 ++-- .../flink/table/descriptors/Schema.scala | 14 +- .../table/descriptors/SchemaValidator.scala | 171 +++- .../flink/table/descriptors/Statistics.scala | 6 +- .../descriptors/StatisticsValidator.scala | 21 +- .../StreamTableSourceDescriptor.scala | 2 +- .../descriptors/TableSourceDescriptor.scala | 3 +- .../table/sources/CsvTableSourceFactory.scala | 43 +- .../table/sources/TableSourceFactory.scala | 11 +- .../sources/TableSourceFactoryService.scala | 26 +- .../tsextractors/StreamRecordTimestamp.scala | 5 +- .../flink/table/descriptors/CsvTest.scala | 83 +- .../descriptors/DescriptorTestBase.scala | 65 +- .../table/descriptors/FileSystemTest.scala | 33 +- .../flink/table/descriptors/JsonTest.scala | 77 -- .../table/descriptors/MetadataTest.scala | 38 +- .../flink/table/descriptors/RowtimeTest.scala | 59 +- .../flink/table/descriptors/SchemaTest.scala | 84 +- .../descriptors/SchemaValidatorTest.scala | 76 ++ .../table/descriptors/StatisticsTest.scala | 72 +- .../TableSourceFactoryServiceTest.scala | 20 +- .../sources/TestTableSourceFactory.scala | 8 +- 71 files changed, 2561 insertions(+), 1451 deletions(-) delete mode 100644 flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties rename flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/{Kafka010TableSourceFactoryTest.java => Kafka010JsonTableSourceFactoryTest.java} (72%) delete mode 100644 flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties rename flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/{Kafka011TableSourceFactoryTest.java => Kafka011JsonTableSourceFactoryTest.java} (72%) delete mode 100644 flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties rename flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/{Kafka08TableSourceFactoryTest.java => Kafka08JsonTableSourceFactoryTest.java} (68%) delete mode 100644 flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties rename flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/{Kafka09TableSourceFactoryTest.java => Kafka09JsonTableSourceFactoryTest.java} (73%) delete mode 100644 flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java create mode 100644 flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java delete mode 100644 flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json create mode 100644 flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java create mode 100644 flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java create mode 100644 flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala delete mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml index 1ae1cd89206324..22efc3470d7abb 100644 --- a/flink-connectors/flink-connector-kafka-0.10/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml @@ -192,6 +192,14 @@ under the License. test + + org.apache.flink + flink-table_${scala.binary.version} + ${project.version} + test-jar + test + + diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java index 1d03f6c4e8c031..c639a44289e643 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java @@ -18,12 +18,13 @@ package org.apache.flink.streaming.connectors.kafka; -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_010; /** * Factory for creating configured instances of {@link Kafka010JsonTableSource}. */ public class Kafka010JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override protected KafkaJsonTableSource.Builder createBuilder() { return new Kafka010JsonTableSource.Builder(); @@ -31,6 +32,6 @@ protected KafkaJsonTableSource.Builder createBuilder() { @Override protected String kafkaVersion() { - return KAFKA_VERSION_VALUE_010; + return CONNECTOR_VERSION_VALUE_010; } } diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties deleted file mode 100644 index 5409b49703088b..00000000000000 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -################################################################################ -# The config file is used to specify the packages of current module where -# to find TableSourceConverter implementation class annotated with TableType. -# If there are multiple packages to scan, put those packages together into a -# string separated with ',', for example, org.package1,org.package2. -# Please notice: -# It's better to have a tableSourceConverter.properties in each connector Module -# which offers converters instead of put all information into the -# tableSourceConverter.properties of flink-table module. -################################################################################ -scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactoryTest.java similarity index 72% rename from flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java rename to flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactoryTest.java index 15b89e8e2f85a4..22cf659ffeb934 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactoryTest.java @@ -18,24 +18,20 @@ package org.apache.flink.streaming.connectors.kafka; -import org.apache.flink.table.descriptors.Kafka; - -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_010; /** * Tests for {@link Kafka010JsonTableSourceFactory}. */ -public class Kafka010TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { - protected String versionForTest() { - return KAFKA_VERSION_VALUE_010; - } +public class Kafka010JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase { - protected KafkaJsonTableSource.Builder builderForTest() { - return Kafka010JsonTableSource.builder(); + @Override + protected String version() { + return CONNECTOR_VERSION_VALUE_010; } @Override - protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { - // no extra settings + protected KafkaJsonTableSource.Builder builder() { + return Kafka010JsonTableSource.builder(); } } diff --git a/flink-connectors/flink-connector-kafka-0.11/pom.xml b/flink-connectors/flink-connector-kafka-0.11/pom.xml index 8a4e339431f6dd..befa33686c18fb 100644 --- a/flink-connectors/flink-connector-kafka-0.11/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.11/pom.xml @@ -201,6 +201,14 @@ under the License. test + + org.apache.flink + flink-table_${scala.binary.version} + ${project.version} + test-jar + test + + diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java index ca4d6ce01374d3..6745bb294f12f7 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java @@ -18,12 +18,13 @@ package org.apache.flink.streaming.connectors.kafka; -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_011; /** * Factory for creating configured instances of {@link Kafka011JsonTableSource}. */ public class Kafka011JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override protected KafkaJsonTableSource.Builder createBuilder() { return new Kafka011JsonTableSource.Builder(); @@ -31,6 +32,6 @@ protected KafkaJsonTableSource.Builder createBuilder() { @Override protected String kafkaVersion() { - return KAFKA_VERSION_VALUE_011; + return CONNECTOR_VERSION_VALUE_011; } } diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties deleted file mode 100644 index 5409b49703088b..00000000000000 --- a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -################################################################################ -# The config file is used to specify the packages of current module where -# to find TableSourceConverter implementation class annotated with TableType. -# If there are multiple packages to scan, put those packages together into a -# string separated with ',', for example, org.package1,org.package2. -# Please notice: -# It's better to have a tableSourceConverter.properties in each connector Module -# which offers converters instead of put all information into the -# tableSourceConverter.properties of flink-table module. -################################################################################ -scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactoryTest.java similarity index 72% rename from flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java rename to flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactoryTest.java index 84ac39b6f47e8b..ed92863ba6d6fa 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactoryTest.java @@ -18,24 +18,20 @@ package org.apache.flink.streaming.connectors.kafka; -import org.apache.flink.table.descriptors.Kafka; - -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_011; /** * Tests for {@link Kafka011JsonTableSourceFactory}. */ -public class Kafka011TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { - protected String versionForTest() { - return KAFKA_VERSION_VALUE_011; - } +public class Kafka011JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase { - protected KafkaJsonTableSource.Builder builderForTest() { - return Kafka011JsonTableSource.builder(); + @Override + protected String version() { + return CONNECTOR_VERSION_VALUE_011; } @Override - protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { - // no extra settings + protected KafkaJsonTableSource.Builder builder() { + return Kafka011JsonTableSource.builder(); } } diff --git a/flink-connectors/flink-connector-kafka-0.8/pom.xml b/flink-connectors/flink-connector-kafka-0.8/pom.xml index 02d36f5fceb409..7750ef37e7bf2f 100644 --- a/flink-connectors/flink-connector-kafka-0.8/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.8/pom.xml @@ -204,6 +204,14 @@ under the License. test + + org.apache.flink + flink-table_${scala.binary.version} + ${project.version} + test-jar + test + + diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java index e4e50960b627c5..2da805a24b8eef 100644 --- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java +++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java @@ -18,12 +18,13 @@ package org.apache.flink.streaming.connectors.kafka; -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_08; /** * Factory for creating configured instances of {@link Kafka08JsonTableSource}. */ public class Kafka08JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override protected KafkaJsonTableSource.Builder createBuilder() { return new Kafka08JsonTableSource.Builder(); @@ -31,6 +32,6 @@ protected KafkaJsonTableSource.Builder createBuilder() { @Override protected String kafkaVersion() { - return KAFKA_VERSION_VALUE_08; + return CONNECTOR_VERSION_VALUE_08; } } diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties deleted file mode 100644 index 5409b49703088b..00000000000000 --- a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -################################################################################ -# The config file is used to specify the packages of current module where -# to find TableSourceConverter implementation class annotated with TableType. -# If there are multiple packages to scan, put those packages together into a -# string separated with ',', for example, org.package1,org.package2. -# Please notice: -# It's better to have a tableSourceConverter.properties in each connector Module -# which offers converters instead of put all information into the -# tableSourceConverter.properties of flink-table module. -################################################################################ -scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactoryTest.java similarity index 68% rename from flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java rename to flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactoryTest.java index a2edc09ec125f9..0238b2bf4f935d 100644 --- a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java +++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactoryTest.java @@ -18,25 +18,20 @@ package org.apache.flink.streaming.connectors.kafka; -import org.apache.flink.table.descriptors.Kafka; - -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_08; /** * Tests for {@link Kafka08JsonTableSourceFactory}. */ -public class Kafka08TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { - protected String versionForTest() { - return KAFKA_VERSION_VALUE_08; - } +public class Kafka08JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase { - protected KafkaJsonTableSource.Builder builderForTest() { - return Kafka08JsonTableSource.builder(); + @Override + protected String version() { + return CONNECTOR_VERSION_VALUE_08; } @Override - protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { - builder.getKafkaProps().put("zookeeper.connect", "localhost:1111"); - kafka.zookeeperConnect("localhost:1111"); + protected KafkaJsonTableSource.Builder builder() { + return Kafka08JsonTableSource.builder(); } } diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml index ee0c458171096d..a4b0cfc3c1ef90 100644 --- a/flink-connectors/flink-connector-kafka-0.9/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml @@ -173,6 +173,14 @@ under the License. test + + org.apache.flink + flink-table_${scala.binary.version} + ${project.version} + test-jar + test + + org.apache.hadoop hadoop-minikdc diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java index bbda4ae66d6367..9207426869e501 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java @@ -18,12 +18,13 @@ package org.apache.flink.streaming.connectors.kafka; -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_09; /** * Factory for creating configured instances of {@link Kafka09JsonTableSource}. */ public class Kafka09JsonTableSourceFactory extends KafkaJsonTableSourceFactory { + @Override protected KafkaJsonTableSource.Builder createBuilder() { return new Kafka09JsonTableSource.Builder(); @@ -31,6 +32,6 @@ protected KafkaJsonTableSource.Builder createBuilder() { @Override protected String kafkaVersion() { - return KAFKA_VERSION_VALUE_09; + return CONNECTOR_VERSION_VALUE_09; } } diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties deleted file mode 100644 index 5409b49703088b..00000000000000 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -################################################################################ -# The config file is used to specify the packages of current module where -# to find TableSourceConverter implementation class annotated with TableType. -# If there are multiple packages to scan, put those packages together into a -# string separated with ',', for example, org.package1,org.package2. -# Please notice: -# It's better to have a tableSourceConverter.properties in each connector Module -# which offers converters instead of put all information into the -# tableSourceConverter.properties of flink-table module. -################################################################################ -scan.packages=org.apache.flink.streaming.connectors.kafka diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactoryTest.java similarity index 73% rename from flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java rename to flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactoryTest.java index fc85ea77a07cd6..dd545e9395138c 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactoryTest.java @@ -18,24 +18,20 @@ package org.apache.flink.streaming.connectors.kafka; -import org.apache.flink.table.descriptors.Kafka; - -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_09; /** * Factory for creating configured instances of {@link Kafka09JsonTableSource}. */ -public class Kafka09TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase { - protected String versionForTest() { - return KAFKA_VERSION_VALUE_09; - } +public class Kafka09JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase { - protected KafkaJsonTableSource.Builder builderForTest() { - return Kafka09JsonTableSource.builder(); + @Override + protected String version() { + return CONNECTOR_VERSION_VALUE_09; } @Override - protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) { - // no extra settings + protected KafkaJsonTableSource.Builder builder() { + return Kafka09JsonTableSource.builder(); } } diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml index 212a86b9746cb5..1694afe8f24a60 100644 --- a/flink-connectors/flink-connector-kafka-base/pom.xml +++ b/flink-connectors/flink-connector-kafka-base/pom.xml @@ -198,24 +198,18 @@ under the License. test - - org.apache.hadoop - hadoop-minikdc - ${minikdc.version} - test - - org.apache.flink - flink-scala_${scala.binary.version} + flink-table_${scala.binary.version} ${project.version} + test-jar test - org.apache.flink - flink-streaming-scala_${scala.binary.version} - ${project.version} + org.apache.hadoop + hadoop-minikdc + ${minikdc.version} test diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java index d2dafe72b34951..b2bb8ff773bf46 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java @@ -84,23 +84,29 @@ protected JsonRowDeserializationSchema getDeserializationSchema() { @Override public String explainSource() { - return "KafkaJSONTableSource"; + return "KafkaJsonTableSource"; } @Override - public boolean equals(Object other) { - if (super.equals(other)) { - KafkaJsonTableSource otherSource = (KafkaJsonTableSource) other; - return Objects.equals(failOnMissingField, otherSource.failOnMissingField) - && Objects.equals(jsonSchema, otherSource.jsonSchema) - && Objects.equals(fieldMapping, otherSource.fieldMapping); + public boolean equals(Object o) { + if (this == o) { + return true; } - return false; + if (!(o instanceof KafkaJsonTableSource)) { + return false; + } + if (!super.equals(o)) { + return false; + } + KafkaJsonTableSource that = (KafkaJsonTableSource) o; + return failOnMissingField == that.failOnMissingField && + Objects.equals(jsonSchema, that.jsonSchema) && + Objects.equals(fieldMapping, that.fieldMapping); } @Override public int hashCode() { - return 31 * super.hashCode() + Objects.hash(failOnMissingField, jsonSchema, fieldMapping); + return Objects.hash(super.hashCode(), jsonSchema, fieldMapping, failOnMissingField); } //////// SETTERS FOR OPTIONAL PARAMETERS diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java index 918b83357ff569..28973149abd333 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java @@ -21,65 +21,74 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.formats.json.JsonSchemaConverter; import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.descriptors.DescriptorProperties; import org.apache.flink.table.descriptors.JsonValidator; import org.apache.flink.table.descriptors.KafkaValidator; import org.apache.flink.table.descriptors.SchemaValidator; +import org.apache.flink.table.sources.RowtimeAttributeDescriptor; import org.apache.flink.table.sources.TableSource; import org.apache.flink.table.sources.TableSourceFactory; import org.apache.flink.types.Row; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; +import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_PROPERTY_VERSION; import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE; import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION; +import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA; +import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_PROPERTY_VERSION; import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_TYPE; -import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_VERSION; import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD; -import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA_STRING; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_JSON_SCHEMA; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA; import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_TYPE_VALUE; -import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS; -import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE; -import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID; -import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD; -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION; -import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET; -import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION; -import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS; -import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE; -import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_EARLIEST; -import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_GROUP_OFFSETS; -import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_LATEST; -import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_SPECIFIC_OFFSETS; -import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD; -import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING; -import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC; -import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT; -import static org.apache.flink.table.descriptors.SchemaValidator.PROCTIME; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_KEY; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_VALUE; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_OFFSET; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_PARTITION; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_STARTUP_MODE; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TOPIC; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE_KAFKA; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_CLASS; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_FROM; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_SERIALIZED; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_TYPE; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_CLASS; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_DELAY; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_SERIALIZED; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_TYPE; import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA; -import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_VERSION; - -import scala.Option; -import scala.collection.JavaConversions; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_FROM; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_NAME; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_PROCTIME; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_TYPE; /** * Factory for creating configured instances of {@link KafkaJsonTableSource}. */ public abstract class KafkaJsonTableSourceFactory implements TableSourceFactory { + @Override public Map requiredContext() { Map context = new HashMap<>(); - context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE); // kafka connector - context.put(FORMAT_TYPE(), FORMAT_TYPE_VALUE()); // Json format - context.put(KAFKA_VERSION, kafkaVersion()); // for different implementations - context.put(CONNECTOR_VERSION(), "1"); - context.put(FORMAT_VERSION(), "1"); - context.put(SCHEMA_VERSION(), "1"); + context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_KAFKA); // kafka + context.put(CONNECTOR_VERSION(), kafkaVersion()); + + context.put(FORMAT_TYPE(), FORMAT_TYPE_VALUE); // json format + + context.put(CONNECTOR_PROPERTY_VERSION(), "1"); // backwards compatibility + context.put(FORMAT_PROPERTY_VERSION(), "1"); + return context; } @@ -88,124 +97,137 @@ public List supportedProperties() { List properties = new ArrayList<>(); // kafka - properties.add(KAFKA_VERSION); - properties.add(BOOTSTRAP_SERVERS); - properties.add(GROUP_ID); - properties.add(ZOOKEEPER_CONNECT); - properties.add(TOPIC); - properties.add(STARTUP_MODE); - properties.add(SPECIFIC_OFFSETS + ".#." + PARTITION); - properties.add(SPECIFIC_OFFSETS + ".#." + OFFSET); + properties.add(CONNECTOR_TOPIC); + properties.add(CONNECTOR_PROPERTIES); + properties.add(CONNECTOR_PROPERTIES + ".#." + CONNECTOR_PROPERTIES_KEY); + properties.add(CONNECTOR_PROPERTIES + ".#." + CONNECTOR_PROPERTIES_VALUE); + properties.add(CONNECTOR_STARTUP_MODE); + properties.add(CONNECTOR_SPECIFIC_OFFSETS + ".#." + CONNECTOR_SPECIFIC_OFFSETS_PARTITION); + properties.add(CONNECTOR_SPECIFIC_OFFSETS + ".#." + CONNECTOR_SPECIFIC_OFFSETS_OFFSET); // json format - properties.add(FORMAT_SCHEMA_STRING()); - properties.add(FORMAT_FAIL_ON_MISSING_FIELD()); - - // table json mapping - properties.add(TABLE_JSON_MAPPING + ".#." + TABLE_FIELD); - properties.add(TABLE_JSON_MAPPING + ".#." + JSON_FIELD); + properties.add(FORMAT_JSON_SCHEMA); + properties.add(FORMAT_SCHEMA); + properties.add(FORMAT_FAIL_ON_MISSING_FIELD); + properties.add(FORMAT_DERIVE_SCHEMA()); // schema - properties.add(SCHEMA() + ".#." + DescriptorProperties.TYPE()); - properties.add(SCHEMA() + ".#." + DescriptorProperties.NAME()); + properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); + properties.add(SCHEMA() + ".#." + SCHEMA_NAME()); + properties.add(SCHEMA() + ".#." + SCHEMA_FROM()); // time attributes - properties.add(SCHEMA() + ".#." + PROCTIME()); -// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_CLASS()); -// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_TYPE()); + properties.add(SCHEMA() + ".#." + SCHEMA_PROCTIME()); + properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_TYPE()); + properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_FROM()); + properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_CLASS()); + properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_SERIALIZED()); + properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_TYPE()); + properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_CLASS()); + properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_SERIALIZED()); + properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_DELAY()); return properties; } @Override public TableSource create(Map properties) { - DescriptorProperties params = new DescriptorProperties(true); + final DescriptorProperties params = new DescriptorProperties(true); params.putProperties(properties); // validate + new SchemaValidator(true).validate(params); new KafkaValidator().validate(params); new JsonValidator().validate(params); - new SchemaValidator(true).validate(params); // build - KafkaJsonTableSource.Builder builder = createBuilder(); - Properties kafkaProps = new Properties(); - - // Set the required parameters. - String topic = params.getString(TOPIC).get(); - TableSchema tableSchema = params.getTableSchema(SCHEMA()).get(); - - kafkaProps.put(BOOTSTRAP_SERVERS, params.getString(BOOTSTRAP_SERVERS).get()); - kafkaProps.put(GROUP_ID, params.getString(GROUP_ID).get()); - - // Set the zookeeper connect for kafka 0.8. - Option zkConnect = params.getString(ZOOKEEPER_CONNECT); - if (zkConnect.isDefined()) { - kafkaProps.put(ZOOKEEPER_CONNECT, zkConnect.get()); - } - - builder.withKafkaProperties(kafkaProps).forTopic(topic).withSchema(tableSchema); - - // Set the startup mode. - String startupMode = params.getString(STARTUP_MODE).get(); - if (null != startupMode) { - switch (startupMode) { - case STARTUP_MODE_VALUE_EARLIEST: + final KafkaJsonTableSource.Builder builder = createBuilder(); + + // topic + final String topic = params.getString(CONNECTOR_TOPIC); + builder.forTopic(topic); + + // properties + final Properties props = new Properties(); + final List> propsList = params.getFixedIndexedProperties( + CONNECTOR_PROPERTIES, + Arrays.asList(CONNECTOR_PROPERTIES_KEY, CONNECTOR_PROPERTIES_VALUE)); + propsList.forEach(kv -> props.put( + params.getString(kv.get(CONNECTOR_PROPERTIES_KEY)), + params.getString(kv.get(CONNECTOR_PROPERTIES_VALUE)) + )); + builder.withKafkaProperties(props); + + // startup mode + params + .getOptionalString(CONNECTOR_STARTUP_MODE) + .ifPresent(startupMode -> { + switch (startupMode) { + + case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_EARLIEST: builder.fromEarliest(); break; - case STARTUP_MODE_VALUE_LATEST: + + case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_LATEST: builder.fromLatest(); break; - case STARTUP_MODE_VALUE_GROUP_OFFSETS: + + case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS: builder.fromGroupOffsets(); break; - case STARTUP_MODE_VALUE_SPECIFIC_OFFSETS: - Map partitions = JavaConversions. - mapAsJavaMap(params.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION)); - Map offsetMap = new HashMap<>(); - for (int i = 0; i < partitions.size(); i++) { - offsetMap.put( - new KafkaTopicPartition( - topic, - Integer.valueOf(params.getString( - SPECIFIC_OFFSETS + "" + "." + i + "." + PARTITION).get())), - Long.valueOf(params.getString( - SPECIFIC_OFFSETS + "" + "." + i + "." + OFFSET).get())); - } + + case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS: + final Map offsetMap = new HashMap<>(); + + final List> offsetList = params.getFixedIndexedProperties( + CONNECTOR_SPECIFIC_OFFSETS, + Arrays.asList(CONNECTOR_SPECIFIC_OFFSETS_PARTITION, CONNECTOR_SPECIFIC_OFFSETS_OFFSET)); + offsetList.forEach(kv -> { + final int partition = params.getInt(kv.get(CONNECTOR_SPECIFIC_OFFSETS_PARTITION)); + final long offset = params.getLong(kv.get(CONNECTOR_SPECIFIC_OFFSETS_OFFSET)); + final KafkaTopicPartition topicPartition = new KafkaTopicPartition(topic, partition); + offsetMap.put(topicPartition, offset); + }); builder.fromSpecificOffsets(offsetMap); break; - } - } - - // Set whether fail on missing JSON field. - Option failOnMissing = params.getString(FORMAT_FAIL_ON_MISSING_FIELD()); - if (failOnMissing.isDefined()) { - builder.failOnMissingField(Boolean.valueOf(failOnMissing.get())); + } + }); + + // missing field + params.getOptionalBoolean(FORMAT_FAIL_ON_MISSING_FIELD).ifPresent(builder::failOnMissingField); + + // json schema + final TableSchema formatSchema; + if (params.containsKey(FORMAT_SCHEMA)) { + final TypeInformation info = params.getType(FORMAT_SCHEMA); + formatSchema = TableSchema.fromTypeInfo(info); + } else if (params.containsKey(FORMAT_JSON_SCHEMA)) { + final TypeInformation info = JsonSchemaConverter.convert(params.getString(FORMAT_JSON_SCHEMA)); + formatSchema = TableSchema.fromTypeInfo(info); + } else { + formatSchema = SchemaValidator.deriveFormatFields(params); } + builder.forJsonSchema(formatSchema); - // Set the JSON schema. - Option jsonSchema = params.getString(FORMAT_SCHEMA_STRING()); - if (jsonSchema.isDefined()) { - TypeInformation jsonSchemaType = JsonSchemaConverter.convert(jsonSchema.get()); - builder.forJsonSchema(TableSchema.fromTypeInfo(jsonSchemaType)); - } - - // Set the table => JSON fields mapping. - Map mappingTableFields = JavaConversions. - mapAsJavaMap(params.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD)); - - if (!mappingTableFields.isEmpty()) { - Map tableJsonMapping = new HashMap<>(); - for (int i = 0; i < mappingTableFields.size(); i++) { - tableJsonMapping.put(params.getString(TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD).get(), - params.getString(TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD).get() - ); - } - builder.withTableToJsonMapping(tableJsonMapping); + // schema + final TableSchema schema = params.getTableSchema(SCHEMA()); + builder.withSchema(schema); + + // proctime + SchemaValidator.deriveProctimeAttribute(params).ifPresent(builder::withProctimeAttribute); + + // rowtime + final List descriptors = SchemaValidator.deriveRowtimeAttributes(params); + if (descriptors.size() > 1) { + throw new TableException("More than one rowtime attribute is not supported yet."); + } else if (descriptors.size() == 1) { + final RowtimeAttributeDescriptor desc = descriptors.get(0); + builder.withRowtimeAttribute(desc.getAttributeName(), desc.getTimestampExtractor(), desc.getWatermarkStrategy()); } - // Set the time attributes. - setTimeAttributes(tableSchema, params, builder); + // field mapping + final Map mapping = SchemaValidator.deriveFieldMapping(params, Optional.of(formatSchema)); + builder.withTableToJsonMapping(mapping); return builder.build(); } @@ -213,15 +235,4 @@ public TableSource create(Map properties) { protected abstract KafkaJsonTableSource.Builder createBuilder(); protected abstract String kafkaVersion(); - - private void setTimeAttributes(TableSchema schema, DescriptorProperties params, KafkaJsonTableSource.Builder builder) { - // TODO to deal with rowtime fields - Option proctimeField; - for (int i = 0; i < schema.getColumnNum(); i++) { - proctimeField = params.getString(SCHEMA() + "." + i + "." + PROCTIME()); - if (proctimeField.isDefined()) { - builder.withProctimeAttribute(schema.getColumnName(i).get()); - } - } - } } diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java index 9ce3b8ed5a17c0..134c483a950161 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java @@ -141,31 +141,27 @@ public String explainSource() { @Override public boolean equals(Object o) { - if (!o.getClass().equals(this.getClass())) { + if (this == o) { + return true; + } + if (!(o instanceof KafkaTableSource)) { return false; } - KafkaTableSource other = (KafkaTableSource) o; - return Objects.equals(topic, other.topic) - && Objects.equals(schema, other.schema) - && Objects.equals(properties, other.properties) - && Objects.equals(proctimeAttribute, other.proctimeAttribute) - && Objects.equals(returnType, other.returnType) - && Objects.equals(rowtimeAttributeDescriptors, other.rowtimeAttributeDescriptors) - && Objects.equals(specificStartupOffsets, other.specificStartupOffsets) - && Objects.equals(startupMode, other.startupMode); + KafkaTableSource that = (KafkaTableSource) o; + return Objects.equals(schema, that.schema) && + Objects.equals(topic, that.topic) && + Objects.equals(properties, that.properties) && + Objects.equals(returnType, that.returnType) && + Objects.equals(proctimeAttribute, that.proctimeAttribute) && + Objects.equals(rowtimeAttributeDescriptors, that.rowtimeAttributeDescriptors) && + startupMode == that.startupMode && + Objects.equals(specificStartupOffsets, that.specificStartupOffsets); } @Override public int hashCode() { - return Objects.hash( - topic, - schema, - properties, - proctimeAttribute, - returnType, - rowtimeAttributeDescriptors, - specificStartupOffsets, - startupMode); + return Objects.hash(schema, topic, properties, returnType, + proctimeAttribute, rowtimeAttributeDescriptors, startupMode, specificStartupOffsets); } /** @@ -211,9 +207,9 @@ protected void setProctimeAttribute(String proctimeAttribute) { // validate that field exists and is of correct type Option> tpe = schema.getType(proctimeAttribute); if (tpe.isEmpty()) { - throw new ValidationException("Processing time attribute " + proctimeAttribute + " is not present in TableSchema."); + throw new ValidationException("Processing time attribute '" + proctimeAttribute + "' is not present in TableSchema."); } else if (tpe.get() != Types.SQL_TIMESTAMP()) { - throw new ValidationException("Processing time attribute " + proctimeAttribute + " is not of type SQL_TIMESTAMP."); + throw new ValidationException("Processing time attribute '" + proctimeAttribute + "' is not of type SQL_TIMESTAMP."); } } this.proctimeAttribute = proctimeAttribute; @@ -230,9 +226,9 @@ protected void setRowtimeAttributeDescriptors(List r String rowtimeAttribute = desc.getAttributeName(); Option> tpe = schema.getType(rowtimeAttribute); if (tpe.isEmpty()) { - throw new ValidationException("Rowtime attribute " + rowtimeAttribute + " is not present in TableSchema."); + throw new ValidationException("Rowtime attribute '" + rowtimeAttribute + "' is not present in TableSchema."); } else if (tpe.get() != Types.SQL_TIMESTAMP()) { - throw new ValidationException("Rowtime attribute " + rowtimeAttribute + " is not of type SQL_TIMESTAMP."); + throw new ValidationException("Rowtime attribute '" + rowtimeAttribute + "' is not of type SQL_TIMESTAMP."); } } this.rowtimeAttributeDescriptors = rowtimeAttributeDescriptors; diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java index 4733f6e002db84..45359587c1cd08 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java @@ -18,182 +18,199 @@ package org.apache.flink.table.descriptors; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; import org.apache.flink.streaming.connectors.kafka.config.StartupMode; - -import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS; -import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE; -import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID; -import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD; -import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION; -import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET; -import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION; -import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS; -import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD; -import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING; -import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC; -import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT; +import org.apache.flink.util.Preconditions; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; - -import scala.collection.JavaConversions; -import scala.collection.Seq; +import java.util.Properties; +import java.util.stream.Collectors; + +import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_KEY; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_VALUE; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_OFFSET; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_PARTITION; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_STARTUP_MODE; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TOPIC; +import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE_KAFKA; /** - * Connector descriptor for the kafka message queue. + * Connector descriptor for the Apache Kafka message queue. */ public class Kafka extends ConnectorDescriptor { - private Optional version = Optional.empty(); - private Optional bootstrapServers = Optional.empty(); - private Optional groupId = Optional.empty(); - private Optional topic = Optional.empty(); - private Optional zookeeperConnect = Optional.empty(); - private Optional> tableJsonMapping = Optional.empty(); - - private Optional startupMode = Optional.empty(); - private Optional> specificOffsets = Optional.empty(); + private String version; + private String topic; + private StartupMode startupMode; + private Map specificOffsets; + private Map kafkaProperties; + /** + * Connector descriptor for the Apache Kafka message queue. + */ public Kafka() { - super(CONNECTOR_TYPE_VALUE, 1); + super(CONNECTOR_TYPE_VALUE_KAFKA, 1, true); } /** - * Sets the kafka version. + * Sets the Kafka version to be used. * - * @param version - * Could be {@link KafkaValidator#KAFKA_VERSION_VALUE_011}, - * {@link KafkaValidator#KAFKA_VERSION_VALUE_010}, - * {@link KafkaValidator#KAFKA_VERSION_VALUE_09}, - * or {@link KafkaValidator#KAFKA_VERSION_VALUE_08}. + * @param version Kafka version. E.g., "0.8", "0.11", etc. */ public Kafka version(String version) { - this.version = Optional.of(version); + Preconditions.checkNotNull(version); + this.version = version; return this; } /** - * Sets the bootstrap servers for kafka. + * Sets the topic from which the table is read. + * + * @param topic The topic from which the table is read. */ - public Kafka bootstrapServers(String bootstrapServers) { - this.bootstrapServers = Optional.of(bootstrapServers); + public Kafka topic(String topic) { + Preconditions.checkNotNull(topic); + this.topic = topic; return this; } /** - * Sets the consumer group id. + * Sets the configuration properties for the Kafka consumer. Resets previously set properties. + * + * @param properties The configuration properties for the Kafka consumer. */ - public Kafka groupId(String groupId) { - this.groupId = Optional.of(groupId); + public Kafka properties(Properties properties) { + Preconditions.checkNotNull(properties); + if (this.kafkaProperties == null) { + this.kafkaProperties = new HashMap<>(); + } + this.kafkaProperties.clear(); + properties.forEach((k, v) -> this.kafkaProperties.put((String) k, (String) v)); return this; } /** - * Sets the topic to consume. + * Adds a configuration properties for the Kafka consumer. + * + * @param key property key for the Kafka consumer + * @param value property value for the Kafka consumer */ - public Kafka topic(String topic) { - this.topic = Optional.of(topic); + public Kafka property(String key, String value) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(value); + if (this.kafkaProperties == null) { + this.kafkaProperties = new HashMap<>(); + } + kafkaProperties.put(key, value); return this; } /** - * Sets the startup mode. + * Configures to start reading from the earliest offset for all partitions. + * + * @see FlinkKafkaConsumerBase#setStartFromEarliest() */ - public Kafka startupMode(StartupMode startupMode) { - this.startupMode = Optional.of(startupMode); + public Kafka startFromEarliest() { + this.startupMode = StartupMode.EARLIEST; + this.specificOffsets = null; return this; } /** - * Sets the zookeeper hosts. Only required by kafka 0.8. + * Configures to start reading from the latest offset for all partitions. + * + * @see FlinkKafkaConsumerBase#setStartFromLatest() */ - public Kafka zookeeperConnect(String zookeeperConnect) { - this.zookeeperConnect = Optional.of(zookeeperConnect); + public Kafka startFromLatest() { + this.startupMode = StartupMode.LATEST; + this.specificOffsets = null; return this; } /** - * Sets the consume offsets for the topic set with {@link Kafka#topic(String)}. - * Only works in {@link StartupMode#SPECIFIC_OFFSETS} mode. + * Configures to start reading from any committed group offsets found in Zookeeper / Kafka brokers. + * + * @see FlinkKafkaConsumerBase#setStartFromGroupOffsets() */ - public Kafka specificOffsets(Map specificOffsets) { - this.specificOffsets = Optional.of(specificOffsets); + public Kafka startFromGroupOffsets() { + this.startupMode = StartupMode.GROUP_OFFSETS; + this.specificOffsets = null; return this; } /** - * Sets the mapping from logical table schema to json schema. + * Configures to start reading partitions from specific offsets, set independently for each partition. + * Resets previously set offsets. + * + * @param specificOffsets the specified offsets for partitions + * @see FlinkKafkaConsumerBase#setStartFromSpecificOffsets(Map) */ - public Kafka tableJsonMapping(Map jsonTableMapping) { - this.tableJsonMapping = Optional.of(jsonTableMapping); + public Kafka startFromSpecificOffsets(Map specificOffsets) { + this.startupMode = StartupMode.SPECIFIC_OFFSETS; + this.specificOffsets = Preconditions.checkNotNull(specificOffsets); return this; } + /** + * Configures to start reading partitions from specific offsets and specifies the given offset for + * the given partition. + * + * @param partition partition index + * @param specificOffset partition offset to start reading from + * @see FlinkKafkaConsumerBase#setStartFromSpecificOffsets(Map) + */ + public Kafka startFromSpecificOffset(int partition, long specificOffset) { + this.startupMode = StartupMode.SPECIFIC_OFFSETS; + if (this.specificOffsets == null) { + this.specificOffsets = new HashMap<>(); + } + this.specificOffsets.put(partition, specificOffset); + return this; + } + + /** + * Internal method for connector properties conversion. + */ @Override public void addConnectorProperties(DescriptorProperties properties) { - if (version.isPresent()) { - properties.putString(KAFKA_VERSION, version.get()); - } - if (bootstrapServers.isPresent()) { - properties.putString(BOOTSTRAP_SERVERS, bootstrapServers.get()); - } - if (groupId.isPresent()) { - properties.putString(GROUP_ID, groupId.get()); - } - if (topic.isPresent()) { - properties.putString(TOPIC, topic.get()); + if (version != null) { + properties.putString(CONNECTOR_VERSION(), version); } - if (zookeeperConnect.isPresent()) { - properties.putString(ZOOKEEPER_CONNECT, zookeeperConnect.get()); + + if (topic != null) { + properties.putString(CONNECTOR_TOPIC, topic); } - if (startupMode.isPresent()) { - Map map = KafkaValidator.normalizeStartupMode(startupMode.get()); - for (Map.Entry entry : map.entrySet()) { - properties.putString(entry.getKey(), entry.getValue()); - } + + if (startupMode != null) { + properties.putString(CONNECTOR_STARTUP_MODE, KafkaValidator.normalizeStartupMode(startupMode)); } - if (specificOffsets.isPresent()) { - List propertyKeys = new ArrayList<>(); - propertyKeys.add(PARTITION); - propertyKeys.add(OFFSET); - - List> propertyValues = new ArrayList<>(specificOffsets.get().size()); - for (Map.Entry entry : specificOffsets.get().entrySet()) { - List partitionOffset = new ArrayList<>(2); - partitionOffset.add(entry.getKey().toString()); - partitionOffset.add(entry.getValue().toString()); - propertyValues.add(JavaConversions.asScalaBuffer(partitionOffset).toSeq()); + + if (specificOffsets != null) { + final List> values = new ArrayList<>(); + for (Map.Entry specificOffset : specificOffsets.entrySet()) { + values.add(Arrays.asList(specificOffset.getKey().toString(), specificOffset.getValue().toString())); } properties.putIndexedFixedProperties( - SPECIFIC_OFFSETS, - JavaConversions.asScalaBuffer(propertyKeys).toSeq(), - JavaConversions.asScalaBuffer(propertyValues).toSeq() - ); + CONNECTOR_SPECIFIC_OFFSETS, + Arrays.asList(CONNECTOR_SPECIFIC_OFFSETS_PARTITION, CONNECTOR_SPECIFIC_OFFSETS_OFFSET), + values); } - if (tableJsonMapping.isPresent()) { - List propertyKeys = new ArrayList<>(); - propertyKeys.add(TABLE_FIELD); - propertyKeys.add(JSON_FIELD); - - List> mappingFields = new ArrayList<>(tableJsonMapping.get().size()); - for (Map.Entry entry : tableJsonMapping.get().entrySet()) { - List singleMapping = new ArrayList<>(2); - singleMapping.add(entry.getKey()); - singleMapping.add(entry.getValue()); - mappingFields.add(JavaConversions.asScalaBuffer(singleMapping).toSeq()); - } + + if (kafkaProperties != null) { properties.putIndexedFixedProperties( - TABLE_JSON_MAPPING, - JavaConversions.asScalaBuffer(propertyKeys).toSeq(), - JavaConversions.asScalaBuffer(mappingFields).toSeq() - ); + CONNECTOR_PROPERTIES, + Arrays.asList(CONNECTOR_PROPERTIES_KEY, CONNECTOR_PROPERTIES_VALUE), + this.kafkaProperties.entrySet().stream() + .map(e -> Arrays.asList(e.getKey(), e.getValue())) + .collect(Collectors.toList()) + ); } } - - @Override - public boolean needsFormat() { - return true; - } } diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java index a3ca22f90b912e..3adc7c518a4475 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java @@ -19,175 +19,97 @@ package org.apache.flink.table.descriptors; import org.apache.flink.streaming.connectors.kafka.config.StartupMode; -import org.apache.flink.table.api.ValidationException; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; - -import scala.Function0; -import scala.Tuple2; -import scala.collection.JavaConversions; -import scala.runtime.AbstractFunction0; -import scala.runtime.BoxedUnit; - +import java.util.function.Consumer; /** * The validator for {@link Kafka}. */ public class KafkaValidator extends ConnectorDescriptorValidator { - // fields - public static final String CONNECTOR_TYPE_VALUE = "kafka"; - public static final String KAFKA_VERSION = "kafka.version"; - public static final String BOOTSTRAP_SERVERS = "bootstrap.servers"; - public static final String GROUP_ID = "group.id"; - public static final String TOPIC = "topic"; - public static final String STARTUP_MODE = "startup.mode"; - public static final String SPECIFIC_OFFSETS = "specific.offsets"; - public static final String TABLE_JSON_MAPPING = "table.json.mapping"; - - public static final String PARTITION = "partition"; - public static final String OFFSET = "offset"; - - public static final String TABLE_FIELD = "table.field"; - public static final String JSON_FIELD = "json.field"; - - public static final String ZOOKEEPER_CONNECT = "zookeeper.connect"; // only required for 0.8 - // values - public static final String KAFKA_VERSION_VALUE_08 = "0.8"; - public static final String KAFKA_VERSION_VALUE_09 = "0.9"; - public static final String KAFKA_VERSION_VALUE_010 = "0.10"; - public static final String KAFKA_VERSION_VALUE_011 = "0.11"; - - public static final String STARTUP_MODE_VALUE_EARLIEST = "earliest-offset"; - public static final String STARTUP_MODE_VALUE_LATEST = "latest-offset"; - public static final String STARTUP_MODE_VALUE_GROUP_OFFSETS = "group-offsets"; - public static final String STARTUP_MODE_VALUE_SPECIFIC_OFFSETS = "specific-offsets"; - - // utils - public static Map normalizeStartupMode(StartupMode startupMode) { - Map mapPair = new HashMap<>(); - switch (startupMode) { - case EARLIEST: - mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_EARLIEST); - break; - case LATEST: - mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_LATEST); - break; - case GROUP_OFFSETS: - mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_GROUP_OFFSETS); - break; - case SPECIFIC_OFFSETS: - mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_SPECIFIC_OFFSETS); - break; - } - return mapPair; - } + public static final String CONNECTOR_TYPE_VALUE_KAFKA = "kafka"; + public static final String CONNECTOR_VERSION_VALUE_08 = "0.8"; + public static final String CONNECTOR_VERSION_VALUE_09 = "0.9"; + public static final String CONNECTOR_VERSION_VALUE_010 = "0.10"; + public static final String CONNECTOR_VERSION_VALUE_011 = "0.11"; + public static final String CONNECTOR_TOPIC = "connector.topic"; + public static final String CONNECTOR_STARTUP_MODE = "connector.startup-mode"; + public static final String CONNECTOR_STARTUP_MODE_VALUE_EARLIEST = "earliest-offset"; + public static final String CONNECTOR_STARTUP_MODE_VALUE_LATEST = "latest-offset"; + public static final String CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS = "group-offsets"; + public static final String CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS = "specific-offsets"; + public static final String CONNECTOR_SPECIFIC_OFFSETS = "connector.specific-offsets"; + public static final String CONNECTOR_SPECIFIC_OFFSETS_PARTITION = "partition"; + public static final String CONNECTOR_SPECIFIC_OFFSETS_OFFSET = "offset"; + public static final String CONNECTOR_PROPERTIES = "connector.properties"; + public static final String CONNECTOR_PROPERTIES_KEY = "key"; + public static final String CONNECTOR_PROPERTIES_VALUE = "value"; @Override public void validate(DescriptorProperties properties) { super.validate(properties); - - AbstractFunction0 emptyValidator = new AbstractFunction0() { - @Override - public BoxedUnit apply() { - return BoxedUnit.UNIT; - } - }; - - properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE, false); - - AbstractFunction0 version08Validator = new AbstractFunction0() { - @Override - public BoxedUnit apply() { - properties.validateString(ZOOKEEPER_CONNECT, false, 0, Integer.MAX_VALUE); - return BoxedUnit.UNIT; - } - }; - - Map> versionValidatorMap = new HashMap<>(); - versionValidatorMap.put(KAFKA_VERSION_VALUE_08, version08Validator); - versionValidatorMap.put(KAFKA_VERSION_VALUE_09, emptyValidator); - versionValidatorMap.put(KAFKA_VERSION_VALUE_010, emptyValidator); - versionValidatorMap.put(KAFKA_VERSION_VALUE_011, emptyValidator); - properties.validateEnum( - KAFKA_VERSION, + properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_KAFKA, false); + + final List versions = Arrays.asList( + CONNECTOR_VERSION_VALUE_08, + CONNECTOR_VERSION_VALUE_09, + CONNECTOR_VERSION_VALUE_010, + CONNECTOR_VERSION_VALUE_011); + properties.validateEnumValues(CONNECTOR_VERSION(), false, versions); + properties.validateString(CONNECTOR_TOPIC, false, 1, Integer.MAX_VALUE); + + final Map> specificOffsetValidators = new HashMap<>(); + specificOffsetValidators.put( + CONNECTOR_SPECIFIC_OFFSETS_PARTITION, + (prefix) -> properties.validateInt( + prefix + CONNECTOR_SPECIFIC_OFFSETS_PARTITION, false, - toScalaImmutableMap(versionValidatorMap) - ); - - properties.validateString(BOOTSTRAP_SERVERS, false, 1, Integer.MAX_VALUE); - properties.validateString(GROUP_ID, false, 1, Integer.MAX_VALUE); - properties.validateString(TOPIC, false, 1, Integer.MAX_VALUE); - - AbstractFunction0 specificOffsetsValidator = new AbstractFunction0() { - @Override - public BoxedUnit apply() { - Map partitions = JavaConversions.mapAsJavaMap( - properties.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION)); - - Map offsets = JavaConversions.mapAsJavaMap( - properties.getIndexedProperty(SPECIFIC_OFFSETS, OFFSET)); - if (partitions.isEmpty() || offsets.isEmpty()) { - throw new ValidationException("Offsets must be set for SPECIFIC_OFFSETS mode."); - } - for (int i = 0; i < partitions.size(); ++i) { - properties.validateInt( - SPECIFIC_OFFSETS + "." + i + "." + PARTITION, - false, - 0, - Integer.MAX_VALUE); - properties.validateLong( - SPECIFIC_OFFSETS + "." + i + "." + OFFSET, - false, - 0, - Long.MAX_VALUE); - } - return BoxedUnit.UNIT; - } - }; - Map> startupModeValidatorMap = new HashMap<>(); - startupModeValidatorMap.put(STARTUP_MODE_VALUE_GROUP_OFFSETS, emptyValidator); - startupModeValidatorMap.put(STARTUP_MODE_VALUE_EARLIEST, emptyValidator); - startupModeValidatorMap.put(STARTUP_MODE_VALUE_LATEST, emptyValidator); - startupModeValidatorMap.put(STARTUP_MODE_VALUE_SPECIFIC_OFFSETS, specificOffsetsValidator); - - properties.validateEnum(STARTUP_MODE, true, toScalaImmutableMap(startupModeValidatorMap)); - validateTableJsonMapping(properties); + 0, + Integer.MAX_VALUE)); + specificOffsetValidators.put( + CONNECTOR_SPECIFIC_OFFSETS_OFFSET, + (prefix) -> properties.validateLong( + prefix + CONNECTOR_SPECIFIC_OFFSETS_OFFSET, + false, + 0, + Long.MAX_VALUE)); + + final Map> startupModeValidation = new HashMap<>(); + startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS, properties.noValidation()); + startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_EARLIEST, properties.noValidation()); + startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_LATEST, properties.noValidation()); + startupModeValidation.put( + CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS, + prefix -> properties.validateFixedIndexedProperties(CONNECTOR_SPECIFIC_OFFSETS, false, specificOffsetValidators)); + properties.validateEnum(CONNECTOR_STARTUP_MODE, true, startupModeValidation); + + final Map> propertyValidators = new HashMap<>(); + propertyValidators.put( + CONNECTOR_PROPERTIES_KEY, + prefix -> properties.validateString(prefix + CONNECTOR_PROPERTIES_KEY, false, 1, Integer.MAX_VALUE)); + propertyValidators.put( + CONNECTOR_PROPERTIES_VALUE, + prefix -> properties.validateString(prefix + CONNECTOR_PROPERTIES_VALUE, false, 0, Integer.MAX_VALUE)); + properties.validateFixedIndexedProperties(CONNECTOR_PROPERTIES, true, propertyValidators); } - private void validateTableJsonMapping(DescriptorProperties properties) { - Map mappingTableField = JavaConversions.mapAsJavaMap( - properties.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD)); - Map mappingJsonField = JavaConversions.mapAsJavaMap( - properties.getIndexedProperty(TABLE_JSON_MAPPING, JSON_FIELD)); - - if (mappingJsonField.size() != mappingJsonField.size()) { - throw new ValidationException("Table JSON mapping must be one to one."); - } - - for (int i = 0; i < mappingTableField.size(); i++) { - properties.validateString( - TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD, - false, - 1, - Integer.MAX_VALUE); - properties.validateString( - TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD, - false, - 1, - Integer.MAX_VALUE); - } - } + // utilities - @SuppressWarnings("unchecked") - private scala.collection.immutable.Map toScalaImmutableMap(Map javaMap) { - final java.util.List> list = new java.util.ArrayList<>(javaMap.size()); - for (final java.util.Map.Entry entry : javaMap.entrySet()) { - list.add(scala.Tuple2.apply(entry.getKey(), entry.getValue())); + public static String normalizeStartupMode(StartupMode startupMode) { + switch (startupMode) { + case EARLIEST: + return CONNECTOR_STARTUP_MODE_VALUE_EARLIEST; + case LATEST: + return CONNECTOR_STARTUP_MODE_VALUE_LATEST; + case GROUP_OFFSETS: + return CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS; + case SPECIFIC_OFFSETS: + return CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS; } - final scala.collection.Seq> seq = - scala.collection.JavaConverters.asScalaBufferConverter(list).asScala().toSeq(); - return (scala.collection.immutable.Map) scala.collection.immutable.Map$.MODULE$.apply(seq); + throw new IllegalArgumentException("Invalid startup mode."); } } diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java deleted file mode 100644 index 964a62425481e3..00000000000000 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.streaming.connectors.kafka; - -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; -import org.apache.flink.table.descriptors.Kafka; - -import org.mockito.Mockito; - -/** - * Tests for {@link KafkaJsonTableSourceFactory}. - */ -public abstract class KafkaJsonTableFromDescriptorTestBase { - private static final String GROUP_ID = "test-group"; - private static final String BOOTSTRAP_SERVERS = "localhost:1234"; - private static final String TOPIC = "test-topic"; - - protected abstract String versionForTest(); - - protected abstract KafkaJsonTableSource.Builder builderForTest(); - - protected abstract void extraSettings(KafkaTableSource.Builder builder, Kafka kafka); - - private static StreamExecutionEnvironment env = Mockito.mock(StreamExecutionEnvironment.class); - private static StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env); - -// @Test -// public void buildJsonTableSourceTest() throws Exception { -// final URL url = getClass().getClassLoader().getResource("kafka-json-schema.json"); -// Objects.requireNonNull(url); -// final String schema = FileUtils.readFileUtf8(new File(url.getFile())); -// -// Map tableJsonMapping = new HashMap<>(); -// tableJsonMapping.put("fruit-name", "name"); -// tableJsonMapping.put("fruit-count", "count"); -// tableJsonMapping.put("event-time", "time"); -// -// // Construct with the builder. -// Properties props = new Properties(); -// props.put("group.id", GROUP_ID); -// props.put("bootstrap.servers", BOOTSTRAP_SERVERS); -// -// Map specificOffsets = new HashMap<>(); -// specificOffsets.put(new KafkaTopicPartition(TOPIC, 0), 100L); -// specificOffsets.put(new KafkaTopicPartition(TOPIC, 1), 123L); -// -// KafkaTableSource.Builder builder = builderForTest() -// .forJsonSchema(TableSchema.fromTypeInfo(JsonSchemaConverter.convert(schema))) -// .failOnMissingField(true) -// .withTableToJsonMapping(tableJsonMapping) -// .withKafkaProperties(props) -// .forTopic(TOPIC) -// .fromSpecificOffsets(specificOffsets) -// .withSchema( -// TableSchema.builder() -// .field("fruit-name", Types.STRING) -// .field("fruit-count", Types.INT) -// .field("event-time", Types.LONG) -// .field("proc-time", Types.SQL_TIMESTAMP) -// .build()) -// .withProctimeAttribute("proc-time"); -// -// // Construct with the descriptor. -// Map offsets = new HashMap<>(); -// offsets.put(0, 100L); -// offsets.put(1, 123L); -// Kafka kafka = new Kafka() -// .version(versionForTest()) -// .groupId(GROUP_ID) -// .bootstrapServers(BOOTSTRAP_SERVERS) -// .topic(TOPIC) -// .startupMode(StartupMode.SPECIFIC_OFFSETS) -// .specificOffsets(offsets) -// .tableJsonMapping(tableJsonMapping); -// extraSettings(builder, kafka); -// -// TableSource source = tEnv -// .from(kafka) -// .withFormat( -// new Json() -// .schema(schema) -// .failOnMissingField(true)) -// .withSchema(new Schema() -// .field("fruit-name", Types.STRING) -// .field("fruit-count", Types.INT) -// .field("event-time", Types.LONG) -// .field("proc-time", Types.SQL_TIMESTAMP).proctime()) -// .toTableSource(); -// -// Assert.assertEquals(builder.build(), source); -// } - -// @Test(expected = TableException.class) -// public void buildJsonTableSourceFailTest() { -// tEnv.from( -// new Kafka() -// .version(versionForTest()) -// .groupId(GROUP_ID) -// .bootstrapServers(BOOTSTRAP_SERVERS) -// .topic(TOPIC) -// .startupMode(StartupMode.SPECIFIC_OFFSETS) -// .specificOffsets(new HashMap<>())) -// .withFormat( -// new Json() -// .schema("") -// .failOnMissingField(true)) -// .toTableSource(); -// } -} diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java new file mode 100644 index 00000000000000..2b081a9f9157d7 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.kafka; + +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.formats.json.JsonSchemaConverter; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.descriptors.FormatDescriptor; +import org.apache.flink.table.descriptors.Json; +import org.apache.flink.table.descriptors.Kafka; +import org.apache.flink.table.descriptors.Schema; +import org.apache.flink.table.descriptors.TestTableSourceDescriptor; +import org.apache.flink.table.sources.TableSource; +import org.apache.flink.table.sources.TableSourceFactoryService; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +/** + * Tests for {@link KafkaJsonTableSourceFactory}. + */ +public abstract class KafkaJsonTableSourceFactoryTestBase { + + private static final String JSON_SCHEMA = + "{" + + " 'title': 'Fruit'," + + " 'type': 'object'," + + " 'properties': {" + + " 'name': {" + + " 'type': 'string'" + + " }," + + " 'count': {" + + " 'type': 'integer'" + + " }," + + " 'time': {" + + " 'description': 'Age in years'," + + " 'type': 'number'" + + " }" + " }," + + " 'required': ['name', 'count', 'time']" + + "}"; + + private static final String TOPIC = "test-topic"; + + protected abstract String version(); + + protected abstract KafkaJsonTableSource.Builder builder(); + + @Test + public void testTableSourceFromJsonSchema() { + testTableSource( + new Json() + .jsonSchema(JSON_SCHEMA) + .failOnMissingField(true) + ); + } + + @Test + public void testTableSourceDerivedSchema() { + testTableSource( + new Json() + .deriveSchema() + .failOnMissingField(true) + ); + } + + private void testTableSource(FormatDescriptor format) { + // construct table source using a builder + + final Map tableJsonMapping = new HashMap<>(); + tableJsonMapping.put("fruit-name", "name"); + tableJsonMapping.put("count", "count"); + tableJsonMapping.put("event-time", "time"); + + final Properties props = new Properties(); + props.put("group.id", "test-group"); + props.put("bootstrap.servers", "localhost:1234"); + + final Map specificOffsets = new HashMap<>(); + specificOffsets.put(new KafkaTopicPartition(TOPIC, 0), 100L); + specificOffsets.put(new KafkaTopicPartition(TOPIC, 1), 123L); + + final KafkaTableSource builderSource = builder() + .forJsonSchema(TableSchema.fromTypeInfo(JsonSchemaConverter.convert(JSON_SCHEMA))) + .failOnMissingField(true) + .withTableToJsonMapping(tableJsonMapping) + .withKafkaProperties(props) + .forTopic(TOPIC) + .fromSpecificOffsets(specificOffsets) + .withSchema( + TableSchema.builder() + .field("fruit-name", Types.STRING) + .field("count", Types.BIG_INT) + .field("event-time", Types.BIG_DEC) + .field("proc-time", Types.SQL_TIMESTAMP) + .build()) + .withProctimeAttribute("proc-time") + .build(); + + // construct table source using descriptors and table source factory + + final Map offsets = new HashMap<>(); + offsets.put(0, 100L); + offsets.put(1, 123L); + + final TestTableSourceDescriptor testDesc = new TestTableSourceDescriptor( + new Kafka() + .version(version()) + .topic(TOPIC) + .properties(props) + .startFromSpecificOffsets(offsets)) + .addFormat(format) + .addSchema( + new Schema() + .field("fruit-name", Types.STRING).from("name") + .field("count", Types.BIG_INT) // no from so it must match with the input + .field("event-time", Types.BIG_DEC).from("time") + .field("proc-time", Types.SQL_TIMESTAMP).proctime()); + + final TableSource factorySource = TableSourceFactoryService.findAndCreateTableSource(testDesc); + + assertEquals(builderSource, factorySource); + } +} diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java new file mode 100644 index 00000000000000..f3d96f1c443aa4 --- /dev/null +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * Tests for the {@link Kafka} descriptor. + */ +public class KafkaTest extends DescriptorTestBase { + + @Override + public List descriptors() { + final Descriptor earliestDesc = + new Kafka() + .version("0.8") + .startFromEarliest() + .topic("WhateverTopic"); + + final Descriptor specificOffsetsDesc = + new Kafka() + .version("0.11") + .topic("MyTable") + .startFromSpecificOffset(0, 42L) + .startFromSpecificOffset(1, 300L) + .property("zookeeper.stuff", "12") + .property("kafka.stuff", "42"); + + final Map offsets = new HashMap<>(); + offsets.put(0, 42L); + offsets.put(1, 300L); + + final Properties properties = new Properties(); + properties.put("zookeeper.stuff", "12"); + properties.put("kafka.stuff", "42"); + + final Descriptor specificOffsetsMapDesc = + new Kafka() + .version("0.11") + .topic("MyTable") + .startFromSpecificOffsets(offsets) + .properties(properties); + + return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc); + } + + @Override + public List> properties() { + final Map props1 = new HashMap<>(); + props1.put("connector.property-version", "1"); + props1.put("connector.type", "kafka"); + props1.put("connector.version", "0.8"); + props1.put("connector.topic", "WhateverTopic"); + props1.put("connector.startup-mode", "earliest-offset"); + + final Map props2 = new HashMap<>(); + props2.put("connector.property-version", "1"); + props2.put("connector.type", "kafka"); + props2.put("connector.version", "0.11"); + props2.put("connector.topic", "MyTable"); + props2.put("connector.startup-mode", "specific-offsets"); + props2.put("connector.specific-offsets.0.partition", "0"); + props2.put("connector.specific-offsets.0.offset", "42"); + props2.put("connector.specific-offsets.1.partition", "1"); + props2.put("connector.specific-offsets.1.offset", "300"); + props2.put("connector.properties.0.key", "zookeeper.stuff"); + props2.put("connector.properties.0.value", "12"); + props2.put("connector.properties.1.key", "kafka.stuff"); + props2.put("connector.properties.1.value", "42"); + + final Map props3 = new HashMap<>(); + props3.put("connector.property-version", "1"); + props3.put("connector.type", "kafka"); + props3.put("connector.version", "0.11"); + props3.put("connector.topic", "MyTable"); + props3.put("connector.startup-mode", "specific-offsets"); + props3.put("connector.specific-offsets.0.partition", "0"); + props3.put("connector.specific-offsets.0.offset", "42"); + props3.put("connector.specific-offsets.1.partition", "1"); + props3.put("connector.specific-offsets.1.offset", "300"); + props3.put("connector.properties.0.key", "zookeeper.stuff"); + props3.put("connector.properties.0.value", "12"); + props3.put("connector.properties.1.key", "kafka.stuff"); + props3.put("connector.properties.1.value", "42"); + + return Arrays.asList(props1, props2, props3); + } + + @Override + public DescriptorValidator validator() { + return new KafkaValidator(); + } +} diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json deleted file mode 100644 index 5167e5e8724057..00000000000000 --- a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -{ - "title": "Fruit", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "count": { - "type": "integer" - }, - "time": { - "description": "Age in years", - "type": "number" - } - }, - "required": ["name", "count", "time"] -} diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index 019047c56b5a6a..d8477fe2ec6a9b 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -337,22 +337,6 @@ under the License. - - - org.apache.flink - flink-avro - ${project.version} - provided - - - - org.apache.flink - flink-json - ${project.version} - provided - - - diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml index d0f55ab43efa99..3a80b0eaae9fe2 100644 --- a/flink-formats/flink-json/pom.xml +++ b/flink-formats/flink-json/pom.xml @@ -50,6 +50,33 @@ under the License. ${project.version} provided - + + org.apache.flink + + flink-table_2.11 + ${project.version} + provided + + true + + + + + + org.apache.flink + + flink-table_2.11 + ${project.version} + test-jar + test + + + + + org.scala-lang + scala-compiler + test + + diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java new file mode 100644 index 00000000000000..9c121916d5416a --- /dev/null +++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.typeutils.TypeStringUtils; +import org.apache.flink.util.Preconditions; + +import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_JSON_SCHEMA; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA; +import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_TYPE_VALUE; + +/** + * Format descriptor for JSON. + */ +public class Json extends FormatDescriptor { + + private Boolean failOnMissingField; + private Boolean deriveSchema; + private String jsonSchema; + private String schema; + + /** + * Format descriptor for JSON. + */ + public Json() { + super(FORMAT_TYPE_VALUE, 1); + } + + /** + * Sets flag whether to fail if a field is missing or not. + * + * @param failOnMissingField If set to true, the operation fails if there is a missing field. + * If set to false, a missing field is set to null. + */ + public Json failOnMissingField(boolean failOnMissingField) { + this.failOnMissingField = failOnMissingField; + return this; + } + + /** + * Sets the JSON schema string with field names and the types according to the JSON schema + * specification [[http://json-schema.org/specification.html]]. + * + *

    The schema might be nested. + * + * @param jsonSchema JSON schema + */ + public Json jsonSchema(String jsonSchema) { + Preconditions.checkNotNull(jsonSchema); + this.jsonSchema = jsonSchema; + this.schema = null; + this.deriveSchema = null; + return this; + } + + /** + * Sets the schema using type information. + * + *

    JSON objects are represented as ROW types. + * + *

    The schema might be nested. + * + * @param schemaType type information that describes the schema + */ + public Json schema(TypeInformation schemaType) { + Preconditions.checkNotNull(schemaType); + this.schema = TypeStringUtils.writeTypeInfo(schemaType); + this.jsonSchema = null; + this.deriveSchema = null; + return this; + } + + /** + * Derives the format schema from the table's schema described using {@link Schema}. + * + *

    This allows for defining schema information only once. + * + *

    The names, types, and field order of the format are determined by the table's + * schema. Time attributes are ignored. A "from" definition is interpreted as a field renaming + * in the format. + */ + public Json deriveSchema() { + this.deriveSchema = true; + this.schema = null; + this.jsonSchema = null; + return this; + } + + /** + * Internal method for format properties conversion. + */ + @Override + public void addFormatProperties(DescriptorProperties properties) { + if (deriveSchema != null) { + properties.putBoolean(FORMAT_DERIVE_SCHEMA(), deriveSchema); + } + + if (jsonSchema != null) { + properties.putString(FORMAT_JSON_SCHEMA, jsonSchema); + } + + if (schema != null) { + properties.putString(FORMAT_SCHEMA, schema); + } + + if (failOnMissingField != null) { + properties.putBoolean(FORMAT_FAIL_ON_MISSING_FIELD, failOnMissingField); + } + } +} diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java new file mode 100644 index 00000000000000..fea7cf55b5d339 --- /dev/null +++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors; + +import org.apache.flink.table.api.ValidationException; + +/** + * Validator for {@link Json}. + */ +public class JsonValidator extends FormatDescriptorValidator { + + public static final String FORMAT_TYPE_VALUE = "json"; + public static final String FORMAT_SCHEMA = "format.schema"; + public static final String FORMAT_JSON_SCHEMA = "format.json-schema"; + public static final String FORMAT_FAIL_ON_MISSING_FIELD = "format.fail-on-missing-field"; + + @Override + public void validate(DescriptorProperties properties) { + super.validate(properties); + properties.validateBoolean(FORMAT_DERIVE_SCHEMA(), true); + final boolean deriveSchema = properties.getOptionalBoolean(FORMAT_DERIVE_SCHEMA()).orElse(false); + final boolean hasSchema = properties.containsKey(FORMAT_SCHEMA); + final boolean hasSchemaString = properties.containsKey(FORMAT_JSON_SCHEMA); + if (deriveSchema && (hasSchema || hasSchemaString)) { + throw new ValidationException( + "Format cannot define a schema and derive from the table's schema at the same time."); + } else if (!deriveSchema && hasSchema && hasSchemaString) { + throw new ValidationException("A definition of both a schema and JSON schema is not allowed."); + } else if (!deriveSchema && !hasSchema && !hasSchemaString) { + throw new ValidationException("A definition of a schema or JSON schema is required."); + } else if (hasSchema) { + properties.validateType(FORMAT_SCHEMA, false); + } else if (hasSchemaString) { + properties.validateString(FORMAT_JSON_SCHEMA, false, 1); + } + + properties.validateBoolean(FORMAT_FAIL_ON_MISSING_FIELD, true); + } +} diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java new file mode 100644 index 00000000000000..6e370a02c13ed5 --- /dev/null +++ b/flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.api.Types; +import org.apache.flink.table.api.ValidationException; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests for the {@link Json} descriptor. + */ +public class JsonTest extends DescriptorTestBase { + + private static final String JSON_SCHEMA = + "{" + + " 'title': 'Person'," + + " 'type': 'object'," + + " 'properties': {" + + " 'firstName': {" + + " 'type': 'string'" + + " }," + + " 'lastName': {" + + " 'type': 'string'" + + " }," + + " 'age': {" + + " 'description': 'Age in years'," + + " 'type': 'integer'," + + " 'minimum': 0" + + " }" + + " }," + + " 'required': ['firstName', 'lastName']" + + "}"; + + @Test(expected = ValidationException.class) + public void testInvalidMissingField() { + addPropertyAndVerify(descriptors().get(0), "format.fail-on-missing-field", "DDD"); + } + + @Test(expected = ValidationException.class) + public void testMissingSchema() { + removePropertyAndVerify(descriptors().get(0), "format.json-schema"); + } + + @Test(expected = ValidationException.class) + public void testDuplicateSchema() { + // we add an additional non-json schema + addPropertyAndVerify(descriptors().get(0), "format.schema", "DDD"); + } + + // -------------------------------------------------------------------------------------------- + + @Override + public List descriptors() { + final Descriptor desc1 = new Json().jsonSchema("test"); + + final Descriptor desc2 = new Json().jsonSchema(JSON_SCHEMA).failOnMissingField(true); + + final Descriptor desc3 = new Json() + .schema( + Types.ROW( + new String[]{"test1", "test2"}, + new TypeInformation[]{Types.STRING(), Types.SQL_TIMESTAMP()})) + .failOnMissingField(true); + + final Descriptor desc4 = new Json().deriveSchema(); + + return Arrays.asList(desc1, desc2, desc3, desc4); + } + + @Override + public List> properties() { + final Map props1 = new HashMap<>(); + props1.put("format.type", "json"); + props1.put("format.property-version", "1"); + props1.put("format.json-schema", "test"); + + final Map props2 = new HashMap<>(); + props2.put("format.type", "json"); + props2.put("format.property-version", "1"); + props2.put("format.json-schema", JSON_SCHEMA); + props2.put("format.fail-on-missing-field", "true"); + + final Map props3 = new HashMap<>(); + props3.put("format.type", "json"); + props3.put("format.property-version", "1"); + props3.put("format.schema", "ROW(test1 VARCHAR, test2 TIMESTAMP)"); + props3.put("format.fail-on-missing-field", "true"); + + final Map props4 = new HashMap<>(); + props4.put("format.type", "json"); + props4.put("format.property-version", "1"); + props4.put("format.derive-schema", "true"); + + return Arrays.asList(props1, props2, props3, props4); + } + + @Override + public DescriptorValidator validator() { + return new JsonValidator(); + } +} diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java index 9e7413c2d3e1a4..8c40885d36a72f 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java @@ -476,7 +476,7 @@ private TableEnvironment createTableEnvironment(Environment env) { } env.getSources().forEach((name, source) -> { - TableSource tableSource = TableSourceFactoryService.findTableSourceFactory(source); + TableSource tableSource = TableSourceFactoryService.findAndCreateTableSource(source); tableEnv.registerTableSource(name, tableSource); }); diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml index b4b0eee0050736..53a3233d80f72c 100644 --- a/flink-libraries/flink-table/pom.xml +++ b/flink-libraries/flink-table/pom.xml @@ -317,6 +317,18 @@ under the License. + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + org.scalastyle diff --git a/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties b/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties index 86a48a8667b0e6..ec4657988ad620 100644 --- a/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties +++ b/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties @@ -16,6 +16,13 @@ # limitations under the License. ################################################################################ +################################################################################ +# NOTE: THIS APPROACH IS DEPRECATED AND WILL BE REMOVED IN FUTURE VERSIONS! +# +# We recommend to use a org.apache.flink.table.sources.TableSourceFactory +# instead. They allow to define new factories by using Java Service Providers. +################################################################################ + ################################################################################ # The config file is used to specify the packages of current module where # to find TableSourceConverter implementation class annotated with TableType. diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala index 1e88d932ed6f99..6958b3d15da8af 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala @@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.common.typeutils.CompositeType import _root_.scala.collection.mutable.ArrayBuffer +import _root_.java.util.Objects /** * A TableSchema represents a Table's structure. @@ -94,7 +95,7 @@ class TableSchema( /** * Returns the number of columns. */ - def getColumnNum: Int = columnNames.length + def getColumnCount: Int = columnNames.length /** * Returns all column names as an array. @@ -134,6 +135,9 @@ class TableSchema( def canEqual(other: Any): Boolean = other.isInstanceOf[TableSchema] + override def hashCode(): Int = { + Objects.hash(columnNames, columnTypes) + } } object TableSchema { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala index fc7f7a39eab7f3..ef14b8af259f2a 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala @@ -23,6 +23,7 @@ import java.util.{HashMap => JHashMap, Map => JMap} import org.apache.flink.table.api.{TableException, TableSchema} import org.apache.flink.table.catalog.ExternalCatalogTable._ +import org.apache.flink.table.descriptors.DescriptorProperties.toScala import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, METADATA_CREATION_TIME, METADATA_LAST_ACCESS_TIME} import org.apache.flink.table.descriptors._ import org.apache.flink.table.plan.stats.TableStats @@ -73,8 +74,7 @@ class ExternalCatalogTable( lazy val tableType: String = { val props = new DescriptorProperties() connectorDesc.addProperties(props) - props - .getString(CONNECTOR_LEGACY_TYPE) + toScala(props.getOptionalString(CONNECTOR_LEGACY_TYPE)) .getOrElse(throw new TableException("Could not find a legacy table type to return.")) } @@ -88,8 +88,7 @@ class ExternalCatalogTable( lazy val schema: TableSchema = { val props = new DescriptorProperties() connectorDesc.addProperties(props) - props - .getTableSchema(CONNECTOR_LEGACY_SCHEMA) + toScala(props.getOptionalTableSchema(CONNECTOR_LEGACY_SCHEMA)) .getOrElse(throw new TableException("Could not find a legacy schema to return.")) } @@ -105,7 +104,7 @@ class ExternalCatalogTable( val props = new DescriptorProperties(normalizeKeys = false) val legacyProps = new JHashMap[String, String]() connectorDesc.addProperties(props) - props.asMap.flatMap { case (k, v) => + props.asMap.asScala.flatMap { case (k, v) => if (k.startsWith(CONNECTOR_LEGACY_PROPERTY)) { // remove "connector.legacy-property-" Some(legacyProps.put(k.substring(CONNECTOR_LEGACY_PROPERTY.length + 1), v)) @@ -138,7 +137,7 @@ class ExternalCatalogTable( metadataDesc match { case Some(meta) => meta.addProperties(normalizedProps) - normalizedProps.getString(METADATA_COMMENT).orNull + normalizedProps.getOptionalString(METADATA_COMMENT).orElse(null) case None => null } @@ -157,7 +156,7 @@ class ExternalCatalogTable( metadataDesc match { case Some(meta) => meta.addProperties(normalizedProps) - normalizedProps.getLong(METADATA_CREATION_TIME).map(v => Long.box(v)).orNull + normalizedProps.getOptionalLong(METADATA_CREATION_TIME).orElse(null) case None => null } @@ -176,7 +175,7 @@ class ExternalCatalogTable( metadataDesc match { case Some(meta) => meta.addProperties(normalizedProps) - normalizedProps.getLong(METADATA_LAST_ACCESS_TIME).map(v => Long.box(v)).orNull + normalizedProps.getOptionalLong(METADATA_LAST_ACCESS_TIME).orElse(null) case None => null } @@ -267,7 +266,7 @@ object ExternalCatalogTable { tableType: String, schema: TableSchema, legacyProperties: JMap[String, String]) - extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1) { + extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1, formatNeeded = false) { override protected def addConnectorProperties(properties: DescriptorProperties): Unit = { properties.putString(CONNECTOR_LEGACY_TYPE, tableType) @@ -276,8 +275,6 @@ object ExternalCatalogTable { properties.putString(s"$CONNECTOR_LEGACY_PROPERTY-$k", v) } } - - override private[flink] def needsFormat() = false } def toConnectorDescriptor( diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala index 3bc5dc067b4d75..2288522b8e4897 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala @@ -58,7 +58,7 @@ object ExternalTableSourceUtil extends Logging { } // use the factory approach else { - val source = TableSourceFactoryService.findTableSourceFactory(externalCatalogTable) + val source = TableSourceFactoryService.findAndCreateTableSource(externalCatalogTable) tableEnv match { // check for a batch table source in this batch environment case _: BatchTableEnvironment => diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala index ed9ee7dbb97900..afdd84c78a9c2e 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala @@ -43,7 +43,7 @@ class BatchTableSourceDescriptor(tableEnv: BatchTableEnvironment, connector: Con * Searches for the specified table source, configures it accordingly, and returns it. */ def toTableSource: TableSource[_] = { - val source = TableSourceFactoryService.findTableSourceFactory(this) + val source = TableSourceFactoryService.findAndCreateTableSource(this) source match { case _: BatchTableSource[_] => source case _ => throw new TableException( diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala index f691b4fd104b52..dc344f31bb594d 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.descriptors -import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION} +import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION} /** * Describes a connector to an other system. @@ -27,7 +27,8 @@ import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTO */ abstract class ConnectorDescriptor( private val tpe: String, - private val version: Int) + private val version: Int, + private val formatNeeded: Boolean) extends Descriptor { override def toString: String = this.getClass.getSimpleName @@ -37,7 +38,7 @@ abstract class ConnectorDescriptor( */ final private[flink] def addProperties(properties: DescriptorProperties): Unit = { properties.putString(CONNECTOR_TYPE, tpe) - properties.putLong(CONNECTOR_VERSION, version) + properties.putLong(CONNECTOR_PROPERTY_VERSION, version) addConnectorProperties(properties) } @@ -49,6 +50,6 @@ abstract class ConnectorDescriptor( /** * Internal method that defines if this connector requires a format descriptor. */ - private[flink] def needsFormat(): Boolean + private[flink] def needsFormat(): Boolean = formatNeeded } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala index 8ab0f45fa64451..211d374de58aa7 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.descriptors -import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION} +import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION} /** * Validator for [[ConnectorDescriptor]]. @@ -27,13 +27,27 @@ class ConnectorDescriptorValidator extends DescriptorValidator { override def validate(properties: DescriptorProperties): Unit = { properties.validateString(CONNECTOR_TYPE, isOptional = false, minLen = 1) - properties.validateInt(CONNECTOR_VERSION, isOptional = true, 0, Integer.MAX_VALUE) + properties.validateInt(CONNECTOR_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE) } } object ConnectorDescriptorValidator { + /** + * Key for describing the type of the connector. Usually used for factory discovery. + */ val CONNECTOR_TYPE = "connector.type" + + /** + * Key for describing the property version. This property can be used for backwards + * compatibility in case the property format changes. + */ + val CONNECTOR_PROPERTY_VERSION = "connector.property-version" + + /** + * Key for describing the version of the connector. This property can be used for different + * connector versions (e.g. Kafka 0.8 or Kafka 0.11). + */ val CONNECTOR_VERSION = "connector.version" } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala index 0493d9912a9a3c..7e69feabf54336 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala @@ -23,6 +23,7 @@ import org.apache.flink.table.api.{TableSchema, ValidationException} import org.apache.flink.table.descriptors.CsvValidator._ import scala.collection.mutable +import scala.collection.JavaConverters._ /** * Format descriptor for comma-separated values (CSV). @@ -31,7 +32,7 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) { private var fieldDelim: Option[String] = None private var lineDelim: Option[String] = None - private val formatSchema: mutable.LinkedHashMap[String, String] = + private val schema: mutable.LinkedHashMap[String, String] = mutable.LinkedHashMap[String, String]() private var quoteCharacter: Option[Character] = None private var commentPrefix: Option[String] = None @@ -67,7 +68,7 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) { * @param schema the table schema */ def schema(schema: TableSchema): Csv = { - this.formatSchema.clear() + this.schema.clear() DescriptorProperties.normalizeTableSchema(schema).foreach { case (n, t) => field(n, t) } @@ -96,10 +97,10 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) { * @param fieldType the type string of the field */ def field(fieldName: String, fieldType: String): Csv = { - if (formatSchema.contains(fieldName)) { + if (schema.contains(fieldName)) { throw new ValidationException(s"Duplicate field name $fieldName.") } - formatSchema += (fieldName -> fieldType) + schema += (fieldName -> fieldType) this } @@ -145,7 +146,9 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) { override protected def addFormatProperties(properties: DescriptorProperties): Unit = { fieldDelim.foreach(properties.putString(FORMAT_FIELD_DELIMITER, _)) lineDelim.foreach(properties.putString(FORMAT_LINE_DELIMITER, _)) - properties.putTableSchema(FORMAT_FIELDS, formatSchema.toIndexedSeq) + properties.putTableSchema( + FORMAT_FIELDS, + schema.toIndexedSeq.map(DescriptorProperties.toJava[String, String]).asJava) quoteCharacter.foreach(properties.putCharacter(FORMAT_QUOTE_CHARACTER, _)) commentPrefix.foreach(properties.putString(FORMAT_COMMENT_PREFIX, _)) isIgnoreFirstLine.foreach(properties.putBoolean(FORMAT_IGNORE_FIRST_LINE, _)) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala index d11273251c9436..555d92db6e6817 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala @@ -21,24 +21,32 @@ package org.apache.flink.table.descriptors import java.io.Serializable import java.lang.{Boolean => JBoolean, Double => JDouble, Integer => JInt, Long => JLong} import java.util +import java.util.function.{Consumer, Supplier} import java.util.regex.Pattern +import java.util.{Optional, List => JList, Map => JMap} import org.apache.commons.codec.binary.Base64 import org.apache.commons.lang.StringEscapeUtils import org.apache.flink.api.common.typeinfo.TypeInformation -import org.apache.flink.table.api.{TableSchema, ValidationException} -import org.apache.flink.table.descriptors.DescriptorProperties.{NAME, TYPE, normalizeTableSchema} +import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2} +import org.apache.flink.table.api.{TableException, TableSchema, ValidationException} +import org.apache.flink.table.descriptors.DescriptorProperties.{NAME, TYPE, normalizeTableSchema, toJava} import org.apache.flink.table.typeutils.TypeStringUtils import org.apache.flink.util.InstantiationUtil import org.apache.flink.util.Preconditions.checkNotNull -import scala.collection.mutable import scala.collection.JavaConverters._ +import scala.collection.mutable /** * Utility class for having a unified string-based representation of Table API related classes * such as [[TableSchema]], [[TypeInformation]], etc. * + * '''Note to implementers''': Please try to reuse key names as much as possible. Key-names + * should be hierarchical and lower case. Use "-" instead of dots or camel case. + * E.g., connector.schema.start-from = from-earliest. Try not to use the higher level in a + * key-name. E.g., instead of connector.kafka.kafka-version use connector.kafka.version. + * * @param normalizeKeys flag that indicates if keys should be normalized (this flag is * necessary for backwards compatibility) */ @@ -46,39 +54,18 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { private val properties: mutable.Map[String, String] = new mutable.HashMap[String, String]() - private def put(key: String, value: String): Unit = { - if (properties.contains(key)) { - throw new IllegalStateException("Property already present.") - } - if (normalizeKeys) { - properties.put(key.toLowerCase, value) - } else { - properties.put(key, value) - } - } - - // for testing - private[flink] def unsafePut(key: String, value: String): Unit = { - properties.put(key, value) - } - - // for testing - private[flink] def unsafeRemove(key: String): Unit = { - properties.remove(key) - } - - def putProperties(properties: Map[String, String]): Unit = { - properties.foreach { case (k, v) => - put(k, v) - } - } - - def putProperties(properties: util.Map[String, String]): Unit = { + /** + * Adds a set of properties. + */ + def putProperties(properties: JMap[String, String]): Unit = { properties.asScala.foreach { case (k, v) => put(k, v) } } + /** + * Adds a class under the given key. + */ def putClass(key: String, clazz: Class[_]): Unit = { checkNotNull(key) checkNotNull(clazz) @@ -89,43 +76,62 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { put(key, clazz.getName) } + /** + * Adds a string under the given key. + */ def putString(key: String, str: String): Unit = { checkNotNull(key) checkNotNull(str) put(key, str) } + /** + * Adds a boolean under the given key. + */ def putBoolean(key: String, b: Boolean): Unit = { checkNotNull(key) put(key, b.toString) } + /** + * Adds a long under the given key. + */ def putLong(key: String, l: Long): Unit = { checkNotNull(key) put(key, l.toString) } + /** + * Adds an integer under the given key. + */ def putInt(key: String, i: Int): Unit = { checkNotNull(key) put(key, i.toString) } + /** + * Adds a character under the given key. + */ def putCharacter(key: String, c: Character): Unit = { checkNotNull(key) checkNotNull(c) put(key, c.toString) } + /** + * Adds a table schema under the given key. + */ def putTableSchema(key: String, schema: TableSchema): Unit = { + checkNotNull(key) + checkNotNull(schema) putTableSchema(key, normalizeTableSchema(schema)) } - def putTableSchema(key: String, nameAndType: Seq[(String, String)]): Unit = { - putIndexedFixedProperties( - key, - Seq(NAME, TYPE), - nameAndType.map(t => Seq(t._1, t._2)) - ) + /** + * Adds a table schema under the given key. + */ + def putTableSchema(key: String, nameAndType: JList[JTuple2[String, String]]): Unit = { + putTableSchema(key, nameAndType.asScala.map(t => (t.f0, t.f1))) } /** @@ -140,19 +146,12 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { */ def putIndexedFixedProperties( key: String, - propertyKeys: Seq[String], - propertyValues: Seq[Seq[String]]) + propertyKeys: JList[String], + propertyValues: JList[JList[String]]) : Unit = { checkNotNull(key) checkNotNull(propertyValues) - propertyValues.zipWithIndex.foreach { case (values, idx) => - if (values.lengthCompare(propertyKeys.size) != 0) { - throw new ValidationException("Values must have same arity as keys.") - } - values.zipWithIndex.foreach { case (value, keyIdx) => - put(s"$key.$idx.${propertyKeys(keyIdx)}", value) - } - } + putIndexedFixedProperties(key, propertyKeys.asScala, propertyValues.asScala.map(_.asScala)) } /** @@ -167,65 +166,163 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { */ def putIndexedVariableProperties( key: String, - propertySets: Seq[Map[String, String]]) + propertySets: JList[JMap[String, String]]) : Unit = { checkNotNull(key) checkNotNull(propertySets) - propertySets.zipWithIndex.foreach { case (propertySet, idx) => - propertySet.foreach { case (k, v) => - put(s"$key.$idx.$k", v) - } - } + putIndexedVariableProperties(key, propertySets.asScala.map(_.asScala.toMap)) } // ---------------------------------------------------------------------------------------------- - def getString(key: String): Option[String] = { - properties.get(key) + /** + * Returns a string value under the given key if it exists. + */ + def getOptionalString(key: String): Optional[String] = toJava(properties.get(key)) + + /** + * Returns a string value under the given existing key. + */ + def getString(key: String): String = { + get(key) } - def getCharacter(key: String): Option[Character] = getString(key) match { - case Some(c) => + /** + * Returns a character value under the given key if it exists. + */ + def getOptionalCharacter(key: String): Optional[Character] = { + val value = properties.get(key).map { c => if (c.length != 1) { throw new ValidationException(s"The value of $key must only contain one character.") } - Some(c.charAt(0)) + Char.box(c.charAt(0)) + } + toJava(value) + } - case None => None + /** + * Returns a character value under the given existing key. + */ + def getCharacter(key: String): Char = { + getOptionalCharacter(key).orElseThrow(exceptionSupplier(key)) } - def getBoolean(key: String): Option[Boolean] = getString(key) match { - case Some(b) => Some(JBoolean.parseBoolean(b)) + /** + * Returns a class value under the given key if it exists. + */ + def getOptionalClass[T](key: String, superClass: Class[T]): Optional[Class[T]] = { + val value = properties.get(key).map { name => + val clazz = try { + Class.forName( + name, + true, + Thread.currentThread().getContextClassLoader).asInstanceOf[Class[T]] + } catch { + case e: Exception => + throw new ValidationException(s"Could not get class '$name' for key '$key'.", e) + } + if (!superClass.isAssignableFrom(clazz)) { + throw new ValidationException(s"Class '$name' does not extend from the required " + + s"class '${superClass.getName}' for key '$key'.") + } + clazz + } + toJava(value) + } - case None => None + /** + * Returns a class value under the given existing key. + */ + def getClass[T](key: String, superClass: Class[T]): Class[T] = { + getOptionalClass(key, superClass).orElseThrow(exceptionSupplier(key)) } - def getInt(key: String): Option[Int] = getString(key) match { - case Some(l) => Some(JInt.parseInt(l)) + /** + * Returns a boolean value under the given key if it exists. + */ + def getOptionalBoolean(key: String): Optional[JBoolean] = { + val value = properties.get(key).map(JBoolean.parseBoolean(_)).map(Boolean.box) + toJava(value) + } + + /** + * Returns a boolean value under the given existing key. + */ + def getBoolean(key: String): Boolean = { + getOptionalBoolean(key).orElseThrow(exceptionSupplier(key)) + } - case None => None + /** + * Returns an integer value under the given key if it exists. + */ + def getOptionalInt(key: String): Optional[JInt] = { + val value = properties.get(key).map(JInt.parseInt(_)).map(Int.box) + toJava(value) } - def getLong(key: String): Option[Long] = getString(key) match { - case Some(l) => Some(JLong.parseLong(l)) + /** + * Returns an integer value under the given existing key. + */ + def getInt(key: String): Int = { + getOptionalInt(key).orElseThrow(exceptionSupplier(key)) + } - case None => None + /** + * Returns a long value under the given key if it exists. + */ + def getOptionalLong(key: String): Optional[JLong] = { + val value = properties.get(key).map(JLong.parseLong(_)).map(Long.box) + toJava(value) + } + + /** + * Returns a long value under the given existing key. + */ + def getLong(key: String): Long = { + getOptionalLong(key).orElseThrow(exceptionSupplier(key)) + } + + /** + * Returns a double value under the given key if it exists. + */ + def getOptionalDouble(key: String): Optional[JDouble] = { + val value = properties.get(key).map(JDouble.parseDouble(_)).map(Double.box) + toJava(value) + } + + /** + * Returns a double value under the given key if it exists. + */ + def getDouble(key: String): Double = { + getOptionalDouble(key).orElseThrow(exceptionSupplier(key)) } - def getDouble(key: String): Option[Double] = getString(key) match { - case Some(d) => Some(JDouble.parseDouble(d)) + /** + * Returns the type information under the given key if it exists. + */ + def getOptionalType(key: String): Optional[TypeInformation[_]] = { + val value = properties.get(key).map(TypeStringUtils.readTypeInfo) + toJava(value) + } - case None => None + /** + * Returns the type information under the given existing key. + */ + def getType(key: String): TypeInformation[_] = { + getOptionalType(key).orElseThrow(exceptionSupplier(key)) } - def getTableSchema(key: String): Option[TableSchema] = { + /** + * Returns a table schema under the given key if it exists. + */ + def getOptionalTableSchema(key: String): Optional[TableSchema] = { // filter for number of columns val fieldCount = properties .filterKeys(k => k.startsWith(key) && k.endsWith(s".$NAME")) .size if (fieldCount == 0) { - return None + return toJava(None) } // validate fields and build schema @@ -243,16 +340,186 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { ) ) } - Some(schemaBuilder.build()) + toJava(Some(schemaBuilder.build())) + } + + /** + * Returns a table schema under the given existing key. + */ + def getTableSchema(key: String): TableSchema = { + getOptionalTableSchema(key).orElseThrow(exceptionSupplier(key)) + } + + /** + * Returns the property keys of fixed indexed properties. + * + * For example: + * + * schema.fields.0.type = INT, schema.fields.0.name = test + * schema.fields.1.type = LONG, schema.fields.1.name = test2 + * + * getFixedIndexedProperties("schema.fields", List("type", "name")) leads to: + * + * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name") + * 1: Map("type" -> "schema.fields.1.type", "name" -> "schema.fields.1.name") + */ + def getFixedIndexedProperties( + key: String, + propertyKeys: JList[String]) + : JList[JMap[String, String]] = { + + val keys = propertyKeys.asScala + + // filter for index + val escapedKey = Pattern.quote(key) + val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)\\.(.*)") + + // extract index and property keys + val indexes = properties.keys.flatMap { k => + val matcher = pattern.matcher(k) + if (matcher.find()) { + Some(JInt.parseInt(matcher.group(1))) + } else { + None + } + } + + // determine max index + val maxIndex = indexes.reduceOption(_ max _).getOrElse(-1) + + // validate and create result + val list = new util.ArrayList[JMap[String, String]]() + for (i <- 0 to maxIndex) { + val map = new util.HashMap[String, String]() + + keys.foreach { subKey => + val fullKey = s"$key.$i.$subKey" + // check for existence of full key + if (!containsKey(fullKey)) { + throw exceptionSupplier(fullKey).get() + } + map.put(subKey, fullKey) + } + + list.add(map) + } + list + } + + /** + * Returns the property keys of variable indexed properties. + * + * For example: + * + * schema.fields.0.type = INT, schema.fields.0.name = test + * schema.fields.1.type = LONG + * + * getFixedIndexedProperties("schema.fields", List("type")) leads to: + * + * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name") + * 1: Map("type" -> "schema.fields.1.type") + */ + def getVariableIndexedProperties( + key: String, + requiredKeys: JList[String]) + : JList[JMap[String, String]] = { + + val keys = requiredKeys.asScala + + // filter for index + val escapedKey = Pattern.quote(key) + val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)\\.(.*)") + + // extract index and property keys + val indexes = properties.keys.flatMap { k => + val matcher = pattern.matcher(k) + if (matcher.find()) { + Some((JInt.parseInt(matcher.group(1)), matcher.group(2))) + } else { + None + } + } + + // determine max index + val maxIndex = indexes.map(_._1).reduceOption(_ max _).getOrElse(-1) + + // validate and create result + val list = new util.ArrayList[JMap[String, String]]() + for (i <- 0 to maxIndex) { + val map = new util.HashMap[String, String]() + + // check and add required keys + keys.foreach { subKey => + val fullKey = s"$key.$i.$subKey" + // check for existence of full key + if (!containsKey(fullKey)) { + throw exceptionSupplier(fullKey).get() + } + map.put(subKey, fullKey) + } + + // add optional keys + indexes.filter(_._1 == i).foreach { case (_, subKey) => + val fullKey = s"$key.$i.$subKey" + map.put(subKey, fullKey) + } + + list.add(map) + } + list + } + + /** + * Returns all properties under a given key that contains an index in between. + * + * E.g. rowtime.0.name -> returns all rowtime.#.name properties + */ + def getIndexedProperty(key: String, property: String): JMap[String, String] = { + val escapedKey = Pattern.quote(key) + properties.filterKeys(k => k.matches(s"$escapedKey\\.\\d+\\.$property")).asJava + } + + /** + * Returns a prefix subset of properties. + */ + def getPrefix(prefixKey: String): JMap[String, String] = { + val prefix = prefixKey + '.' + properties.filterKeys(_.startsWith(prefix)).toSeq.map{ case (k, v) => + k.substring(prefix.length) -> v // remove prefix + }.toMap.asJava } // ---------------------------------------------------------------------------------------------- + /** + * Validates a string property. + */ + def validateString( + key: String, + isOptional: Boolean) + : Unit = { + validateString(key, isOptional, 0, Integer.MAX_VALUE) + } + + /** + * Validates a string property. The boundaries are inclusive. + */ def validateString( key: String, isOptional: Boolean, - minLen: Int = 0, // inclusive - maxLen: Int = Integer.MAX_VALUE) // inclusive + minLen: Int) // inclusive + : Unit = { + validateString(key, isOptional, minLen, Integer.MAX_VALUE) + } + + /** + * Validates a string property. The boundaries are inclusive. + */ + def validateString( + key: String, + isOptional: Boolean, + minLen: Int, // inclusive + maxLen: Int) // inclusive : Unit = { if (!properties.contains(key)) { @@ -269,11 +536,35 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validates an integer property. + */ + def validateInt( + key: String, + isOptional: Boolean) + : Unit = { + validateInt(key, isOptional, Int.MinValue, Int.MaxValue) + } + + /** + * Validates an integer property. The boundaries are inclusive. + */ def validateInt( key: String, isOptional: Boolean, - min: Int = Int.MinValue, // inclusive - max: Int = Int.MaxValue) // inclusive + min: Int) // inclusive + : Unit = { + validateInt(key, isOptional, min, Int.MaxValue) + } + + /** + * Validates an integer property. The boundaries are inclusive. + */ + def validateInt( + key: String, + isOptional: Boolean, + min: Int, // inclusive + max: Int) // inclusive : Unit = { if (!properties.contains(key)) { @@ -295,11 +586,35 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validates a long property. + */ + def validateLong( + key: String, + isOptional: Boolean) + : Unit = { + validateLong(key, isOptional, Long.MinValue, Long.MaxValue) + } + + /** + * Validates a long property. The boundaries are inclusive. + */ + def validateLong( + key: String, + isOptional: Boolean, + min: Long) // inclusive + : Unit = { + validateLong(key, isOptional, min, Long.MaxValue) + } + + /** + * Validates a long property. The boundaries are inclusive. + */ def validateLong( key: String, isOptional: Boolean, - min: Long = Long.MinValue, // inclusive - max: Long = Long.MaxValue) // inclusive + min: Long, // inclusive + max: Long) // inclusive : Unit = { if (!properties.contains(key)) { @@ -321,6 +636,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validates that a certain value is present under the given key. + */ def validateValue(key: String, value: String, isOptional: Boolean): Unit = { if (!properties.contains(key)) { if (!isOptional) { @@ -334,6 +652,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validates that a boolean value is present under the given key. + */ def validateBoolean(key: String, isOptional: Boolean): Unit = { if (!properties.contains(key)) { if (!isOptional) { @@ -348,11 +669,35 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validates a double property. + */ + def validateDouble( + key: String, + isOptional: Boolean) + : Unit = { + validateDouble(key, isOptional, Double.MinValue, Double.MaxValue) + } + + /** + * Validates a double property. The boundaries are inclusive. + */ + def validateDouble( + key: String, + isOptional: Boolean, + min: Double) // inclusive + : Unit = { + validateDouble(key, isOptional, min, Double.MaxValue) + } + + /** + * Validates a double property. The boundaries are inclusive. + */ def validateDouble( key: String, isOptional: Boolean, - min: Double = Double.MinValue, // inclusive - max: Double = Double.MaxValue) // inclusive + min: Double, // inclusive + max: Double) // inclusive : Unit = { if (!properties.contains(key)) { @@ -374,25 +719,117 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validation for variable indexed properties. + * + * For example: + * + * schema.fields.0.type = INT, schema.fields.0.name = test + * schema.fields.1.type = LONG + * + * The propertyKeys map defines e.g. "type" and a validation logic for the given full key. + * + * The validation consumer takes the current prefix e.g. "schema.fields.1.". + */ + def validateVariableIndexedProperties( + key: String, + allowEmpty: Boolean, + propertyKeys: JMap[String, Consumer[String]], + requiredKeys: JList[String]) + : Unit = { + + val keys = propertyKeys.asScala + + // filter for index + val escapedKey = Pattern.quote(key) + val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)\\.(.*)") + + // extract index and property keys + val indexes = properties.keys.flatMap { k => + val matcher = pattern.matcher(k) + if (matcher.find()) { + Some(JInt.parseInt(matcher.group(1))) + } else { + None + } + } + + // determine max index + val maxIndex = indexes.reduceOption(_ max _).getOrElse(-1) + + if (maxIndex < 0 && !allowEmpty) { + throw new ValidationException(s"Property key '$key' must not be empty.") + } + + // validate + for (i <- 0 to maxIndex) { + keys.foreach { case (subKey, validation) => + val fullKey = s"$key.$i.$subKey" + // only validate if it exists + if (properties.contains(fullKey)) { + validation.accept(s"$key.$i.") + } else { + // check if it is required + if (requiredKeys.contains(subKey)) { + throw new ValidationException(s"Required property key '$fullKey' is missing.") + } + } + } + } + } + + /** + * Validation for fixed indexed properties. + * + * For example: + * + * schema.fields.0.type = INT, schema.fields.0.name = test + * schema.fields.1.type = LONG, schema.fields.1.name = test2 + * + * The propertyKeys map must define e.g. "type" and "name" and a validation logic for the + * given full key. + */ + def validateFixedIndexedProperties( + key: String, + allowEmpty: Boolean, + propertyKeys: JMap[String, Consumer[String]]) + : Unit = { + + validateVariableIndexedProperties( + key, + allowEmpty, + propertyKeys, + new util.ArrayList(propertyKeys.keySet())) + } + + /** + * Validates a table schema property. + */ def validateTableSchema(key: String, isOptional: Boolean): Unit = { - // filter for name columns - val names = getIndexedProperty(key, NAME) - // filter for type columns - val types = getIndexedProperty(key, TYPE) - if (names.isEmpty && types.isEmpty && !isOptional) { - throw new ValidationException( - s"Could not find the required schema for property '$key'.") + val nameValidation = (prefix: String) => { + validateString(prefix + NAME, isOptional = false, minLen = 1) } - for (i <- 0 until Math.max(names.size, types.size)) { - validateString(s"$key.$i.$NAME", isOptional = false, minLen = 1) - validateType(s"$key.$i.$TYPE", isOptional = false) + val typeValidation = (prefix: String) => { + validateType(prefix + TYPE, isOptional = false) } + + validateFixedIndexedProperties( + key, + isOptional, + Map( + NAME -> toJava(nameValidation), + TYPE -> toJava(typeValidation) + ).asJava + ) } + /** + * Validates a enum property with a set of validation logic for each enum value. + */ def validateEnum( key: String, isOptional: Boolean, - enumToValidation: Map[String, () => Unit]) + enumToValidation: JMap[String, Consumer[String]]) : Unit = { if (!properties.contains(key)) { @@ -401,15 +838,26 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } else { val value = properties(key) - if (!enumToValidation.contains(value)) { + if (!enumToValidation.containsKey(value)) { throw new ValidationException(s"Unknown value for property '$key'. " + - s"Supported values [${enumToValidation.keys.mkString(", ")}] but was: $value") + s"Supported values [${enumToValidation.keySet().asScala.mkString(", ")}] but was: $value") } else { - enumToValidation(value).apply() // run validation logic + // run validation logic + enumToValidation.get(value).accept(key) } } } + /** + * Validates a enum property with a set of enum values. + */ + def validateEnumValues(key: String, isOptional: Boolean, values: JList[String]): Unit = { + validateEnum(key, isOptional, values.asScala.map((_, noValidation())).toMap.asJava) + } + + /** + * Validates a type property. + */ def validateType(key: String, isOptional: Boolean): Unit = { if (!properties.contains(key)) { if (!isOptional) { @@ -420,6 +868,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validates that the given prefix is not included in these properties. + */ def validatePrefixExclusion(prefix: String): Unit = { val invalidField = properties.find(_._1.startsWith(prefix)) if (invalidField.isDefined) { @@ -428,6 +879,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { } } + /** + * Validates that the given key is not included in these properties. + */ def validateExclusion(key: String): Unit = { if (properties.contains(key)) { throw new ValidationException(s"Property '$key' is not allowed in this context.") @@ -436,28 +890,159 @@ class DescriptorProperties(normalizeKeys: Boolean = true) { // ---------------------------------------------------------------------------------------------- - def getIndexedProperty(key: String, property: String): Map[String, String] = { - val escapedKey = Pattern.quote(key) - properties.filterKeys(k => k.matches(s"$escapedKey\\.\\d+\\.$property")).toMap + /** + * Returns if any property contains parts of a given string. + */ + def containsString(str: String): Boolean = { + properties.exists(e => e._1.contains(str)) } - def contains(str: String): Boolean = { - properties.exists(e => e._1.contains(str)) + /** + * Returns if the given key is contained. + */ + def containsKey(key: String): Boolean = { + properties.contains(key) } + /** + * Returns if a given prefix exists in the properties. + */ def hasPrefix(prefix: String): Boolean = { properties.exists(e => e._1.startsWith(prefix)) } - def asMap: Map[String, String] = { - properties.toMap + /** + * Returns a Scala Map. + */ + def asMap: JMap[String, String] = { + properties.toMap.asJava + } + + // ---------------------------------------------------------------------------------------------- + + /** + * Returns an empty validation logic. + */ + def noValidation(): Consumer[String] = DescriptorProperties.emptyConsumer + + def exceptionSupplier(key: String): Supplier[TableException] = new Supplier[TableException] { + override def get(): TableException = { + new TableException(s"Property with key '$key' could not be found. " + + s"This is a bug because the validation logic should have checked that before.") + } + } + + // ---------------------------------------------------------------------------------------------- + + /** + * Adds a property. + */ + private def put(key: String, value: String): Unit = { + if (properties.contains(key)) { + throw new IllegalStateException("Property already present.") + } + if (normalizeKeys) { + properties.put(key.toLowerCase, value) + } else { + properties.put(key, value) + } + } + + /** + * Gets an existing property. + */ + private def get(key: String): String = { + properties.getOrElse( + key, + throw exceptionSupplier(key).get()) + } + + /** + * Raw access to the underlying properties map for testing purposes. + */ + private[flink] def unsafePut(key: String, value: String): Unit = { + properties.put(key, value) + } + + /** + * Raw access to the underlying properties map for testing purposes. + */ + private[flink] def unsafeRemove(key: String): Unit = { + properties.remove(key) + } + + /** + * Adds a table schema under the given key. + */ + private def putTableSchema(key: String, nameAndType: Seq[(String, String)]): Unit = { + putIndexedFixedProperties( + key, + Seq(NAME, TYPE), + nameAndType.map(t => Seq(t._1, t._2)) + ) + } + + /** + * Adds an indexed sequence of properties (with sub-properties) under a common key. + * + * For example: + * + * schema.fields.0.type = INT, schema.fields.0.name = test + * schema.fields.1.type = LONG, schema.fields.1.name = test2 + * + * The arity of each propertyValue must match the arity of propertyKeys. + */ + private def putIndexedFixedProperties( + key: String, + propertyKeys: Seq[String], + propertyValues: Seq[Seq[String]]) + : Unit = { + checkNotNull(key) + checkNotNull(propertyValues) + propertyValues.zipWithIndex.foreach { case (values, idx) => + if (values.lengthCompare(propertyKeys.size) != 0) { + throw new ValidationException("Values must have same arity as keys.") + } + values.zipWithIndex.foreach { case (value, keyIdx) => + put(s"$key.$idx.${propertyKeys(keyIdx)}", value) + } + } + } + + /** + * Adds an indexed mapping of properties under a common key. + * + * For example: + * + * schema.fields.0.type = INT, schema.fields.0.name = test + * schema.fields.1.name = test2 + * + * The arity of the propertySets can differ. + */ + private def putIndexedVariableProperties( + key: String, + propertySets: Seq[Map[String, String]]) + : Unit = { + checkNotNull(key) + checkNotNull(propertySets) + propertySets.zipWithIndex.foreach { case (propertySet, idx) => + propertySet.foreach { case (k, v) => + put(s"$key.$idx.$k", v) + } + } } } object DescriptorProperties { - val TYPE = "type" - val NAME = "name" + private val emptyConsumer: Consumer[String] = new Consumer[String] { + override def accept(t: String): Unit = { + // nothing to do + } + } + + val TYPE: String = "type" + val NAME: String = "name" // the string representation should be equal to SqlTypeName def normalizeTypeInfo(typeInfo: TypeInformation[_]): String = { @@ -487,6 +1072,24 @@ object DescriptorProperties { } } + def deserialize[T](data: String, expected: Class[T]): T = { + try { + val byteData = Base64.decodeBase64(data) + val obj = InstantiationUtil.deserializeObject[T]( + byteData, + Thread.currentThread.getContextClassLoader) + if (!expected.isAssignableFrom(obj.getClass)) { + throw new ValidationException( + s"Serialized data contains an object of unexpected type. " + + s"Expected '${expected.getName}' but was '${obj.getClass.getName}'") + } + obj + } catch { + case e: Exception => + throw new ValidationException(s"Could not deserialize data: '$data'", e) + } + } + def toString(keyOrValue: String): String = { StringEscapeUtils.escapeJava(keyOrValue) } @@ -494,4 +1097,24 @@ object DescriptorProperties { def toString(key: String, value: String): String = { toString(key) + "=" + toString(value) } + + // the following methods help for Scala <-> Java interfaces + // most of these methods are not necessary once we upgraded to Scala 2.12 + + def toJava[T](option: Option[T]): Optional[T] = option match { + case Some(v) => Optional.of(v) + case None => Optional.empty() + } + + def toScala[T](option: Optional[T]): Option[T] = Option(option.orElse(null.asInstanceOf[T])) + + def toJava[T](func: Function[T, Unit]): Consumer[T] = new Consumer[T] { + override def accept(t: T): Unit = { + func.apply(t) + } + } + + def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] = { + new JTuple2[T0, T1](tuple._1, tuple._2) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala index b1d900f8e316f3..f306b5aa72a501 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala @@ -23,7 +23,8 @@ import org.apache.flink.table.descriptors.FileSystemValidator.{CONNECTOR_PATH, C /** * Connector descriptor for a file system. */ -class FileSystem extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1) { +class FileSystem extends ConnectorDescriptor( + CONNECTOR_TYPE_VALUE, version = 1, formatNeeded = true) { private var path: Option[String] = None @@ -43,8 +44,6 @@ class FileSystem extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1) override protected def addConnectorProperties(properties: DescriptorProperties): Unit = { path.foreach(properties.putString(CONNECTOR_PATH, _)) } - - override private[flink] def needsFormat() = true } /** diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala index 86f6229f903cdb..bca67c6ea5799a 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.descriptors -import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION} +import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_PROPERTY_VERSION} /** * Describes the format of data. @@ -37,7 +37,7 @@ abstract class FormatDescriptor( */ final private[flink] def addProperties(properties: DescriptorProperties): Unit = { properties.putString(FORMAT_TYPE, tpe) - properties.putInt(FORMAT_VERSION, version) + properties.putInt(FORMAT_PROPERTY_VERSION, version) addFormatProperties(properties) } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala index 1aaa39987edeb5..301189a17966b4 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.descriptors -import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION} +import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_PROPERTY_VERSION, FORMAT_TYPE} /** * Validator for [[FormatDescriptor]]. @@ -27,13 +27,32 @@ class FormatDescriptorValidator extends DescriptorValidator { override def validate(properties: DescriptorProperties): Unit = { properties.validateString(FORMAT_TYPE, isOptional = false, minLen = 1) - properties.validateInt(FORMAT_VERSION, isOptional = true, 0, Integer.MAX_VALUE) + properties.validateInt(FORMAT_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE) } } object FormatDescriptorValidator { + /** + * Key for describing the type of the format. Usually used for factory discovery. + */ val FORMAT_TYPE = "format.type" + + /** + * Key for describing the property version. This property can be used for backwards + * compatibility in case the property format changes. + */ + val FORMAT_PROPERTY_VERSION = "format.property-version" + + /** + * Key for describing the version of the format. This property can be used for different + * format versions (e.g. Avro 1.8.2 or Avro 2.0). + */ val FORMAT_VERSION = "format.version" + /** + * Key for deriving the schema of the format from the table's schema. + */ + val FORMAT_DERIVE_SCHEMA = "format.derive-schema" + } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala deleted file mode 100644 index cc46d9cfc601bd..00000000000000 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.descriptors - -import org.apache.flink.table.descriptors.JsonValidator.{FORMAT_FAIL_ON_MISSING_FIELD, FORMAT_SCHEMA_STRING, FORMAT_TYPE_VALUE} - -/** - * Encoding descriptor for JSON. - */ -class Json extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) { - - private var failOnMissingField: Option[Boolean] = None - - private var schema: Option[String] = None - - /** - * Sets flag whether to fail if a field is missing or not. - * - * @param failOnMissingField If set to true, the operation fails if there is a missing field. - * If set to false, a missing field is set to null. - * @return The builder. - */ - def failOnMissingField(failOnMissingField: Boolean): Json = { - this.failOnMissingField = Some(failOnMissingField) - this - } - - /** - * Sets the JSON schema string with field names and the types according to the JSON schema - * specification [[http://json-schema.org/specification.html]]. Required. - * - * The schema might be nested. - * - * @param schema JSON schema - */ - def schema(schema: String): Json = { - this.schema = Some(schema) - this - } - - /** - * Internal method for format properties conversion. - */ - override protected def addFormatProperties(properties: DescriptorProperties): Unit = { - // we distinguish between "schema string" and "schema" to allow parsing of a - // schema object in the future (such that the entire JSON schema can be defined in a YAML - // file instead of one large string) - schema.foreach(properties.putString(FORMAT_SCHEMA_STRING, _)) - failOnMissingField.foreach(properties.putBoolean(FORMAT_FAIL_ON_MISSING_FIELD, _)) - } -} - -/** - * Encoding descriptor for JSON. - */ -object Json { - - /** - * Encoding descriptor for JSON. - */ - def apply(): Json = new Json() -} diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala deleted file mode 100644 index 9f11caf128f8fc..00000000000000 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.descriptors - -import org.apache.flink.table.descriptors.JsonValidator.{FORMAT_FAIL_ON_MISSING_FIELD, FORMAT_SCHEMA_STRING} - -/** - * Validator for [[Json]]. - */ -class JsonValidator extends FormatDescriptorValidator { - - override def validate(properties: DescriptorProperties): Unit = { - super.validate(properties) - properties.validateString(FORMAT_SCHEMA_STRING, isOptional = false, minLen = 1) - properties.validateBoolean(FORMAT_FAIL_ON_MISSING_FIELD, isOptional = true) - } -} - -object JsonValidator { - - val FORMAT_TYPE_VALUE = "json" - val FORMAT_SCHEMA_STRING = "format.schema-string" - val FORMAT_FAIL_ON_MISSING_FIELD = "format.fail-on-missing-field" - -} diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala index a8d580c5d316d9..6631e22e8d0deb 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.descriptors -import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, METADATA_CREATION_TIME, METADATA_LAST_ACCESS_TIME, METADATA_VERSION} +import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, METADATA_CREATION_TIME, METADATA_LAST_ACCESS_TIME, METADATA_PROPERTY_VERSION} /** * Validator for [[Metadata]]. @@ -26,7 +26,7 @@ import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, M class MetadataValidator extends DescriptorValidator { override def validate(properties: DescriptorProperties): Unit = { - properties.validateInt(METADATA_VERSION, isOptional = true, 0, Integer.MAX_VALUE) + properties.validateInt(METADATA_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE) properties.validateString(METADATA_COMMENT, isOptional = true) properties.validateLong(METADATA_CREATION_TIME, isOptional = true) properties.validateLong(METADATA_LAST_ACCESS_TIME, isOptional = true) @@ -35,7 +35,7 @@ class MetadataValidator extends DescriptorValidator { object MetadataValidator { - val METADATA_VERSION = "metadata.version" + val METADATA_PROPERTY_VERSION = "metadata.property-version" val METADATA_COMMENT = "metadata.comment" val METADATA_CREATION_TIME = "metadata.creation-time" val METADATA_LAST_ACCESS_TIME = "metadata.last-access-time" diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala index a1c80f58409961..ed3854df36b647 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala @@ -19,11 +19,12 @@ package org.apache.flink.table.descriptors import org.apache.flink.table.api.Types -import org.apache.flink.table.descriptors.RowtimeValidator.{ROWTIME, ROWTIME_VERSION, normalizeTimestampExtractor, normalizeWatermarkStrategy} +import org.apache.flink.table.descriptors.RowtimeValidator.{normalizeTimestampExtractor, normalizeWatermarkStrategy} import org.apache.flink.table.sources.tsextractors.{ExistingField, StreamRecordTimestamp, TimestampExtractor} import org.apache.flink.table.sources.wmstrategies.{AscendingTimestamps, BoundedOutOfOrderTimestamps, PreserveWatermarks, WatermarkStrategy} import scala.collection.mutable +import scala.collection.JavaConverters._ /** * Rowtime descriptor for describing an event time attribute in the schema. @@ -111,12 +112,9 @@ class Rowtime extends Descriptor { */ final override def addProperties(properties: DescriptorProperties): Unit = { val props = mutable.HashMap[String, String]() - props.put(ROWTIME_VERSION, "1") timestampExtractor.foreach(normalizeTimestampExtractor(_).foreach(e => props.put(e._1, e._2))) watermarkStrategy.foreach(normalizeWatermarkStrategy(_).foreach(e => props.put(e._1, e._2))) - - // use a list for the rowtime to support multiple rowtime attributes in the future - properties.putIndexedVariableProperties(ROWTIME, Seq(props.toMap)) + properties.putProperties(props.toMap.asJava) } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala index 74e49f14d84c4a..fdec82008aee94 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala @@ -18,58 +18,62 @@ package org.apache.flink.table.descriptors -import org.apache.flink.table.descriptors.DescriptorProperties.serialize +import org.apache.flink.table.descriptors.DescriptorProperties.{serialize, toJava} import org.apache.flink.table.descriptors.RowtimeValidator._ import org.apache.flink.table.sources.tsextractors.{ExistingField, StreamRecordTimestamp, TimestampExtractor} import org.apache.flink.table.sources.wmstrategies.{AscendingTimestamps, BoundedOutOfOrderTimestamps, PreserveWatermarks, WatermarkStrategy} +import scala.collection.JavaConverters._ + /** * Validator for [[Rowtime]]. */ class RowtimeValidator(val prefix: String = "") extends DescriptorValidator { override def validate(properties: DescriptorProperties): Unit = { - properties.validateInt(prefix + ROWTIME_VERSION, isOptional = true, 0, Integer.MAX_VALUE) - - val noValidation = () => {} - - val timestampExistingField = () => { - properties.validateString(prefix + TIMESTAMPS_FROM, isOptional = false, minLen = 1) + val timestampExistingField = (_: String) => { + properties.validateString( + prefix + ROWTIME_TIMESTAMPS_FROM, isOptional = false, minLen = 1) } - val timestampCustom = () => { - properties.validateString(prefix + TIMESTAMPS_CLASS, isOptional = false, minLen = 1) - properties.validateString(prefix + TIMESTAMPS_SERIALIZED, isOptional = false, minLen = 1) + val timestampCustom = (_: String) => { + properties.validateString( + prefix + ROWTIME_TIMESTAMPS_CLASS, isOptional = false, minLen = 1) + properties.validateString( + prefix + ROWTIME_TIMESTAMPS_SERIALIZED, isOptional = false, minLen = 1) } properties.validateEnum( - prefix + TIMESTAMPS_TYPE, + prefix + ROWTIME_TIMESTAMPS_TYPE, isOptional = false, Map( - TIMESTAMPS_TYPE_VALUE_FROM_FIELD -> timestampExistingField, - TIMESTAMPS_TYPE_VALUE_FROM_SOURCE -> noValidation, - TIMESTAMPS_TYPE_VALUE_CUSTOM -> timestampCustom - ) + ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD -> toJava(timestampExistingField), + ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE -> properties.noValidation(), + ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM -> toJava(timestampCustom) + ).asJava ) - val watermarkPeriodicBounding = () => { - properties.validateLong(prefix + WATERMARKS_DELAY, isOptional = false, min = 0) + val watermarkPeriodicBounded = (_: String) => { + properties.validateLong( + prefix + ROWTIME_WATERMARKS_DELAY, isOptional = false, min = 0) } - val watermarkCustom = () => { - properties.validateString(prefix + WATERMARKS_CLASS, isOptional = false, minLen = 1) - properties.validateString(prefix + WATERMARKS_SERIALIZED, isOptional = false, minLen = 1) + val watermarkCustom = (_: String) => { + properties.validateString( + prefix + ROWTIME_WATERMARKS_CLASS, isOptional = false, minLen = 1) + properties.validateString( + prefix + ROWTIME_WATERMARKS_SERIALIZED, isOptional = false, minLen = 1) } properties.validateEnum( - prefix + WATERMARKS_TYPE, + prefix + ROWTIME_WATERMARKS_TYPE, isOptional = false, Map( - WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> noValidation, - WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDING -> watermarkPeriodicBounding, - WATERMARKS_TYPE_VALUE_FROM_SOURCE -> noValidation, - WATERMARKS_TYPE_VALUE_CUSTOM -> watermarkCustom - ) + ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> properties.noValidation(), + ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED -> toJava(watermarkPeriodicBounded), + ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE -> properties.noValidation(), + ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM -> toJava(watermarkCustom) + ).asJava ) } } @@ -77,58 +81,113 @@ class RowtimeValidator(val prefix: String = "") extends DescriptorValidator { object RowtimeValidator { val ROWTIME = "rowtime" - - // per rowtime properties - - val ROWTIME_VERSION = "version" - val TIMESTAMPS_TYPE = "timestamps.type" - val TIMESTAMPS_TYPE_VALUE_FROM_FIELD = "from-field" - val TIMESTAMPS_TYPE_VALUE_FROM_SOURCE = "from-source" - val TIMESTAMPS_TYPE_VALUE_CUSTOM = "custom" - val TIMESTAMPS_FROM = "timestamps.from" - val TIMESTAMPS_CLASS = "timestamps.class" - val TIMESTAMPS_SERIALIZED = "timestamps.serialized" - - val WATERMARKS_TYPE = "watermarks.type" - val WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING = "periodic-ascending" - val WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDING = "periodic-bounding" - val WATERMARKS_TYPE_VALUE_FROM_SOURCE = "from-source" - val WATERMARKS_TYPE_VALUE_CUSTOM = "custom" - val WATERMARKS_CLASS = "watermarks.class" - val WATERMARKS_SERIALIZED = "watermarks.serialized" - val WATERMARKS_DELAY = "watermarks.delay" + val ROWTIME_TIMESTAMPS_TYPE = "rowtime.timestamps.type" + val ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD = "from-field" + val ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE = "from-source" + val ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM = "custom" + val ROWTIME_TIMESTAMPS_FROM = "rowtime.timestamps.from" + val ROWTIME_TIMESTAMPS_CLASS = "rowtime.timestamps.class" + val ROWTIME_TIMESTAMPS_SERIALIZED = "rowtime.timestamps.serialized" + + val ROWTIME_WATERMARKS_TYPE = "rowtime.watermarks.type" + val ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING = "periodic-ascending" + val ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED = "periodic-bounded" + val ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE = "from-source" + val ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM = "custom" + val ROWTIME_WATERMARKS_CLASS = "rowtime.watermarks.class" + val ROWTIME_WATERMARKS_SERIALIZED = "rowtime.watermarks.serialized" + val ROWTIME_WATERMARKS_DELAY = "rowtime.watermarks.delay" // utilities def normalizeTimestampExtractor(extractor: TimestampExtractor): Map[String, String] = extractor match { + case existing: ExistingField => Map( - TIMESTAMPS_TYPE -> TIMESTAMPS_TYPE_VALUE_FROM_FIELD, - TIMESTAMPS_FROM -> existing.getArgumentFields.apply(0)) + ROWTIME_TIMESTAMPS_TYPE -> ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD, + ROWTIME_TIMESTAMPS_FROM -> existing.getArgumentFields.apply(0)) + case _: StreamRecordTimestamp => - Map(TIMESTAMPS_TYPE -> TIMESTAMPS_TYPE_VALUE_FROM_SOURCE) + Map(ROWTIME_TIMESTAMPS_TYPE -> ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE) + case _: TimestampExtractor => Map( - TIMESTAMPS_TYPE -> TIMESTAMPS_TYPE_VALUE_CUSTOM, - TIMESTAMPS_CLASS -> extractor.getClass.getName, - TIMESTAMPS_SERIALIZED -> serialize(extractor)) + ROWTIME_TIMESTAMPS_TYPE -> ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM, + ROWTIME_TIMESTAMPS_CLASS -> extractor.getClass.getName, + ROWTIME_TIMESTAMPS_SERIALIZED -> serialize(extractor)) } def normalizeWatermarkStrategy(strategy: WatermarkStrategy): Map[String, String] = strategy match { + case _: AscendingTimestamps => - Map(WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING) - case bounding: BoundedOutOfOrderTimestamps => + Map(ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING) + + case bounded: BoundedOutOfOrderTimestamps => Map( - WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDING, - WATERMARKS_DELAY -> bounding.delay.toString) + ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED, + ROWTIME_WATERMARKS_DELAY -> bounded.delay.toString) + case _: PreserveWatermarks => - Map(WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_FROM_SOURCE) + Map(ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE) + case _: WatermarkStrategy => Map( - WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_CUSTOM, - WATERMARKS_CLASS -> strategy.getClass.getName, - WATERMARKS_SERIALIZED -> serialize(strategy)) + ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM, + ROWTIME_WATERMARKS_CLASS -> strategy.getClass.getName, + ROWTIME_WATERMARKS_SERIALIZED -> serialize(strategy)) } + + def getRowtimeComponents(properties: DescriptorProperties, prefix: String) + : Option[(TimestampExtractor, WatermarkStrategy)] = { + + // create timestamp extractor + val t = properties.getOptionalString(prefix + ROWTIME_TIMESTAMPS_TYPE) + if (!t.isPresent) { + return None + } + val extractor: TimestampExtractor = t.get() match { + + case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD => + val field = properties.getString(prefix + ROWTIME_TIMESTAMPS_FROM) + new ExistingField(field) + + case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE => + new StreamRecordTimestamp + + case ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM => + val clazz = properties.getClass( + ROWTIME_TIMESTAMPS_CLASS, + classOf[TimestampExtractor]) + DescriptorProperties.deserialize( + properties.getString(prefix + ROWTIME_TIMESTAMPS_SERIALIZED), + clazz) + } + + // create watermark strategy + val s = properties.getString(prefix + ROWTIME_WATERMARKS_TYPE) + val strategy: WatermarkStrategy = s match { + + case ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING => + new AscendingTimestamps() + + case ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED => + val delay = properties.getLong(prefix + ROWTIME_WATERMARKS_DELAY) + new BoundedOutOfOrderTimestamps(delay) + + case ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE => + PreserveWatermarks.INSTANCE + + case ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM => + val clazz = properties.getClass( + prefix + ROWTIME_WATERMARKS_CLASS, + classOf[WatermarkStrategy]) + DescriptorProperties.deserialize( + properties.getString(prefix + ROWTIME_WATERMARKS_SERIALIZED), + clazz) + } + + Some((extractor, strategy)) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala index 2f3a3897a4af17..fcbb2c7ea22914 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala @@ -24,6 +24,7 @@ import org.apache.flink.table.descriptors.DescriptorProperties.{normalizeTableSc import org.apache.flink.table.descriptors.SchemaValidator._ import scala.collection.mutable +import scala.collection.JavaConverters._ /** * Describes a schema of a table. @@ -80,7 +81,7 @@ class Schema extends Descriptor { } val fieldProperties = mutable.LinkedHashMap[String, String]() - fieldProperties += (TYPE -> fieldType) + fieldProperties += (SCHEMA_TYPE -> fieldType) tableSchema += (fieldName -> fieldProperties) @@ -100,7 +101,7 @@ class Schema extends Descriptor { lastField match { case None => throw new ValidationException("No field previously defined. Use field() before.") case Some(f) => - tableSchema(f) += (FROM -> originFieldName) + tableSchema(f) += (SCHEMA_FROM -> originFieldName) lastField = None } this @@ -115,7 +116,7 @@ class Schema extends Descriptor { lastField match { case None => throw new ValidationException("No field defined previously. Use field() before.") case Some(f) => - tableSchema(f) += (PROCTIME -> PROCTIME_VALUE_TRUE) + tableSchema(f) += (SCHEMA_PROCTIME -> "true") lastField = None } this @@ -132,7 +133,7 @@ class Schema extends Descriptor { case Some(f) => val fieldProperties = new DescriptorProperties() rowtime.addProperties(fieldProperties) - tableSchema(f) ++= fieldProperties.asMap + tableSchema(f) ++= fieldProperties.asMap.asScala lastField = None } this @@ -142,12 +143,11 @@ class Schema extends Descriptor { * Internal method for properties conversion. */ final override private[flink] def addProperties(properties: DescriptorProperties): Unit = { - properties.putInt(SCHEMA_VERSION, 1) properties.putIndexedVariableProperties( SCHEMA, tableSchema.toSeq.map { case (name, props) => - Map(NAME -> name) ++ props - } + (Map(SCHEMA_NAME -> name) ++ props).asJava + }.asJava ) } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala index 19c0e411dd233f..0a2391175bf019 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala @@ -18,9 +18,17 @@ package org.apache.flink.table.descriptors -import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME +import java.util +import java.util.Optional + +import org.apache.flink.table.api.{TableSchema, ValidationException} +import org.apache.flink.table.descriptors.DescriptorProperties.{toJava, toScala} +import org.apache.flink.table.descriptors.RowtimeValidator.{ROWTIME, ROWTIME_TIMESTAMPS_TYPE} import org.apache.flink.table.descriptors.SchemaValidator._ +import org.apache.flink.table.sources.RowtimeAttributeDescriptor + +import scala.collection.JavaConverters._ +import scala.collection.mutable /** * Validator for [[Schema]]. @@ -28,29 +36,39 @@ import org.apache.flink.table.descriptors.SchemaValidator._ class SchemaValidator(isStreamEnvironment: Boolean = true) extends DescriptorValidator { override def validate(properties: DescriptorProperties): Unit = { - properties.validateInt(SCHEMA_VERSION, isOptional = true, 0, Integer.MAX_VALUE) - - val names = properties.getIndexedProperty(SCHEMA, NAME) - val types = properties.getIndexedProperty(SCHEMA, TYPE) + val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME) + val types = properties.getIndexedProperty(SCHEMA, SCHEMA_TYPE) if (names.isEmpty && types.isEmpty) { - throw new ValidationException(s"Could not find the required schema for property '$SCHEMA'.") + throw new ValidationException( + s"Could not find the required schema in property '$SCHEMA'.") } + var proctimeFound = false + for (i <- 0 until Math.max(names.size, types.size)) { - properties.validateString(s"$SCHEMA.$i.$NAME", isOptional = false, minLen = 1) - properties.validateType(s"$SCHEMA.$i.$TYPE", isOptional = false) - properties.validateString(s"$SCHEMA.$i.$FROM", isOptional = true, minLen = 1) + properties + .validateString(s"$SCHEMA.$i.$SCHEMA_NAME", isOptional = false, minLen = 1) + properties + .validateType(s"$SCHEMA.$i.$SCHEMA_TYPE", isOptional = false) + properties + .validateString(s"$SCHEMA.$i.$SCHEMA_FROM", isOptional = true, minLen = 1) // either proctime or rowtime - val proctime = s"$SCHEMA.$i.$PROCTIME" + val proctime = s"$SCHEMA.$i.$SCHEMA_PROCTIME" val rowtime = s"$SCHEMA.$i.$ROWTIME" - if (properties.contains(proctime)) { + if (properties.containsKey(proctime)) { + // check the environment if (!isStreamEnvironment) { throw new ValidationException( s"Property '$proctime' is not allowed in a batch environment.") } + // check for only one proctime attribute + else if (proctimeFound) { + throw new ValidationException("A proctime attribute must only be defined once.") + } // check proctime properties.validateBoolean(proctime, isOptional = false) + proctimeFound = properties.getBoolean(proctime) // no rowtime properties.validatePrefixExclusion(rowtime) } else if (properties.hasPrefix(rowtime)) { @@ -67,14 +85,129 @@ class SchemaValidator(isStreamEnvironment: Boolean = true) extends DescriptorVal object SchemaValidator { val SCHEMA = "schema" - val SCHEMA_VERSION = "schema.version" + val SCHEMA_NAME = "name" + val SCHEMA_TYPE = "type" + val SCHEMA_PROCTIME = "proctime" + val SCHEMA_FROM = "from" + + // utilities + + /** + * Finds the proctime attribute if defined. + */ + def deriveProctimeAttribute(properties: DescriptorProperties): Optional[String] = { + val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME) + + for (i <- 0 until names.size) { + val isProctime = toScala( + properties.getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME")) + isProctime.foreach { isSet => + if (isSet) { + return toJava(names.asScala.get(s"$SCHEMA.$i.$SCHEMA_NAME")) + } + } + } + toJava(None) + } + + /** + * Finds the rowtime attributes if defined. + */ + def deriveRowtimeAttributes(properties: DescriptorProperties) + : util.List[RowtimeAttributeDescriptor] = { + + val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME) + + var attributes = new mutable.ArrayBuffer[RowtimeAttributeDescriptor]() + + // check for rowtime in every field + for (i <- 0 until names.size) { + RowtimeValidator + .getRowtimeComponents(properties, s"$SCHEMA.$i.") + .foreach { case (extractor, strategy) => + // create descriptor + attributes += new RowtimeAttributeDescriptor( + properties.getString(s"$SCHEMA.$i.$SCHEMA_NAME"), + extractor, + strategy) + } + } + + attributes.asJava + } + + /** + * Finds a table source field mapping. + */ + def deriveFieldMapping( + properties: DescriptorProperties, + sourceSchema: Optional[TableSchema]) + : util.Map[String, String] = { + + val mapping = mutable.Map[String, String]() + + val schema = properties.getTableSchema(SCHEMA) + + // add all schema fields first for implicit mappings + schema.getColumnNames.foreach { name => + mapping.put(name, name) + } + + val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME) + + for (i <- 0 until names.size) { + val name = properties.getString(s"$SCHEMA.$i.$SCHEMA_NAME") + toScala(properties.getOptionalString(s"$SCHEMA.$i.$SCHEMA_FROM")) match { - // per column properties + // add explicit mapping + case Some(source) => + mapping.put(name, source) - val NAME = "name" - val TYPE = "type" - val PROCTIME = "proctime" - val PROCTIME_VALUE_TRUE = "true" - val FROM = "from" + // implicit mapping or time + case None => + val isProctime = properties + .getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME") + .orElse(false) + val isRowtime = properties + .containsKey(s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_TYPE") + // remove proctime/rowtime from mapping + if (isProctime || isRowtime) { + mapping.remove(name) + } + // check for invalid fields + else if (toScala(sourceSchema).forall(s => !s.getColumnNames.contains(name))) { + throw new ValidationException(s"Could not map the schema field '$name' to a field " + + s"from source. Please specify the source field from which it can be derived.") + } + } + } + mapping.toMap.asJava + } + + /** + * Finds the fields that can be used for a format schema (without time attributes). + */ + def deriveFormatFields(properties: DescriptorProperties): TableSchema = { + + val builder = TableSchema.builder() + + val schema = properties.getTableSchema(SCHEMA) + + schema.getColumnNames.zip(schema.getTypes).zipWithIndex.foreach { case ((n, t), i) => + val isProctime = properties + .getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME") + .orElse(false) + val isRowtime = properties + .containsKey(s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_TYPE") + if (!isProctime && !isRowtime) { + // check for a aliasing + val fieldName = properties.getOptionalString(s"$SCHEMA.$i.$SCHEMA_FROM") + .orElse(n) + builder.field(fieldName, t) + } + } + + builder.build() + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala index 303728610d5b42..f87a868915b4a6 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala @@ -135,12 +135,12 @@ class Statistics extends Descriptor { * Internal method for properties conversion. */ final override def addProperties(properties: DescriptorProperties): Unit = { - properties.putInt(STATISTICS_VERSION, 1) + properties.putInt(STATISTICS_PROPERTY_VERSION, 1) rowCount.foreach(rc => properties.putLong(STATISTICS_ROW_COUNT, rc)) val namedStats = columnStats.map { case (name, stats) => // name should not be part of the properties key - (stats + (NAME -> name)).toMap - }.toSeq + (stats + (NAME -> name)).toMap.asJava + }.toList.asJava properties.putIndexedVariableProperties(STATISTICS_COLUMNS, namedStats) } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala index a78e42239b5927..691cb218497f5a 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala @@ -19,7 +19,8 @@ package org.apache.flink.table.descriptors import org.apache.flink.table.api.ValidationException -import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_ROW_COUNT, STATISTICS_VERSION, validateColumnStats} +import org.apache.flink.table.descriptors.DescriptorProperties.toScala +import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_PROPERTY_VERSION, STATISTICS_ROW_COUNT, validateColumnStats} import org.apache.flink.table.plan.stats.ColumnStats import scala.collection.mutable @@ -30,7 +31,7 @@ import scala.collection.mutable class StatisticsValidator extends DescriptorValidator { override def validate(properties: DescriptorProperties): Unit = { - properties.validateInt(STATISTICS_VERSION, isOptional = true, 0, Integer.MAX_VALUE) + properties.validateInt(STATISTICS_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE) properties.validateLong(STATISTICS_ROW_COUNT, isOptional = true, min = 0) validateColumnStats(properties, STATISTICS_COLUMNS) } @@ -38,7 +39,7 @@ class StatisticsValidator extends DescriptorValidator { object StatisticsValidator { - val STATISTICS_VERSION = "statistics.version" + val STATISTICS_PROPERTY_VERSION = "statistics.property-version" val STATISTICS_ROW_COUNT = "statistics.row-count" val STATISTICS_COLUMNS = "statistics.columns" @@ -99,16 +100,16 @@ object StatisticsValidator { val columnCount = properties.getIndexedProperty(key, NAME).size val stats = for (i <- 0 until columnCount) yield { - val name = properties.getString(s"$key.$i.$NAME").getOrElse( + val name = toScala(properties.getOptionalString(s"$key.$i.$NAME")).getOrElse( throw new ValidationException(s"Could not find name of property '$key.$i.$NAME'.")) val stats = ColumnStats( - properties.getLong(s"$key.$i.$DISTINCT_COUNT").map(v => Long.box(v)).orNull, - properties.getLong(s"$key.$i.$NULL_COUNT").map(v => Long.box(v)).orNull, - properties.getDouble(s"$key.$i.$AVG_LENGTH").map(v => Double.box(v)).orNull, - properties.getInt(s"$key.$i.$MAX_LENGTH").map(v => Int.box(v)).orNull, - properties.getDouble(s"$key.$i.$MAX_VALUE").map(v => Double.box(v)).orNull, - properties.getDouble(s"$key.$i.$MIN_VALUE").map(v => Double.box(v)).orNull + properties.getOptionalLong(s"$key.$i.$DISTINCT_COUNT").orElse(null), + properties.getOptionalLong(s"$key.$i.$NULL_COUNT").orElse(null), + properties.getOptionalDouble(s"$key.$i.$AVG_LENGTH").orElse(null), + properties.getOptionalInt(s"$key.$i.$MAX_LENGTH").orElse(null), + properties.getOptionalDouble(s"$key.$i.$MAX_VALUE").orElse(null), + properties.getOptionalDouble(s"$key.$i.$MIN_VALUE").orElse(null) ) name -> stats diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala index 5e0b42a380aeb8..8f2e4736f166fb 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala @@ -46,7 +46,7 @@ class StreamTableSourceDescriptor(tableEnv: StreamTableEnvironment, connector: C * Searches for the specified table source, configures it accordingly, and returns it. */ def toTableSource: TableSource[_] = { - val source = TableSourceFactoryService.findTableSourceFactory(this) + val source = TableSourceFactoryService.findAndCreateTableSource(this) source match { case _: StreamTableSource[_] => source case _ => throw new TableException( diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala index a49a41b5962b35..5118489d52eb01 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.descriptors +import org.apache.flink.table.descriptors.DescriptorProperties.toScala import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_ROW_COUNT, readColumnStats} import org.apache.flink.table.plan.stats.TableStats @@ -50,7 +51,7 @@ abstract class TableSourceDescriptor extends Descriptor { protected def getTableStats: Option[TableStats] = { val normalizedProps = new DescriptorProperties() addProperties(normalizedProps) - val rowCount = normalizedProps.getLong(STATISTICS_ROW_COUNT).map(v => Long.box(v)) + val rowCount = toScala(normalizedProps.getOptionalLong(STATISTICS_ROW_COUNT)) rowCount match { case Some(cnt) => val columnStats = readColumnStats(normalizedProps, STATISTICS_COLUMNS) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala index bec456543ece88..06d6bfba0d0383 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala @@ -21,11 +21,12 @@ package org.apache.flink.table.sources import java.util import org.apache.flink.table.api.TableException -import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION} +import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_PROPERTY_VERSION, CONNECTOR_TYPE} import org.apache.flink.table.descriptors.CsvValidator._ +import org.apache.flink.table.descriptors.DescriptorProperties.toScala import org.apache.flink.table.descriptors.FileSystemValidator.{CONNECTOR_PATH, CONNECTOR_TYPE_VALUE} -import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION} -import org.apache.flink.table.descriptors.SchemaValidator.{SCHEMA, SCHEMA_VERSION} +import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_PROPERTY_VERSION, FORMAT_TYPE} +import org.apache.flink.table.descriptors.SchemaValidator.SCHEMA import org.apache.flink.table.descriptors._ import org.apache.flink.types.Row @@ -38,9 +39,8 @@ class CsvTableSourceFactory extends TableSourceFactory[Row] { val context = new util.HashMap[String, String]() context.put(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE) context.put(FORMAT_TYPE, FORMAT_TYPE_VALUE) - context.put(CONNECTOR_VERSION, "1") - context.put(FORMAT_VERSION, "1") - context.put(SCHEMA_VERSION, "1") + context.put(CONNECTOR_PROPERTY_VERSION, "1") + context.put(FORMAT_PROPERTY_VERSION, "1") context } @@ -76,33 +76,36 @@ class CsvTableSourceFactory extends TableSourceFactory[Row] { // build val csvTableSourceBuilder = new CsvTableSource.Builder - val tableSchema = params.getTableSchema(SCHEMA).get - val encodingSchema = params.getTableSchema(FORMAT_FIELDS) + val formatSchema = params.getTableSchema(FORMAT_FIELDS) + val tableSchema = params.getTableSchema(SCHEMA) // the CsvTableSource needs some rework first // for now the schema must be equal to the encoding - if (!encodingSchema.contains(tableSchema)) { + if (!formatSchema.equals(tableSchema)) { throw new TableException( "Encodings that differ from the schema are not supported yet for CsvTableSources.") } - params.getString(CONNECTOR_PATH).foreach(csvTableSourceBuilder.path) - params.getString(FORMAT_FIELD_DELIMITER).foreach(csvTableSourceBuilder.fieldDelimiter) - params.getString(FORMAT_LINE_DELIMITER).foreach(csvTableSourceBuilder.lineDelimiter) + toScala(params.getOptionalString(CONNECTOR_PATH)) + .foreach(csvTableSourceBuilder.path) + toScala(params.getOptionalString(FORMAT_FIELD_DELIMITER)) + .foreach(csvTableSourceBuilder.fieldDelimiter) + toScala(params.getOptionalString(FORMAT_LINE_DELIMITER)) + .foreach(csvTableSourceBuilder.lineDelimiter) - encodingSchema.foreach { schema => - schema.getColumnNames.zip(schema.getTypes).foreach { case (name, tpe) => - csvTableSourceBuilder.field(name, tpe) - } + formatSchema.getColumnNames.zip(formatSchema.getTypes).foreach { case (name, tpe) => + csvTableSourceBuilder.field(name, tpe) } - params.getCharacter(FORMAT_QUOTE_CHARACTER).foreach(csvTableSourceBuilder.quoteCharacter) - params.getString(FORMAT_COMMENT_PREFIX).foreach(csvTableSourceBuilder.commentPrefix) - params.getBoolean(FORMAT_IGNORE_FIRST_LINE).foreach { flag => + toScala(params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER)) + .foreach(csvTableSourceBuilder.quoteCharacter) + toScala(params.getOptionalString(FORMAT_COMMENT_PREFIX)) + .foreach(csvTableSourceBuilder.commentPrefix) + toScala(params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE)).foreach { flag => if (flag) { csvTableSourceBuilder.ignoreFirstLine() } } - params.getBoolean(FORMAT_IGNORE_PARSE_ERRORS).foreach { flag => + toScala(params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS)).foreach { flag => if (flag) { csvTableSourceBuilder.ignoreParseErrors() } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala index f42d765ebca4c8..e5f696503279a1 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala @@ -41,10 +41,10 @@ trait TableSourceFactory[T] { * - connector.type * - format.type * - * Specified versions allow the framework to provide backwards compatible properties in case of - * string format changes: - * - connector.version - * - format.version + * Specified property versions allow the framework to provide backwards compatible properties + * in case of string format changes: + * - connector.property-version + * - format.property-version * * An empty context means that the factory matches for all requests. */ @@ -61,7 +61,8 @@ trait TableSourceFactory[T] { * - format.fields.#.type * - format.fields.#.name * - * Note: Use "#" to denote an array of values where "#" represents one or more digits. + * Note: Use "#" to denote an array of values where "#" represents one or more digits. Property + * versions like "format.property-version" must not be part of the supported properties. */ def supportedProperties(): util.List[String] diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala index 1e8e83691a249b..877cb7b5f39122 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala @@ -21,12 +21,10 @@ package org.apache.flink.table.sources import java.util.{ServiceConfigurationError, ServiceLoader} import org.apache.flink.table.api.{AmbiguousTableSourceException, NoMatchingTableSourceException, TableException, ValidationException} -import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION -import org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_VERSION -import org.apache.flink.table.descriptors.MetadataValidator.METADATA_VERSION -import org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_VERSION -import org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_VERSION -import org.apache.flink.table.descriptors.StatisticsValidator.STATISTICS_VERSION +import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_PROPERTY_VERSION +import org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_PROPERTY_VERSION +import org.apache.flink.table.descriptors.MetadataValidator.METADATA_PROPERTY_VERSION +import org.apache.flink.table.descriptors.StatisticsValidator.STATISTICS_PROPERTY_VERSION import org.apache.flink.table.descriptors._ import org.apache.flink.table.util.Logging @@ -40,13 +38,13 @@ object TableSourceFactoryService extends Logging { private lazy val loader = ServiceLoader.load(classOf[TableSourceFactory[_]]) - def findTableSourceFactory(descriptor: TableSourceDescriptor): TableSource[_] = { + def findAndCreateTableSource(descriptor: TableSourceDescriptor): TableSource[_] = { val properties = new DescriptorProperties() descriptor.addProperties(properties) - findTableSourceFactory(properties.asMap) + findAndCreateTableSource(properties.asMap.asScala.toMap) } - def findTableSourceFactory(properties: Map[String, String]): TableSource[_] = { + def findAndCreateTableSource(properties: Map[String, String]): TableSource[_] = { var matchingFactory: Option[(TableSourceFactory[_], Seq[String])] = None try { val iter = loader.iterator() @@ -73,12 +71,10 @@ object TableSourceFactoryService extends Logging { plainContext ++= requiredContext // we remove the versions for now until we have the first backwards compatibility case // with the version we can provide mappings in case the format changes - plainContext.remove(CONNECTOR_VERSION) - plainContext.remove(FORMAT_VERSION) - plainContext.remove(SCHEMA_VERSION) - plainContext.remove(ROWTIME_VERSION) - plainContext.remove(METADATA_VERSION) - plainContext.remove(STATISTICS_VERSION) + plainContext.remove(CONNECTOR_PROPERTY_VERSION) + plainContext.remove(FORMAT_PROPERTY_VERSION) + plainContext.remove(METADATA_PROPERTY_VERSION) + plainContext.remove(STATISTICS_PROPERTY_VERSION) // check if required context is met if (plainContext.forall(e => properties.contains(e._1) && properties(e._1) == e._2)) { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala index 329f790b9607cf..fcbd63f1bb2048 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala @@ -27,7 +27,7 @@ import org.apache.flink.table.expressions.{Expression, ResolvedFieldReference} * * Note: This extractor only works for StreamTableSources. */ -class StreamRecordTimestamp extends TimestampExtractor { +final class StreamRecordTimestamp extends TimestampExtractor { /** No argument fields required. */ override def getArgumentFields: Array[String] = Array() @@ -42,5 +42,8 @@ class StreamRecordTimestamp extends TimestampExtractor { override def getExpression(fieldAccesses: Array[ResolvedFieldReference]): Expression = { org.apache.flink.table.expressions.StreamRecordTimestamp() } +} +object StreamRecordTimestamp { + val INSTANCE: StreamRecordTimestamp = new StreamRecordTimestamp } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala index 15cf13bd722798..85778ca02b1056 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala @@ -18,16 +18,36 @@ package org.apache.flink.table.descriptors +import java.util + import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.table.api.{TableSchema, Types, ValidationException} import org.junit.Test +import scala.collection.JavaConverters._ + class CsvTest extends DescriptorTestBase { - @Test - def testCsv(): Unit = { - val desc = Csv() + @Test(expected = classOf[ValidationException]) + def testInvalidType(): Unit = { + addPropertyAndVerify(descriptors().get(0), "format.fields.0.type", "WHATEVER") + } + + @Test(expected = classOf[ValidationException]) + def testInvalidField(): Unit = { + addPropertyAndVerify(descriptors().get(0), "format.fields.10.name", "WHATEVER") + } + + @Test(expected = classOf[ValidationException]) + def testInvalidQuoteCharacter(): Unit = { + addPropertyAndVerify(descriptors().get(0), "format.quote-character", "qq") + } + + // ---------------------------------------------------------------------------------------------- + + override def descriptors(): util.List[Descriptor] = { + val desc1 = Csv() .field("field1", "STRING") .field("field2", Types.SQL_TIMESTAMP) .field("field3", TypeExtractor.createTypeInfo(classOf[Class[_]])) @@ -35,9 +55,21 @@ class CsvTest extends DescriptorTestBase { Array[String]("test", "row"), Array[TypeInformation[_]](Types.INT, Types.STRING))) .lineDelimiter("^") - val expected = Seq( + + val desc2 = Csv() + .schema(new TableSchema( + Array[String]("test", "row"), + Array[TypeInformation[_]](Types.INT, Types.STRING))) + .quoteCharacter('#') + .ignoreFirstLine() + + util.Arrays.asList(desc1, desc2) + } + + override def properties(): util.List[util.Map[String, String]] = { + val props1 = Map( "format.type" -> "csv", - "format.version" -> "1", + "format.property-version" -> "1", "format.fields.0.name" -> "field1", "format.fields.0.type" -> "STRING", "format.fields.1.name" -> "field2", @@ -47,53 +79,18 @@ class CsvTest extends DescriptorTestBase { "format.fields.3.name" -> "field4", "format.fields.3.type" -> "ROW(test INT, row VARCHAR)", "format.line-delimiter" -> "^") - verifyProperties(desc, expected) - } - @Test - def testCsvTableSchema(): Unit = { - val desc = Csv() - .schema(new TableSchema( - Array[String]("test", "row"), - Array[TypeInformation[_]](Types.INT, Types.STRING))) - .quoteCharacter('#') - .ignoreFirstLine() - val expected = Seq( + val props2 = Map( "format.type" -> "csv", - "format.version" -> "1", + "format.property-version" -> "1", "format.fields.0.name" -> "test", "format.fields.0.type" -> "INT", "format.fields.1.name" -> "row", "format.fields.1.type" -> "VARCHAR", "format.quote-character" -> "#", "format.ignore-first-line" -> "true") - verifyProperties(desc, expected) - } - @Test(expected = classOf[ValidationException]) - def testInvalidType(): Unit = { - verifyInvalidProperty("format.fields.0.type", "WHATEVER") - } - - @Test(expected = classOf[ValidationException]) - def testInvalidField(): Unit = { - verifyInvalidProperty("format.fields.10.name", "WHATEVER") - } - - @Test(expected = classOf[ValidationException]) - def testInvalidQuoteCharacter(): Unit = { - verifyInvalidProperty("format.quote-character", "qq") - } - - override def descriptor(): Descriptor = { - Csv() - .field("field1", "STRING") - .field("field2", Types.SQL_TIMESTAMP) - .field("field3", TypeExtractor.createTypeInfo(classOf[Class[_]])) - .field("field4", Types.ROW( - Array[String]("test", "row"), - Array[TypeInformation[_]](Types.INT, Types.STRING))) - .lineDelimiter("^") + util.Arrays.asList(props1.asJava, props2.asJava) } override def validator(): DescriptorValidator = { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala index 3a59c9be5133de..7a98b0be23fe5d 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala @@ -18,37 +18,84 @@ package org.apache.flink.table.descriptors +import org.apache.flink.util.Preconditions import org.junit.Assert.assertEquals +import org.junit.Test + +import scala.collection.JavaConverters._ abstract class DescriptorTestBase { /** - * Returns a valid descriptor. + * Returns a set of valid descriptors. + * This method is implemented in both Scala and Java. */ - def descriptor(): Descriptor + def descriptors(): java.util.List[Descriptor] /** - * Returns a validator that can validate this descriptor. + * Returns a set of properties for each valid descriptor. + * This code is implemented in both Scala and Java. + */ + def properties(): java.util.List[java.util.Map[String, String]] + + /** + * Returns a validator that can validate all valid descriptors. */ def validator(): DescriptorValidator - def verifyProperties(descriptor: Descriptor, expected: Seq[(String, String)]): Unit = { + @Test + def testValidation(): Unit = { + val d = descriptors().asScala + val p = properties().asScala + + Preconditions.checkArgument(d.length == p.length) + + d.zip(p).foreach { case (desc, props) => + verifyProperties(desc, props.asScala.toMap) + } + } + + def verifyProperties(descriptor: Descriptor, expected: Map[String, String]): Unit = { val normProps = new DescriptorProperties descriptor.addProperties(normProps) - assertEquals(expected.toMap, normProps.asMap) + + // test produced properties + assertEquals(expected, normProps.asMap.asScala.toMap) + + // test validation logic + validator().validate(normProps) } - def verifyInvalidProperty(property: String, invalidValue: String): Unit = { + def addPropertyAndVerify( + descriptor: Descriptor, + property: String, + invalidValue: String): Unit = { val properties = new DescriptorProperties - descriptor().addProperties(properties) + descriptor.addProperties(properties) properties.unsafePut(property, invalidValue) validator().validate(properties) } - def verifyMissingProperty(removeProperty: String): Unit = { + def removePropertyAndVerify(descriptor: Descriptor, removeProperty: String): Unit = { val properties = new DescriptorProperties - descriptor().addProperties(properties) + descriptor.addProperties(properties) properties.unsafeRemove(removeProperty) validator().validate(properties) } } + +class TestTableSourceDescriptor(connector: ConnectorDescriptor) + extends TableSourceDescriptor { + + this.connectorDescriptor = Some(connector) + + def addFormat(format: FormatDescriptor): TestTableSourceDescriptor = { + this.formatDescriptor = Some(format) + this + } + + def addSchema(schema: Schema): TestTableSourceDescriptor = { + this.schemaDescriptor = Some(schema) + this + } +} diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala index 3452e8d31a1cf0..1162694a01dc73 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala @@ -18,36 +18,41 @@ package org.apache.flink.table.descriptors +import java.util + import org.apache.flink.table.api.ValidationException import org.junit.Test -class FileSystemTest extends DescriptorTestBase { +import scala.collection.JavaConverters._ - @Test - def testFileSystem(): Unit = { - val desc = FileSystem().path("/myfile") - val expected = Seq( - "connector.type" -> "filesystem", - "connector.version" -> "1", - "connector.path" -> "/myfile") - verifyProperties(desc, expected) - } +class FileSystemTest extends DescriptorTestBase { @Test(expected = classOf[ValidationException]) def testInvalidPath(): Unit = { - verifyInvalidProperty("connector.path", "") + addPropertyAndVerify(descriptors().get(0), "connector.path", "") } @Test(expected = classOf[ValidationException]) def testMissingPath(): Unit = { - verifyMissingProperty("connector.path") + removePropertyAndVerify(descriptors().get(0), "connector.path") } - override def descriptor(): Descriptor = { - FileSystem().path("/myfile") + // ---------------------------------------------------------------------------------------------- + + override def descriptors(): util.List[Descriptor] = { + util.Arrays.asList(FileSystem().path("/myfile")) } override def validator(): DescriptorValidator = { new FileSystemValidator() } + + override def properties(): util.List[util.Map[String, String]] = { + val desc = Map( + "connector.type" -> "filesystem", + "connector.property-version" -> "1", + "connector.path" -> "/myfile") + + util.Arrays.asList(desc.asJava) + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala deleted file mode 100644 index 756ca231bb6836..00000000000000 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.descriptors - -import org.apache.flink.table.api.ValidationException -import org.junit.Test - -class JsonTest extends DescriptorTestBase { - - @Test - def testJson(): Unit = { - val schema = - """ - |{ - | "title": "Person", - | "type": "object", - | "properties": { - | "firstName": { - | "type": "string" - | }, - | "lastName": { - | "type": "string" - | }, - | "age": { - | "description": "Age in years", - | "type": "integer", - | "minimum": 0 - | } - | }, - | "required": ["firstName", "lastName"] - |} - |""".stripMargin - val desc = Json() - .schema(schema) - .failOnMissingField(true) - val expected = Seq( - "format.type" -> "json", - "format.version" -> "1", - "format.schema-string" -> schema, - "format.fail-on-missing-field" -> "true") - verifyProperties(desc, expected) - } - - @Test(expected = classOf[ValidationException]) - def testInvalidMissingField(): Unit = { - verifyInvalidProperty("format.fail-on-missing-field", "DDD") - } - - @Test(expected = classOf[ValidationException]) - def testMissingSchema(): Unit = { - verifyMissingProperty("format.schema-string") - } - - override def descriptor(): Descriptor = { - Json().schema("test") - } - - override def validator(): DescriptorValidator = { - new JsonValidator() - } -} diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala index a1854ce8880464..64965b0fa9eea8 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala @@ -18,38 +18,42 @@ package org.apache.flink.table.descriptors +import java.util + import org.apache.flink.table.api.ValidationException import org.junit.Test -class MetadataTest extends DescriptorTestBase { +import scala.collection.JavaConverters._ - @Test - def testMetadata(): Unit = { - val desc = Metadata() - .comment("Some additional comment") - .creationTime(123L) - .lastAccessTime(12020202L) - val expected = Seq( - "metadata.comment" -> "Some additional comment", - "metadata.creation-time" -> "123", - "metadata.last-access-time" -> "12020202" - ) - verifyProperties(desc, expected) - } +class MetadataTest extends DescriptorTestBase { @Test(expected = classOf[ValidationException]) def testInvalidCreationTime(): Unit = { - verifyInvalidProperty("metadata.creation-time", "dfghj") + addPropertyAndVerify(descriptors().get(0), "metadata.creation-time", "dfghj") } - override def descriptor(): Descriptor = { - Metadata() + // ---------------------------------------------------------------------------------------------- + + override def descriptors(): util.List[Descriptor] = { + val desc = Metadata() .comment("Some additional comment") .creationTime(123L) .lastAccessTime(12020202L) + + util.Arrays.asList(desc) } override def validator(): DescriptorValidator = { new MetadataValidator() } + + override def properties(): util.List[util.Map[String, String]] = { + val props = Map( + "metadata.comment" -> "Some additional comment", + "metadata.creation-time" -> "123", + "metadata.last-access-time" -> "12020202" + ) + + util.Arrays.asList(props.asJava) + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala index 80050fc26d4b68..7968b481db3438 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala @@ -18,6 +18,8 @@ package org.apache.flink.table.descriptors +import java.util + import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.table.api.ValidationException import org.apache.flink.table.descriptors.RowtimeTest.CustomAssigner @@ -25,41 +27,58 @@ import org.apache.flink.table.sources.wmstrategies.PunctuatedWatermarkAssigner import org.apache.flink.types.Row import org.junit.Test -class RowtimeTest extends DescriptorTestBase { +import scala.collection.JavaConverters._ - @Test - def testRowtime(): Unit = { - val desc = Rowtime() - .timestampsFromField("otherField") - .watermarksPeriodicBounding(1000L) - val expected = Seq( - "rowtime.0.version" -> "1", - "rowtime.0.timestamps.type" -> "from-field", - "rowtime.0.timestamps.from" -> "otherField", - "rowtime.0.watermarks.type" -> "periodic-bounding", - "rowtime.0.watermarks.delay" -> "1000" - ) - verifyProperties(desc, expected) - } +class RowtimeTest extends DescriptorTestBase { @Test(expected = classOf[ValidationException]) def testInvalidWatermarkType(): Unit = { - verifyInvalidProperty("rowtime.0.watermarks.type", "xxx") + addPropertyAndVerify(descriptors().get(0), "rowtime.watermarks.type", "xxx") } @Test(expected = classOf[ValidationException]) def testMissingWatermarkClass(): Unit = { - verifyMissingProperty("rowtime.0.watermarks.class") + removePropertyAndVerify(descriptors().get(1), "rowtime.watermarks.class") } - override def descriptor(): Descriptor = { - Rowtime() + // ---------------------------------------------------------------------------------------------- + + override def descriptors(): util.List[Descriptor] = { + val desc1 = Rowtime() + .timestampsFromField("otherField") + .watermarksPeriodicBounding(1000L) + + val desc2 = Rowtime() .timestampsFromSource() .watermarksFromStrategy(new CustomAssigner()) + + util.Arrays.asList(desc1, desc2) } override def validator(): DescriptorValidator = { - new RowtimeValidator("rowtime.0.") + new RowtimeValidator() + } + + override def properties(): util.List[util.Map[String, String]] = { + val props1 = Map( + "rowtime.timestamps.type" -> "from-field", + "rowtime.timestamps.from" -> "otherField", + "rowtime.watermarks.type" -> "periodic-bounded", + "rowtime.watermarks.delay" -> "1000" + ) + + val props2 = Map( + "rowtime.timestamps.type" -> "from-source", + "rowtime.watermarks.type" -> "custom", + "rowtime.watermarks.class" -> "org.apache.flink.table.descriptors.RowtimeTest$CustomAssigner", + "rowtime.watermarks.serialized" -> ("rO0ABXNyAD1vcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmRlc2NyaX" + + "B0b3JzLlJvd3RpbWVUZXN0JEN1c3RvbUFzc2lnbmVyeDcuDvfbu0kCAAB4cgBHb3JnLmFwYWNoZS5mbGluay" + + "50YWJsZS5zb3VyY2VzLndtc3RyYXRlZ2llcy5QdW5jdHVhdGVkV2F0ZXJtYXJrQXNzaWduZXKBUc57oaWu9A" + + "IAAHhyAD1vcmcuYXBhY2hlLmZsaW5rLnRhYmxlLnNvdXJjZXMud21zdHJhdGVnaWVzLldhdGVybWFya1N0cm" + + "F0ZWd5mB_uSxDZ8-MCAAB4cA") + ) + + util.Arrays.asList(props1.asJava, props2.asJava) } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala index f663a96be15450..589ec4f582b3de 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala @@ -18,21 +18,54 @@ package org.apache.flink.table.descriptors +import java.util + import org.apache.flink.table.api.{Types, ValidationException} import org.junit.Test +import scala.collection.JavaConverters._ + class SchemaTest extends DescriptorTestBase { - @Test - def testSchema(): Unit = { - val desc = Schema() + @Test(expected = classOf[ValidationException]) + def testInvalidType(): Unit = { + addPropertyAndVerify( + descriptors().get(0), + "schema.1.type", "dfghj") + } + + @Test(expected = classOf[ValidationException]) + def testBothRowtimeAndProctime(): Unit = { + addPropertyAndVerify( + descriptors().get(0), + "schema.2.rowtime.watermarks.type", "from-source") + } + + // ---------------------------------------------------------------------------------------------- + + override def descriptors(): util.List[Descriptor] = { + val desc1 = Schema() .field("myField", Types.BOOLEAN) .field("otherField", "VARCHAR").from("csvField") .field("p", Types.SQL_TIMESTAMP).proctime() .field("r", Types.SQL_TIMESTAMP).rowtime( Rowtime().timestampsFromSource().watermarksFromSource()) - val expected = Seq( - "schema.version" -> "1", + + val desc2 = Schema() + .field("myField", Types.BOOLEAN) + .field("otherField", "VARCHAR").from("csvField") + .field("p", Types.SQL_TIMESTAMP).proctime() + .field("r", Types.SQL_TIMESTAMP) + + util.Arrays.asList(desc1, desc2) + } + + override def validator(): DescriptorValidator = { + new SchemaValidator(isStreamEnvironment = true) + } + + override def properties(): util.List[util.Map[String, String]] = { + val props1 = Map( "schema.0.name" -> "myField", "schema.0.type" -> "BOOLEAN", "schema.1.name" -> "otherField", @@ -43,34 +76,23 @@ class SchemaTest extends DescriptorTestBase { "schema.2.proctime" -> "true", "schema.3.name" -> "r", "schema.3.type" -> "TIMESTAMP", - "schema.3.rowtime.0.version" -> "1", - "schema.3.rowtime.0.watermarks.type" -> "from-source", - "schema.3.rowtime.0.timestamps.type" -> "from-source" + "schema.3.rowtime.watermarks.type" -> "from-source", + "schema.3.rowtime.timestamps.type" -> "from-source" ) - verifyProperties(desc, expected) - } - @Test(expected = classOf[ValidationException]) - def testInvalidType(): Unit = { - verifyInvalidProperty("schema.1.type", "dfghj") - } - - @Test(expected = classOf[ValidationException]) - def testBothRowtimeAndProctime(): Unit = { - verifyInvalidProperty("schema.2.rowtime.0.version", "1") - verifyInvalidProperty("schema.2.rowtime.0.watermarks.type", "from-source") - verifyInvalidProperty("schema.2.rowtime.0.timestamps.type", "from-source") - } - - override def descriptor(): Descriptor = { - Schema() - .field("myField", Types.BOOLEAN) - .field("otherField", "VARCHAR").from("csvField") - .field("p", Types.SQL_TIMESTAMP).proctime() - .field("r", Types.SQL_TIMESTAMP) - } + val props2 = Map( + "schema.0.name" -> "myField", + "schema.0.type" -> "BOOLEAN", + "schema.1.name" -> "otherField", + "schema.1.type" -> "VARCHAR", + "schema.1.from" -> "csvField", + "schema.2.name" -> "p", + "schema.2.type" -> "TIMESTAMP", + "schema.2.proctime" -> "true", + "schema.3.name" -> "r", + "schema.3.type" -> "TIMESTAMP" + ) - override def validator(): DescriptorValidator = { - new SchemaValidator(isStreamEnvironment = true) + util.Arrays.asList(props1.asJava, props2.asJava) } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala new file mode 100644 index 00000000000000..ba05dfff2074db --- /dev/null +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors + +import java.util.Optional + +import org.apache.flink.table.api.{TableSchema, Types} +import org.apache.flink.table.sources.tsextractors.StreamRecordTimestamp +import org.apache.flink.table.sources.wmstrategies.PreserveWatermarks +import org.junit.Assert.{assertEquals, assertTrue} +import org.junit.Test + +import scala.collection.JavaConverters._ + +/** + * Tests for [[SchemaValidator]]. + */ +class SchemaValidatorTest { + + @Test + def testSchema(): Unit = { + val desc1 = Schema() + .field("otherField", Types.STRING).from("csvField") + .field("abcField", Types.STRING) + .field("p", Types.SQL_TIMESTAMP).proctime() + .field("r", Types.SQL_TIMESTAMP).rowtime( + Rowtime().timestampsFromSource().watermarksFromSource()) + val props = new DescriptorProperties() + desc1.addProperties(props) + + val inputSchema = TableSchema.builder() + .field("csvField", Types.STRING) + .field("abcField", Types.STRING) + .field("myField", Types.BOOLEAN) + .build() + + // test proctime + assertEquals(Optional.of("p"), SchemaValidator.deriveProctimeAttribute(props)) + + // test rowtime + val rowtime = SchemaValidator.deriveRowtimeAttributes(props).get(0) + assertEquals("r", rowtime.getAttributeName) + assertTrue(rowtime.getTimestampExtractor.isInstanceOf[StreamRecordTimestamp]) + assertTrue(rowtime.getWatermarkStrategy.isInstanceOf[PreserveWatermarks]) + + // test field mapping + val expectedMapping = Map("otherField" -> "csvField", "abcField" -> "abcField").asJava + assertEquals( + expectedMapping, + SchemaValidator.deriveFieldMapping(props, Optional.of(inputSchema))) + + // test field format + val formatSchema = SchemaValidator.deriveFormatFields(props) + val expectedFormatSchema = TableSchema.builder() + .field("csvField", Types.STRING) // aliased + .field("abcField", Types.STRING) + .build() + assertEquals(expectedFormatSchema, formatSchema) + } +} diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala index 3b248b47b76159..2def0c317a4982 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala @@ -24,17 +24,44 @@ import org.apache.flink.table.api.ValidationException import org.apache.flink.table.plan.stats.{ColumnStats, TableStats} import org.junit.Test +import scala.collection.JavaConverters._ + class StatisticsTest extends DescriptorTestBase { - @Test - def testStatistics(): Unit = { - val desc = Statistics() + @Test(expected = classOf[ValidationException]) + def testInvalidRowCount(): Unit = { + addPropertyAndVerify(descriptors().get(0), "statistics.row-count", "abx") + } + + @Test(expected = classOf[ValidationException]) + def testMissingName(): Unit = { + removePropertyAndVerify(descriptors().get(0), "statistics.columns.0.name") + } + + // ---------------------------------------------------------------------------------------------- + + override def descriptors(): util.List[Descriptor] = { + val desc1 = Statistics() .rowCount(1000L) .columnStats("a", ColumnStats(1L, 2L, 3.0, 4, 5, 6)) .columnAvgLength("b", 42.0) .columnNullCount("a", 300) - val expected = Seq( - "statistics.version" -> "1", + + val map = new util.HashMap[String, ColumnStats]() + map.put("a", ColumnStats(null, 2L, 3.0, null, 5, 6)) + val desc2 = Statistics() + .tableStats(TableStats(32L, map)) + + util.Arrays.asList(desc1, desc2) + } + + override def validator(): DescriptorValidator = { + new StatisticsValidator() + } + + override def properties(): util.List[util.Map[String, String]] = { + val props1 = Map( + "statistics.property-version" -> "1", "statistics.row-count" -> "1000", "statistics.columns.0.name" -> "a", "statistics.columns.0.distinct-count" -> "1", @@ -46,17 +73,9 @@ class StatisticsTest extends DescriptorTestBase { "statistics.columns.1.name" -> "b", "statistics.columns.1.avg-length" -> "42.0" ) - verifyProperties(desc, expected) - } - @Test - def testStatisticsTableStats(): Unit = { - val map = new util.HashMap[String, ColumnStats]() - map.put("a", ColumnStats(null, 2L, 3.0, null, 5, 6)) - val desc = Statistics() - .tableStats(TableStats(32L, map)) - val expected = Seq( - "statistics.version" -> "1", + val props2 = Map( + "statistics.property-version" -> "1", "statistics.row-count" -> "32", "statistics.columns.0.name" -> "a", "statistics.columns.0.null-count" -> "2", @@ -64,28 +83,7 @@ class StatisticsTest extends DescriptorTestBase { "statistics.columns.0.max-value" -> "5", "statistics.columns.0.min-value" -> "6" ) - verifyProperties(desc, expected) - } - @Test(expected = classOf[ValidationException]) - def testInvalidRowCount(): Unit = { - verifyInvalidProperty("statistics.row-count", "abx") - } - - @Test(expected = classOf[ValidationException]) - def testMissingName(): Unit = { - verifyMissingProperty("statistics.columns.0.name") - } - - override def descriptor(): Descriptor = { - Statistics() - .rowCount(1000L) - .columnStats("a", ColumnStats(1L, 2L, 3.0, 4, 5, 6)) - .columnAvgLength("b", 42.0) - .columnNullCount("a", 300) - } - - override def validator(): DescriptorValidator = { - new StatisticsValidator() + util.Arrays.asList(props1.asJava, props2.asJava) } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala index 5e9b5a24c022b2..279e9a41344b1e 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala @@ -19,8 +19,8 @@ package org.apache.flink.table.sources import org.apache.flink.table.api.{NoMatchingTableSourceException, TableException, ValidationException} -import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION} -import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION} +import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION} +import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_PROPERTY_VERSION} import org.junit.Assert.assertTrue import org.junit.Test @@ -31,44 +31,44 @@ class TableSourceFactoryServiceTest { @Test def testValidProperties(): Unit = { val props = properties() - assertTrue(TableSourceFactoryService.findTableSourceFactory(props.toMap) != null) + assertTrue(TableSourceFactoryService.findAndCreateTableSource(props.toMap) != null) } @Test(expected = classOf[NoMatchingTableSourceException]) def testInvalidContext(): Unit = { val props = properties() props.put(CONNECTOR_TYPE, "FAIL") - TableSourceFactoryService.findTableSourceFactory(props.toMap) + TableSourceFactoryService.findAndCreateTableSource(props.toMap) } @Test def testDifferentContextVersion(): Unit = { val props = properties() - props.put(CONNECTOR_VERSION, "2") + props.put(CONNECTOR_PROPERTY_VERSION, "2") // the table source should still be found - assertTrue(TableSourceFactoryService.findTableSourceFactory(props.toMap) != null) + assertTrue(TableSourceFactoryService.findAndCreateTableSource(props.toMap) != null) } @Test(expected = classOf[ValidationException]) def testUnsupportedProperty(): Unit = { val props = properties() props.put("format.path_new", "/new/path") - TableSourceFactoryService.findTableSourceFactory(props.toMap) + TableSourceFactoryService.findAndCreateTableSource(props.toMap) } @Test(expected = classOf[TableException]) def testFailingFactory(): Unit = { val props = properties() props.put("failing", "true") - TableSourceFactoryService.findTableSourceFactory(props.toMap) + TableSourceFactoryService.findAndCreateTableSource(props.toMap) } private def properties(): mutable.Map[String, String] = { val properties = mutable.Map[String, String]() properties.put(CONNECTOR_TYPE, "test") properties.put(FORMAT_TYPE, "test") - properties.put(CONNECTOR_VERSION, "1") - properties.put(FORMAT_VERSION, "1") + properties.put(CONNECTOR_PROPERTY_VERSION, "1") + properties.put(FORMAT_PROPERTY_VERSION, "1") properties.put("format.path", "/path/to/target") properties.put("schema.0.name", "a") properties.put("schema.1.name", "b") diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala index ae75f99e14d637..ee3d637b7b1429 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala @@ -22,8 +22,8 @@ import java.util import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.table.api.TableSchema -import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION} -import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION} +import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION} +import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_PROPERTY_VERSION} import org.apache.flink.types.Row class TestTableSourceFactory extends TableSourceFactory[Row] { @@ -32,8 +32,8 @@ class TestTableSourceFactory extends TableSourceFactory[Row] { val context = new util.HashMap[String, String]() context.put(CONNECTOR_TYPE, "test") context.put(FORMAT_TYPE, "test") - context.put(CONNECTOR_VERSION, "1") - context.put(FORMAT_VERSION, "1") + context.put(CONNECTOR_PROPERTY_VERSION, "1") + context.put(FORMAT_PROPERTY_VERSION, "1") context } From f89fce83eaadddcc3aa8dee2167188a2b53b0c8e Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Mon, 26 Feb 2018 16:41:24 +0100 Subject: [PATCH 0046/2294] [FLINK-8791] [docs] Fix documentation about configuring dependencies This closes #5586 --- docs/dev/linking.md | 96 ------- docs/dev/linking_with_flink.md | 146 ----------- docs/redirects/linking_with_flink.md | 25 ++ .../linking_with_optional_modules.md | 25 ++ docs/start/dependencies.md | 244 ++++++++++++++++++ 5 files changed, 294 insertions(+), 242 deletions(-) delete mode 100644 docs/dev/linking.md delete mode 100644 docs/dev/linking_with_flink.md create mode 100644 docs/redirects/linking_with_flink.md create mode 100644 docs/redirects/linking_with_optional_modules.md create mode 100644 docs/start/dependencies.md diff --git a/docs/dev/linking.md b/docs/dev/linking.md deleted file mode 100644 index 78ef54494831f4..00000000000000 --- a/docs/dev/linking.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -nav-title: "Linking with Optional Modules" -title: "Linking with modules not contained in the binary distribution" -nav-parent_id: start -nav-pos: 10 ---- - - -The binary distribution contains jar packages in the `lib` folder that are automatically -provided to the classpath of your distributed programs. Almost all of Flink classes are -located there with a few exceptions, for example the streaming connectors and some freshly -added modules. To run code depending on these modules you need to make them accessible -during runtime, for which we suggest two options: - -1. Either copy the required jar files to the `lib` folder onto all of your TaskManagers. -Note that you have to restart your TaskManagers after this. -2. Or package them with your code. - -The latter version is recommended as it respects the classloader management in Flink. - -### Packaging dependencies with your usercode with Maven - -To provide these dependencies not included by Flink we suggest two options with Maven. - -1. The maven assembly plugin builds a so-called uber-jar (executable jar) containing all your dependencies. -The assembly configuration is straight-forward, but the resulting jar might become bulky. -See [maven-assembly-plugin](http://maven.apache.org/plugins/maven-assembly-plugin/usage.html) for further information. -2. The maven unpack plugin unpacks the relevant parts of the dependencies and -then packages it with your code. - -Using the latter approach in order to bundle the Kafka connector, `flink-connector-kafka` -you would need to add the classes from both the connector and the Kafka API itself. Add -the following to your plugins section. - -~~~xml - - org.apache.maven.plugins - maven-dependency-plugin - 2.9 - - - unpack - - prepare-package - - unpack - - - - - - org.apache.flink - flink-connector-kafka - {{ site.version }} - jar - false - ${project.build.directory}/classes - org/apache/flink/** - - - - org.apache.kafka - kafka_ - - jar - false - ${project.build.directory}/classes - kafka/** - - - - - - -~~~ - -Now when running `mvn clean package` the produced jar includes the required dependencies. - -{% top %} diff --git a/docs/dev/linking_with_flink.md b/docs/dev/linking_with_flink.md deleted file mode 100644 index f2380b23a07f68..00000000000000 --- a/docs/dev/linking_with_flink.md +++ /dev/null @@ -1,146 +0,0 @@ ---- -title: "Linking with Flink" -nav-parent_id: start -nav-pos: 2 ---- - - -To write programs with Flink, you need to include the Flink library corresponding to -your programming language in your project. - -The simplest way to do this is to use one of the quickstart scripts: either for -[Java]({{ site.baseurl }}/quickstart/java_api_quickstart.html) or for [Scala]({{ site.baseurl }}/quickstart/scala_api_quickstart.html). They -create a blank project from a template (a Maven Archetype), which sets up everything for you. To -manually create the project, you can use the archetype and create a project by calling: - -

    -
    -{% highlight bash %} -mvn archetype:generate \ - -DarchetypeGroupId=org.apache.flink \ - -DarchetypeArtifactId=flink-quickstart-java \ - -DarchetypeVersion={{site.version }} -{% endhighlight %} -
    -
    -{% highlight bash %} -mvn archetype:generate \ - -DarchetypeGroupId=org.apache.flink \ - -DarchetypeArtifactId=flink-quickstart-scala \ - -DarchetypeVersion={{site.version }} -{% endhighlight %} -
    -
    - -The archetypes are working for stable releases and preview versions (`-SNAPSHOT`). - -If you want to add Flink to an existing Maven project, add the following entry to your -*dependencies* section in the *pom.xml* file of your project: - -
    -
    -{% highlight xml %} - - - org.apache.flink - flink-streaming-java{{ site.scala_version_suffix }} - {{site.version }} - - - - org.apache.flink - flink-java - {{site.version }} - - - org.apache.flink - flink-clients{{ site.scala_version_suffix }} - {{site.version }} - -{% endhighlight %} -
    -
    -{% highlight xml %} - - - org.apache.flink - flink-streaming-scala{{ site.scala_version_suffix }} - {{site.version }} - - - - org.apache.flink - flink-scala{{ site.scala_version_suffix }} - {{site.version }} - - - org.apache.flink - flink-clients{{ site.scala_version_suffix }} - {{site.version }} - -{% endhighlight %} - -**Important:** When working with the Scala API you must have one of these two imports: -{% highlight scala %} -import org.apache.flink.api.scala._ -{% endhighlight %} - -or - -{% highlight scala %} -import org.apache.flink.api.scala.createTypeInformation -{% endhighlight %} - -The reason is that Flink analyzes the types that are used in a program and generates serializers -and comparators for them. By having either of those imports you enable an implicit conversion -that creates the type information for Flink operations. - -If you would rather use SBT, see [here]({{ site.baseurl }}/quickstart/scala_api_quickstart.html#sbt). -
    -
    - -#### Scala Dependency Versions - -Because Scala 2.10 binary is not compatible with Scala 2.11 binary, we provide multiple artifacts -to support both Scala versions. - -Starting from the 0.10 line, we cross-build all Flink modules for both 2.10 and 2.11. If you want -to run your program on Flink with Scala 2.11, you need to add a `_2.11` suffix to the `artifactId` -values of the Flink modules in your dependencies section. - -If you are looking for building Flink with Scala 2.11, please check -[build guide]({{ site.baseurl }}/start/building.html#scala-versions). - -#### Hadoop Dependency Versions - -If you are using Flink together with Hadoop, the version of the dependency may vary depending on the -version of Hadoop (or more specifically, HDFS) that you want to use Flink with. Please refer to the -[downloads page](http://flink.apache.org/downloads.html) for a list of available versions, and instructions -on how to link with custom versions of Hadoop. - -In order to link against the latest SNAPSHOT versions of the code, please follow -[this guide](http://flink.apache.org/how-to-contribute.html#snapshots-nightly-builds). - -The *flink-clients* dependency is only necessary to invoke the Flink program locally (for example to -run it standalone for testing and debugging). If you intend to only export the program as a JAR -file and [run it on a cluster]({{ site.baseurl }}/dev/cluster_execution.html), you can skip that dependency. - -{% top %} - diff --git a/docs/redirects/linking_with_flink.md b/docs/redirects/linking_with_flink.md new file mode 100644 index 00000000000000..1289487c9ff22f --- /dev/null +++ b/docs/redirects/linking_with_flink.md @@ -0,0 +1,25 @@ +--- +title: "Linking with Flink" +layout: redirect +redirect: /start/dependencies.html +permalink: /dev/linking_with_flink.html +--- + + diff --git a/docs/redirects/linking_with_optional_modules.md b/docs/redirects/linking_with_optional_modules.md new file mode 100644 index 00000000000000..e494fbc54d082d --- /dev/null +++ b/docs/redirects/linking_with_optional_modules.md @@ -0,0 +1,25 @@ +--- +title: "Linking with Optional Modules" +layout: redirect +redirect: /start/dependencies.html +permalink: /dev/linking.html +--- + + diff --git a/docs/start/dependencies.md b/docs/start/dependencies.md new file mode 100644 index 00000000000000..1375c6f30f8eac --- /dev/null +++ b/docs/start/dependencies.md @@ -0,0 +1,244 @@ +--- +title: "Configuring Dependencies, Connectors, Libraries" +nav-parent_id: start +nav-pos: 2 +--- + + +Every Flink application depends on a set of Flink libraries. At the bare minimum, the application depends +on the Flink APIs. Many applications depend in addition on certain connector libraries (like Kafka, Cassandra, etc.). +When running Flink applications (either in a distributed deployment, or in the IDE for testing), the Flink +runtime library must be available as well. + + +## Flink Core and Application Dependencies + +As with most systems that run user-defined applications, there are two broad categories of dependencies and libraries in Flink: + + - **Flink Core Dependenies**: Flink itself consists of a set of classes and dependencies that are needed to run the system, for example + coordination, networking, checkpoints, failover, APIs, operations (such as windowing), resource management, etc. + The set of all these classes and dependencies forms the core of Flink's runtime and must be present when a Flink + application is started. + + These core classes and dependencies are packaged in the `flink-dist` jar. They are part of Flink's `lib` folder and + part of the basic Flink container images. Think of these dependencies as similar to Java's core library (`rt.jar`, `charsets.jar`, etc.), + which contains the classes like `String` and `List`. + + The Flink Core Dependencies do not contain any connectors or libraries (CEP, SQL, ML, etc.) in order to avoid having an excessive + number of dependencies and classes in the classpath by default. In fact, we try to keep the core dependencies as slim as possible + to keep the default classpath small and avoid dependency clashes. + + - The **User Application Dependencies** are all connectors, formats, or libraries that a specific user application needs. + + The user application is typically packaged into an *application jar*, which contains the application code and the required + connector and library dependencies. + + The user application dependencies explicitly do not include the Flink DataSet / DataStream APIs and runtime dependencies, + because those are already part of Flink's Core Dependencies. + + +## Setting up a Project: Basic Dependencies + +Every Flink application needs as the bare minimum the API dependencies, to develop against. +For Maven, you can use the [Java Project Template]({{ site.baseurl }}/quickstart/java_api_quickstart.html) +or [Scala Project Template]({{ site.baseurl }}/quickstart/scala_api_quickstart.html) to create +a program skeleton with these initial dependencies. + +When setting up a project manually, you need to add the following dependencies for the Java/Scala API +(here presented in Maven syntax, but the same dependencies apply to other build tools (Gradle, SBT, etc.) as well. + +
    +
    +{% highlight xml %} + + org.apache.flink + flink-java + {{site.version }} + provided + + + org.apache.flink + flink-streaming-java{{ site.scala_version_suffix }} + {{site.version }} + provided + +{% endhighlight %} +
    +
    +{% highlight xml %} + + org.apache.flink + flink-scala{{ site.scala_version_suffix }} + {{site.version }} + provided + + + org.apache.flink + flink-streaming-scala{{ site.scala_version_suffix }} + {{site.version }} + provided + +{% endhighlight %} +
    +
    + +**Important:** Please note that all these dependencies have their scope set to *provided*. +That means that they are needed to compile against, but that they should not be packaged into the +project's resulting application jar file - these dependencies are Flink Core Dependencies, +which are already available in any setup. + +It is highly recommended to keep the dependencies in scope *provided*. If they are not set to *provided*, +the best case is that the resulting JAR becomes excessively large, because it also contains all Flink core +dependencies. The worst case is that the Flink core dependencies that are added to the application's jar file +clash with some of your own dependency versions (which is normally avoided through inverted classloading). + +**Note on IntelliJ:** To make the applications run within IntelliJ IDEA, the Flink dependencies need +to be declared in scope *compile* rather than *provided*. Otherwise IntelliJ will not add them to the classpath and +the in-IDE execution will fail with a `NoClassDefFountError`. To avoid having to declare the +dependency scope as *compile* (which is not recommended, see above), the above linked Java- and Scala +project templates use a trick: They add a profile that selectively activates when the application +is run in IntelliJ and only then promotes the dependencies to scope *compile*, without affecting +the packaging of the JAR files. + + +## Adding Connector and Library Dependencies + +Most applications need specific connectors or libraries to run, for example a connector to Kafka, Cassandra, etc. +These connectors are not part of Flink's core dependencies and must hence be added as dependencies to the application + +Below is an example adding the connector for Kafka 0.10 as a dependency (Maven syntax): +{% highlight xml %} + + org.apache.flink + flink-connector-kafka-0.10{{ site.scala_version_suffix }} + {{site.version }} + +{% endhighlight %} + +We recommend to package the application code and all its required dependencies into one *jar-with-dependencies* which +we refer to as the *application jar*. The application jar can be submitted to an already running Flink cluster, +or added to a Flink application container image. + +Projects created from the [Java Project Template]({{ site.baseurl }}/quickstart/java_api_quickstart.html) or +[Scala Project Template]({{ site.baseurl }}/quickstart/scala_api_quickstart.html) are configured to automatically include +the application dependencies into the application jar when running `mvn clean package`. For projects that are +not set up from those templates, we recommend to add the Maven Shade Plugin (as listed in the Appendix below) +to build the application jar with all required dependencies. + +**Important:** For Maven (and other build tools) to correctly package the dependencies into the application jar, +these application dependencies must be specified in scope *compile* (unlike the core dependencies, which +must be specified in scope *provided*). + + +## Scala Versions + +Scala versions (2.10, 2.11, 2.12, etc.) are not binary compatible with one another. +For that reason, Flink for Scala 2.11 cannot be used with an application that uses +Scala 2.12. + +All Flink dependencies that (transitively) depend on Scala are suffixed with the +Scala version that they are built for, for example `flink-streaming-scala_2.11`. + +Developers that only use Java can pick any Scala version, Scala developers need to +pick the Scala version that matches their application's Scala version. + +Please refer to the [build guide]({{ site.baseurl }}/start/building.html#scala-versions) +for details on how to build Flink for a specific Scala version. + +**Note:** Because of major breaking changes in Scala 2.12, Flink 1.5 currently builds only for Scala 2.11. +We aim to add support for Scala 2.12 in the next versions. + + +## Hadoop Dependencies + +**General rule: It should never be necessary to add Hadoop dependencies directly to your application.** +*(The only exception being when using existing Hadoop input-/output formats with Flink's Hadoop compatibility wrappers)* + +If you want to use Flink with Hadoop, you need to have a Flink setup that includes the Hadoop dependencies, rather than +adding Hadoop as an application dependency. Please refer to the [Hadoop Setup Guide]({{ site.baseurl }}/ops/deployment/hadoop.html) +for details. + +There are two main reasons for that design: + + - Some Hadoop interaction happens in Flink's core, possibly before the user application is started, for example + setting up HDFS for checkpoints, authenticating via Hadoop's Kerberos tokens, or deployment on YARN. + + - Flink's inverted classloading approach hides many transitive dependencies from the core dependencies. That applies not only + to Flink's own core dependencies, but also to Hadoop's dependencies when present in the setup. + That way, applications can use different versions of the same dependencies without running into dependency conflicts (and + trust us, that's a big deal, because Hadoops dependency tree is huge.) + +If you need Hadoop dependencies during testing or development inside the IDE (for example for HDFS access), please configure +these dependencies similar to the scope of the dependencies to *test* or to *provided*. + + +## Appendix: Template for bulding a Jar with Dependencies + +To build an application JAR that contains all dependencies required for declared connectors and libraries, +you can use the following shade plugin definition: + +{% highlight xml %} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + my.prorgams.main.clazz + + + + + + + + +{% endhighlight %} + +{% top %} + From 71baa2784edab8b851dabe15855a023c73b4fc1c Mon Sep 17 00:00:00 2001 From: gyao Date: Tue, 27 Feb 2018 16:58:53 +0100 Subject: [PATCH 0047/2294] [FLINK-8787][flip6] Do not copy flinkConfiguration in AbstractYarnClusterDescriptor This closes #5591. --- .../org/apache/flink/yarn/AbstractYarnClusterDescriptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java index e6c36f6a2d29ad..6b930163896c00 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java @@ -482,7 +482,7 @@ protected ClusterClient deployInternal( flinkConfiguration.setString(ClusterEntrypoint.EXECUTION_MODE, executionMode.toString()); ApplicationReport report = startAppMaster( - new Configuration(flinkConfiguration), + flinkConfiguration, yarnClusterEntrypoint, jobGraph, yarnClient, From 970d94e821d2341d200baf692ee2fbbc85e395b5 Mon Sep 17 00:00:00 2001 From: gyao Date: Wed, 21 Feb 2018 16:02:01 +0100 Subject: [PATCH 0048/2294] [FLINK-8730][REST] JSON serialize entire SerializedThrowable Do not only serialize the serialized exception but the entire SerializedThrowable object. This makes it possible to throw the SerializedThrowable itself without deserializing it. This closes #5546. --- .../flink/util/SerializedThrowable.java | 14 ---- .../json/SerializedThrowableDeserializer.java | 15 ++-- .../json/SerializedThrowableSerializer.java | 7 +- .../SerializedThrowableSerializerTest.java | 71 +++++++++++++++++++ 4 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableSerializerTest.java diff --git a/flink-core/src/main/java/org/apache/flink/util/SerializedThrowable.java b/flink-core/src/main/java/org/apache/flink/util/SerializedThrowable.java index de6358c7ce5158..13f8d77d973a60 100644 --- a/flink-core/src/main/java/org/apache/flink/util/SerializedThrowable.java +++ b/flink-core/src/main/java/org/apache/flink/util/SerializedThrowable.java @@ -25,8 +25,6 @@ import java.util.HashSet; import java.util.Set; -import static java.util.Objects.requireNonNull; - /** * Utility class for dealing with user-defined Throwable types that are serialized (for * example during RPC/Actor communication), but cannot be resolved with the default @@ -64,18 +62,6 @@ public SerializedThrowable(Throwable exception) { this(exception, new HashSet<>()); } - /** - * Creates a new SerializedThrowable from a serialized exception provided as a byte array. - */ - public SerializedThrowable( - final byte[] serializedException, - final String originalErrorClassName, - final String fullStringifiedStackTrace) { - this.serializedException = requireNonNull(serializedException); - this.originalErrorClassName = requireNonNull(originalErrorClassName); - this.fullStringifiedStackTrace = requireNonNull(fullStringifiedStackTrace); - } - private SerializedThrowable(Throwable exception, Set alreadySeen) { super(getMessageOrError(exception)); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableDeserializer.java index 3217cce8dad08c..d0f71ce93d6141 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableDeserializer.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.rest.messages.json; +import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; @@ -27,9 +28,7 @@ import java.io.IOException; -import static org.apache.flink.runtime.rest.messages.json.SerializedThrowableSerializer.FIELD_NAME_CLASS; -import static org.apache.flink.runtime.rest.messages.json.SerializedThrowableSerializer.FIELD_NAME_SERIALIZED_EXCEPTION; -import static org.apache.flink.runtime.rest.messages.json.SerializedThrowableSerializer.FIELD_NAME_STACK_TRACE; +import static org.apache.flink.runtime.rest.messages.json.SerializedThrowableSerializer.FIELD_NAME_SERIALIZED_THROWABLE; /** * JSON deserializer for {@link SerializedThrowable}. @@ -48,10 +47,12 @@ public SerializedThrowable deserialize( final DeserializationContext ctxt) throws IOException { final JsonNode root = p.readValueAsTree(); - final String exceptionClassName = root.get(FIELD_NAME_CLASS).asText(); - final String stackTrace = root.get(FIELD_NAME_STACK_TRACE).asText(); - final byte[] serializedException = root.get(FIELD_NAME_SERIALIZED_EXCEPTION).binaryValue(); - return new SerializedThrowable(serializedException, exceptionClassName, stackTrace); + final byte[] serializedException = root.get(FIELD_NAME_SERIALIZED_THROWABLE).binaryValue(); + try { + return InstantiationUtil.deserializeObject(serializedException, ClassLoader.getSystemClassLoader()); + } catch (ClassNotFoundException e) { + throw new IOException("Failed to deserialize " + SerializedThrowable.class.getCanonicalName(), e); + } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableSerializer.java index cb921a9f5cb40a..51f111f362495c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedThrowableSerializer.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.rest.messages.json; +import org.apache.flink.util.InstantiationUtil; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; @@ -33,12 +34,12 @@ public class SerializedThrowableSerializer extends StdSerializer Date: Tue, 27 Feb 2018 14:43:52 +0800 Subject: [PATCH 0049/2294] [FLINK-8792] [rest] Change MessageQueryParameter.convertStringToValue to convertValueToString This closes #5587. --- .../handlers/AllowNonRestoredStateQueryParameter.java | 2 +- .../webmonitor/handlers/ParallelismQueryParameter.java | 2 +- .../runtime/webmonitor/handlers/StringQueryParameter.java | 2 +- .../handlers/AllowNonRestoredStateQueryParameterTest.java | 4 ++-- .../webmonitor/handlers/ParallelismQueryParameterTest.java | 2 +- .../flink/runtime/rest/messages/MessageQueryParameter.java | 6 +++--- .../rest/messages/RescalingParallelismQueryParameter.java | 2 +- .../rest/messages/TerminationModeQueryParameter.java | 2 +- .../rest/messages/job/metrics/MetricsFilterParameter.java | 2 +- .../apache/flink/runtime/rest/RestServerEndpointITCase.java | 2 +- .../runtime/rest/handler/util/HandlerRequestUtilsTest.java | 2 +- .../flink/runtime/rest/messages/MessageParametersTest.java | 2 +- .../messages/job/metrics/MetricsFilterParameterTest.java | 2 +- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java index 7ad014ea685f85..2ddde3ae2fefe6 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java @@ -39,7 +39,7 @@ public Boolean convertValueFromString(final String value) { } @Override - public String convertStringToValue(final Boolean value) { + public String convertValueToString(final Boolean value) { return value.toString(); } } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java index 26cb16c7b0e330..2ade7eb0a5cfb7 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java @@ -38,7 +38,7 @@ public Integer convertValueFromString(final String value) { } @Override - public String convertStringToValue(final Integer value) { + public String convertValueToString(final Integer value) { return value.toString(); } } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java index 226c5929c5693c..52c0967c7251ea 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java @@ -35,7 +35,7 @@ public final String convertValueFromString(final String value) { } @Override - public final String convertStringToValue(final String value) { + public final String convertValueToString(final String value) { return value; } diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java index 9882637e521375..97b61f635c89eb 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java @@ -34,8 +34,8 @@ public class AllowNonRestoredStateQueryParameterTest extends TestLogger { @Test public void testConvertStringToValue() { - assertEquals("false", allowNonRestoredStateQueryParameter.convertStringToValue(false)); - assertEquals("true", allowNonRestoredStateQueryParameter.convertStringToValue(true)); + assertEquals("false", allowNonRestoredStateQueryParameter.convertValueToString(false)); + assertEquals("true", allowNonRestoredStateQueryParameter.convertValueToString(true)); } @Test diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java index 8189dd5bb93d68..684af219811d71 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java @@ -33,7 +33,7 @@ public class ParallelismQueryParameterTest extends TestLogger { @Test public void testConvertStringToValue() { - assertEquals("42", parallelismQueryParameter.convertStringToValue(42)); + assertEquals("42", parallelismQueryParameter.convertValueToString(42)); } @Test diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java index 506a14b8da5052..29bee66d0cae2f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java @@ -58,11 +58,11 @@ public String convertToString(List values) { boolean first = true; for (X value : values) { if (first) { - sb.append(convertStringToValue(value)); + sb.append(convertValueToString(value)); first = false; } else { sb.append(","); - sb.append(convertStringToValue(value)); + sb.append(convertValueToString(value)); } } return sb.toString(); @@ -74,5 +74,5 @@ public String convertToString(List values) { * @param value parameter value * @return string representation of typed value */ - public abstract String convertStringToValue(X value); + public abstract String convertValueToString(X value); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java index 9230d790a7b9eb..be9eff1eec501f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java @@ -35,7 +35,7 @@ public Integer convertValueFromString(String value) { } @Override - public String convertStringToValue(Integer value) { + public String convertValueToString(Integer value) { return value.toString(); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java index 9873f81ecd41ec..386f22e225ce1f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java @@ -37,7 +37,7 @@ public TerminationMode convertValueFromString(String value) { } @Override - public String convertStringToValue(TerminationMode value) { + public String convertValueToString(TerminationMode value) { return value.name().toLowerCase(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java index b01d2a932dcaab..bcace79801f9bc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java @@ -41,7 +41,7 @@ public String convertValueFromString(String value) { } @Override - public String convertStringToValue(String value) { + public String convertValueToString(String value) { return value; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java index 3ad7ee50ecfae8..e049a2d1e87ea6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java @@ -481,7 +481,7 @@ public JobID convertValueFromString(String value) { } @Override - public String convertStringToValue(JobID value) { + public String convertValueToString(JobID value) { return value.toString(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java index 09433f53021cae..259ae2040871fe 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java @@ -108,7 +108,7 @@ public Boolean convertValueFromString(final String value) { } @Override - public String convertStringToValue(final Boolean value) { + public String convertValueToString(final Boolean value) { return value.toString(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java index 65a1baa7235410..03fcb0aa99b92f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java @@ -109,7 +109,7 @@ public JobID convertValueFromString(String value) { } @Override - public String convertStringToValue(JobID value) { + public String convertValueToString(JobID value) { return value.toString(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java index 7e1812f2e11ccd..10dc921c3e76bd 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java @@ -49,7 +49,7 @@ public void testIsOptionalParameter() { @Test public void testConversions() { - assertThat(metricsFilterParameter.convertStringToValue("test"), equalTo("test")); + assertThat(metricsFilterParameter.convertValueToString("test"), equalTo("test")); assertThat(metricsFilterParameter.convertValueFromString("test"), equalTo("test")); } From e8d16850948231070dbc3344c2b287d07e2647a7 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 28 Feb 2018 10:11:44 +0100 Subject: [PATCH 0050/2294] [FLINK-8792] [rest] Change MessageQueryParameter#convertValueFromString to convertStringToValue --- .../handlers/AllowNonRestoredStateQueryParameter.java | 2 +- .../webmonitor/handlers/ParallelismQueryParameter.java | 2 +- .../runtime/webmonitor/handlers/StringQueryParameter.java | 2 +- .../handlers/AllowNonRestoredStateQueryParameterTest.java | 6 +++--- .../webmonitor/handlers/ParallelismQueryParameterTest.java | 2 +- .../flink/runtime/rest/messages/MessageQueryParameter.java | 4 ++-- .../rest/messages/RescalingParallelismQueryParameter.java | 2 +- .../rest/messages/TerminationModeQueryParameter.java | 2 +- .../rest/messages/job/metrics/MetricsFilterParameter.java | 2 +- .../apache/flink/runtime/rest/RestServerEndpointITCase.java | 2 +- .../runtime/rest/handler/util/HandlerRequestUtilsTest.java | 2 +- .../flink/runtime/rest/messages/MessageParametersTest.java | 2 +- .../messages/job/metrics/MetricsFilterParameterTest.java | 2 +- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java index 2ddde3ae2fefe6..19734d862b1211 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameter.java @@ -34,7 +34,7 @@ protected AllowNonRestoredStateQueryParameter() { } @Override - public Boolean convertValueFromString(final String value) { + public Boolean convertStringToValue(final String value) { return Boolean.valueOf(value); } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java index 2ade7eb0a5cfb7..398bcb06477925 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameter.java @@ -33,7 +33,7 @@ public ParallelismQueryParameter() { } @Override - public Integer convertValueFromString(final String value) { + public Integer convertStringToValue(final String value) { return Integer.valueOf(value); } diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java index 52c0967c7251ea..67e83ff8fe076e 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/StringQueryParameter.java @@ -30,7 +30,7 @@ public StringQueryParameter(final String key, final MessageParameterRequisitenes } @Override - public final String convertValueFromString(final String value) { + public final String convertStringToValue(final String value) { return value; } diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java index 97b61f635c89eb..8bc1327d348b16 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/AllowNonRestoredStateQueryParameterTest.java @@ -40,9 +40,9 @@ public void testConvertStringToValue() { @Test public void testConvertValueFromString() { - assertEquals(false, allowNonRestoredStateQueryParameter.convertValueFromString("false")); - assertEquals(true, allowNonRestoredStateQueryParameter.convertValueFromString("true")); - assertEquals(true, allowNonRestoredStateQueryParameter.convertValueFromString("TRUE")); + assertEquals(false, allowNonRestoredStateQueryParameter.convertStringToValue("false")); + assertEquals(true, allowNonRestoredStateQueryParameter.convertStringToValue("true")); + assertEquals(true, allowNonRestoredStateQueryParameter.convertStringToValue("TRUE")); } } diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java index 684af219811d71..cd9080bb9b2f01 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/ParallelismQueryParameterTest.java @@ -38,7 +38,7 @@ public void testConvertStringToValue() { @Test public void testConvertValueFromString() { - assertEquals(42, (int) parallelismQueryParameter.convertValueFromString("42")); + assertEquals(42, (int) parallelismQueryParameter.convertStringToValue("42")); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java index 29bee66d0cae2f..180f0119d98c9a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/MessageQueryParameter.java @@ -39,7 +39,7 @@ public List convertFromString(String values) { String[] splitValues = values.split(","); List list = new ArrayList<>(); for (String value : splitValues) { - list.add(convertValueFromString(value)); + list.add(convertStringToValue(value)); } return list; } @@ -50,7 +50,7 @@ public List convertFromString(String values) { * @param value string representation of parameter value * @return parameter value */ - public abstract X convertValueFromString(String value); + public abstract X convertStringToValue(String value); @Override public String convertToString(List values) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java index be9eff1eec501f..5c4f912a2d786d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RescalingParallelismQueryParameter.java @@ -30,7 +30,7 @@ public RescalingParallelismQueryParameter() { } @Override - public Integer convertValueFromString(String value) { + public Integer convertStringToValue(String value) { return Integer.valueOf(value); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java index 386f22e225ce1f..889b6d5621ec38 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/TerminationModeQueryParameter.java @@ -32,7 +32,7 @@ public TerminationModeQueryParameter() { } @Override - public TerminationMode convertValueFromString(String value) { + public TerminationMode convertStringToValue(String value) { return TerminationMode.valueOf(value.toUpperCase()); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java index bcace79801f9bc..9c6c0fd66b41a0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameter.java @@ -36,7 +36,7 @@ public MetricsFilterParameter() { } @Override - public String convertValueFromString(String value) { + public String convertStringToValue(String value) { return value; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java index e049a2d1e87ea6..c9817ff19e437c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java @@ -476,7 +476,7 @@ static class JobIDQueryParameter extends MessageQueryParameter { } @Override - public JobID convertValueFromString(String value) { + public JobID convertStringToValue(String value) { return JobID.fromHexString(value); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java index 259ae2040871fe..5001e3daf6a312 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/util/HandlerRequestUtilsTest.java @@ -103,7 +103,7 @@ private TestBooleanQueryParameter() { } @Override - public Boolean convertValueFromString(final String value) { + public Boolean convertStringToValue(final String value) { return Boolean.parseBoolean(value); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java index 03fcb0aa99b92f..8d73231350a092 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/MessageParametersTest.java @@ -104,7 +104,7 @@ private static class TestQueryParameter extends MessageQueryParameter { } @Override - public JobID convertValueFromString(String value) { + public JobID convertStringToValue(String value) { return JobID.fromHexString(value); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java index 10dc921c3e76bd..b13cb01f647005 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/metrics/MetricsFilterParameterTest.java @@ -50,7 +50,7 @@ public void testIsOptionalParameter() { @Test public void testConversions() { assertThat(metricsFilterParameter.convertValueToString("test"), equalTo("test")); - assertThat(metricsFilterParameter.convertValueFromString("test"), equalTo("test")); + assertThat(metricsFilterParameter.convertStringToValue("test"), equalTo("test")); } } From 6c837d738f90ccf140ad2288cd24f3706babd63c Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Thu, 22 Feb 2018 17:22:54 +0100 Subject: [PATCH 0051/2294] [FLINK-8451] [serializers] Make Scala tuple serializer deserialization more failure tolerant This closes #5567. --- .../TupleSerializerConfigSnapshot.java | 2 +- .../apache/flink/util/InstantiationUtil.java | 56 +++++++++-- .../flink-1.3.2-scala-types-serializer-data | Bin 0 -> 97 bytes ...link-1.3.2-scala-types-serializer-snapshot | Bin 0 -> 7634 bytes .../TupleSerializerCompatibilityTest.scala | 86 ++++++++++++++++ ...SerializerCompatibilityTestGenerator.scala | 94 ++++++++++++++++++ 6 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 flink-scala/src/test/resources/flink-1.3.2-scala-types-serializer-data create mode 100644 flink-scala/src/test/resources/flink-1.3.2-scala-types-serializer-snapshot create mode 100644 flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerCompatibilityTest.scala create mode 100644 flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerCompatibilityTestGenerator.scala diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java index 705099e9b2dbca..eac5200da9c6f7 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerConfigSnapshot.java @@ -61,7 +61,7 @@ public void read(DataInputView in) throws IOException { super.read(in); try (final DataInputViewStream inViewWrapper = new DataInputViewStream(in)) { - tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader()); + tupleClass = InstantiationUtil.deserializeObject(inViewWrapper, getUserCodeClassLoader(), true); } catch (ClassNotFoundException e) { throw new IOException("Could not find requested tuple class in classpath.", e); } diff --git a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java index 11e3990077a595..978d270e05f97a 100644 --- a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java +++ b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java @@ -113,7 +113,7 @@ protected Class resolveClass(ObjectStreamClass desc) throws IOException, Clas * *

    This can be removed once 1.2 is no longer supported. */ - private static Set scalaSerializerClassnames = new HashSet<>(); + private static final Set scalaSerializerClassnames = new HashSet<>(); static { scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TraversableSerializer"); scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.CaseClassSerializer"); @@ -121,10 +121,53 @@ protected Class resolveClass(ObjectStreamClass desc) throws IOException, Clas scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EnumValueSerializer"); scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.OptionSerializer"); scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TrySerializer"); - scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer"); scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.UnitSerializer"); } + /** + * The serialVersionUID might change between Scala versions and since those classes are + * part of the tuple serializer config snapshots we need to ignore them. + * + * @see FLINK-8451 + */ + private static final Set scalaTypes = new HashSet<>(); + static { + scalaTypes.add("scala.Tuple1"); + scalaTypes.add("scala.Tuple2"); + scalaTypes.add("scala.Tuple3"); + scalaTypes.add("scala.Tuple4"); + scalaTypes.add("scala.Tuple5"); + scalaTypes.add("scala.Tuple6"); + scalaTypes.add("scala.Tuple7"); + scalaTypes.add("scala.Tuple8"); + scalaTypes.add("scala.Tuple9"); + scalaTypes.add("scala.Tuple10"); + scalaTypes.add("scala.Tuple11"); + scalaTypes.add("scala.Tuple12"); + scalaTypes.add("scala.Tuple13"); + scalaTypes.add("scala.Tuple14"); + scalaTypes.add("scala.Tuple15"); + scalaTypes.add("scala.Tuple16"); + scalaTypes.add("scala.Tuple17"); + scalaTypes.add("scala.Tuple18"); + scalaTypes.add("scala.Tuple19"); + scalaTypes.add("scala.Tuple20"); + scalaTypes.add("scala.Tuple21"); + scalaTypes.add("scala.Tuple22"); + scalaTypes.add("scala.Tuple1$mcJ$sp"); + scalaTypes.add("scala.Tuple1$mcI$sp"); + scalaTypes.add("scala.Tuple1$mcD$sp"); + scalaTypes.add("scala.Tuple2$mcJJ$sp"); + scalaTypes.add("scala.Tuple2$mcJI$sp"); + scalaTypes.add("scala.Tuple2$mcJD$sp"); + scalaTypes.add("scala.Tuple2$mcIJ$sp"); + scalaTypes.add("scala.Tuple2$mcII$sp"); + scalaTypes.add("scala.Tuple2$mcID$sp"); + scalaTypes.add("scala.Tuple2$mcDJ$sp"); + scalaTypes.add("scala.Tuple2$mcDI$sp"); + scalaTypes.add("scala.Tuple2$mcDD$sp"); + } + /** * An {@link ObjectInputStream} that ignores serialVersionUID mismatches when deserializing objects of * anonymous classes or our Scala serializer classes and also replaces occurences of GenericData.Array @@ -158,12 +201,13 @@ protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFo } } - Class localClass = resolveClass(streamClassDescriptor); - if (scalaSerializerClassnames.contains(localClass.getName()) || localClass.isAnonymousClass() + final Class localClass = resolveClass(streamClassDescriptor); + final String name = localClass.getName(); + if (scalaSerializerClassnames.contains(name) || scalaTypes.contains(name) || localClass.isAnonymousClass() // isAnonymousClass does not work for anonymous Scala classes; additionally check by classname - || localClass.getName().contains("$anon$") || localClass.getName().contains("$anonfun")) { + || name.contains("$anon$") || name.contains("$anonfun")) { - ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass); + final ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass); if (localClassDescriptor != null && localClassDescriptor.getSerialVersionUID() != streamClassDescriptor.getSerialVersionUID()) { LOG.warn("Ignoring serialVersionUID mismatch for anonymous class {}; was {}, now {}.", diff --git a/flink-scala/src/test/resources/flink-1.3.2-scala-types-serializer-data b/flink-scala/src/test/resources/flink-1.3.2-scala-types-serializer-data new file mode 100644 index 0000000000000000000000000000000000000000..ddd6ac01112405f127b5c3f3d9412bf060009d3f GIT binary patch literal 97 zcmZQzV9;V@^GMCf$!B0I&qyq>chE=!0|o{L9w^oV(ioB{K*1A0oC4L(2vwm2(f%I@ H8i3LO{3sa1 literal 0 HcmV?d00001 diff --git a/flink-scala/src/test/resources/flink-1.3.2-scala-types-serializer-snapshot b/flink-scala/src/test/resources/flink-1.3.2-scala-types-serializer-snapshot new file mode 100644 index 0000000000000000000000000000000000000000..1ebdabb29fc94d31123061db091237561394f4f2 GIT binary patch literal 7634 zcmeHMO>7%Q6n?v@LzR-Gv=BiB5>!6)(p|-A%8!vM#r=_sC6FA1+5*+Yo!FDEcg^g$ z#1%i{0yo5=2jGejrQ*f`Rq6#eASzB=xKyf&%AZ8V1#tn9zM0*4H_60}z3a*dTUy(* z*1mb~dw<_N1AqbQHTosMBVXZHT#sJ<>=tBv4ljDVYLN!1)Tp)SF!wAjnB`Z9LoD8O z1*=nQzS(eSnQ}%PcAoNrS8ou(7MR0CYo7Yz7^P3_m!lMwnN;it#0KJlLJ~~0?9FR)=G{nfaL}*GAk=9|2BE3=tgheL0taH zw;ulF^3>n&OMI0)j0?T9Hb`1PK&_%HBD3SlahThheM-z)Bg@RKI#hUW>(h5GePg|P zL~8d6yrH6YRx37$+O|KeBTUpNk5qf}PlJA#eS6jfvjh_kU5u$M;9##_p5&}ri;0p- zj<;3KYgN?)Sr#0vjueh#DW#5~V`jAS)|>vk%VHWm^Qe`*c5&v%RS2b=Gack?l0Aj3 zCjM+#0H%Nb=lCxz^TgMy8_P;$s%Vww+#^@0%F|j^^bpN?o>^%L=f zA>z5EeU+)!d*8m>0dZ;Jj`rIggzBW$0lhW>*%I1Se4qKn$A=z#DsMuj2nIV0rkE~) zK?UqCsSRvd_U#~5r3XY#z(n{{;5)p#P2?DpD(_VV%7CI)YS|?Xzvc<_ zl#I$u=&|2$iT6gz*dspz7(*P#F9J+l0GQkhuqy}fz}4`X8(|L}%vzaJb1~qU7V8b^ z`5fV-P6g!-_5HQAJV0EbmIN#Glkr1{`M`GoBbDynWf|(!K>E^*W%*8{BgynL;J7x@ zHp%Ms-Cdj3n7-=uJBmM#-R5PyZmPWiZib=MO`X5101svcCxlBD+iM~6Nh2rqkRD9L zdx0s4@sSror2EM|v}|v)6U2I=+9nDSqZ=Og)^<--z_5NvN+*wJ0fA2MxR)vgPdz0i z>yv|ugk?v9s1TI$#*!TmCi_J~SG(0M)b%xVbPcb+q~dt}6lQX2ho#g#HJHUSDP9u$ zAybf(uGlS?K{hkV Date: Mon, 26 Feb 2018 10:54:53 +0800 Subject: [PATCH 0052/2294] [FLINK-8777][checkpointing] Cleanup local state more eagerly in recovery This closes #5578. --- .../runtime/state/TaskLocalStateStore.java | 8 ++ .../state/TaskLocalStateStoreImpl.java | 108 +++++++++++++----- .../runtime/state/TaskStateManagerImpl.java | 6 +- .../state/TaskLocalStateStoreImplTest.java | 42 +++++-- .../state/TestTaskLocalStateStore.java | 6 + 5 files changed, 130 insertions(+), 40 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java index 7089894c1c5afb..686f4f6936f0c1 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStore.java @@ -24,6 +24,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.function.LongPredicate; + /** * Classes that implement this interface serve as a task-manager-level local storage for local checkpointed state. * The purpose is to provide access to a state that is stored locally for a faster recovery compared to the state that @@ -62,4 +64,10 @@ void storeLocalState( * and removes all local states with a checkpoint id that is smaller than the newly confirmed checkpoint id. */ void confirmCheckpoint(long confirmedCheckpointId); + + /** + * Remove all checkpoints from the store that match the given predicate. + * @param matcher the predicate that selects the checkpoints for pruning. + */ + void pruneMatchingCheckpoints(LongPredicate matcher); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java index bb4f0116dffe5b..29adc4ada399e9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.state; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.JobID; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; @@ -35,9 +36,10 @@ import java.io.File; import java.io.IOException; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -46,6 +48,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; +import java.util.function.LongPredicate; /** * Main implementation of a {@link TaskLocalStateStore}. @@ -56,7 +59,8 @@ public class TaskLocalStateStoreImpl implements TaskLocalStateStore { private static final Logger LOG = LoggerFactory.getLogger(TaskLocalStateStoreImpl.class); /** Dummy value to use instead of null to satisfy {@link ConcurrentHashMap}. */ - private static final TaskStateSnapshot NULL_DUMMY = new TaskStateSnapshot(0); + @VisibleForTesting + static final TaskStateSnapshot NULL_DUMMY = new TaskStateSnapshot(0); /** JobID from the owning subtask. */ @Nonnull @@ -103,14 +107,36 @@ public TaskLocalStateStoreImpl( @Nonnull LocalRecoveryConfig localRecoveryConfig, @Nonnull Executor discardExecutor) { - this.lock = new Object(); - this.storedTaskStateByCheckpointID = new TreeMap<>(); + this( + jobID, + allocationID, + jobVertexID, + subtaskIndex, + localRecoveryConfig, + discardExecutor, + new TreeMap<>(), + new Object()); + } + + @VisibleForTesting + TaskLocalStateStoreImpl( + @Nonnull JobID jobID, + @Nonnull AllocationID allocationID, + @Nonnull JobVertexID jobVertexID, + @Nonnegative int subtaskIndex, + @Nonnull LocalRecoveryConfig localRecoveryConfig, + @Nonnull Executor discardExecutor, + @Nonnull SortedMap storedTaskStateByCheckpointID, + @Nonnull Object lock) { + this.jobID = jobID; this.allocationID = allocationID; this.jobVertexID = jobVertexID; this.subtaskIndex = subtaskIndex; this.discardExecutor = discardExecutor; this.localRecoveryConfig = localRecoveryConfig; + this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID; + this.lock = lock; this.disposed = false; } @@ -133,23 +159,25 @@ public void storeLocalState( checkpointId, jobID, jobVertexID, subtaskIndex); } - Map toDiscard = new HashMap<>(16); + Map.Entry toDiscard = null; synchronized (lock) { if (disposed) { // we ignore late stores and simply discard the state. - toDiscard.put(checkpointId, localState); + toDiscard = new AbstractMap.SimpleEntry<>(checkpointId, localState); } else { TaskStateSnapshot previous = storedTaskStateByCheckpointID.put(checkpointId, localState); if (previous != null) { - toDiscard.put(checkpointId, previous); + toDiscard = new AbstractMap.SimpleEntry<>(checkpointId, previous); } } } - asyncDiscardLocalStateForCollection(toDiscard.entrySet()); + if (toDiscard != null) { + asyncDiscardLocalStateForCollection(Collections.singletonList(toDiscard)); + } } @Override @@ -157,10 +185,13 @@ public void storeLocalState( public TaskStateSnapshot retrieveLocalState(long checkpointID) { TaskStateSnapshot snapshot; + synchronized (lock) { snapshot = storedTaskStateByCheckpointID.get(checkpointID); } + snapshot = (snapshot != NULL_DUMMY) ? snapshot : null; + if (LOG.isTraceEnabled()) { LOG.trace("Found entry for local state for checkpoint {} in subtask ({} - {} - {}) : {}", checkpointID, jobID, jobVertexID, subtaskIndex, snapshot); @@ -169,7 +200,7 @@ public TaskStateSnapshot retrieveLocalState(long checkpointID) { checkpointID, jobID, jobVertexID, subtaskIndex); } - return snapshot != NULL_DUMMY ? snapshot : null; + return snapshot; } @Override @@ -184,30 +215,18 @@ public void confirmCheckpoint(long confirmedCheckpointId) { LOG.debug("Received confirmation for checkpoint {} in subtask ({} - {} - {}). Starting to prune history.", confirmedCheckpointId, jobID, jobVertexID, subtaskIndex); - final List> toRemove = new ArrayList<>(); - - synchronized (lock) { - - Iterator> entryIterator = - storedTaskStateByCheckpointID.entrySet().iterator(); + pruneCheckpoints( + (snapshotCheckpointId) -> snapshotCheckpointId < confirmedCheckpointId, + true); - // remove entries for outdated checkpoints and discard their state. - while (entryIterator.hasNext()) { - - Map.Entry snapshotEntry = entryIterator.next(); - long entryCheckpointId = snapshotEntry.getKey(); + } - if (entryCheckpointId < confirmedCheckpointId) { - toRemove.add(snapshotEntry); - entryIterator.remove(); - } else { - // we can stop because the map is sorted. - break; - } - } - } + @Override + public void pruneMatchingCheckpoints(@Nonnull LongPredicate matcher) { - asyncDiscardLocalStateForCollection(toRemove); + pruneCheckpoints( + matcher, + false); } /** @@ -300,6 +319,35 @@ private void deleteDirectory(File directory) throws IOException { } } + /** + * Pruning the useless checkpoints, it should be called only when holding the {@link #lock}. + */ + private void pruneCheckpoints(LongPredicate pruningChecker, boolean breakOnceCheckerFalse) { + + final List> toRemove = new ArrayList<>(); + + synchronized (lock) { + + Iterator> entryIterator = + storedTaskStateByCheckpointID.entrySet().iterator(); + + while (entryIterator.hasNext()) { + + Map.Entry snapshotEntry = entryIterator.next(); + long entryCheckpointId = snapshotEntry.getKey(); + + if (pruningChecker.test(entryCheckpointId)) { + toRemove.add(snapshotEntry); + entryIterator.remove(); + } else if (breakOnceCheckerFalse) { + break; + } + } + } + + asyncDiscardLocalStateForCollection(toRemove); + } + @Override public String toString() { return "TaskLocalStateStore{" + diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java index e057d1957440c1..e542ba13fda3ad 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java @@ -119,8 +119,12 @@ public PrioritizedOperatorSubtaskState prioritizedOperatorState(OperatorID opera return PrioritizedOperatorSubtaskState.emptyNotRestored(); } + long restoreCheckpointId = jobManagerTaskRestore.getRestoreCheckpointId(); + TaskStateSnapshot localStateSnapshot = - localStateStore.retrieveLocalState(jobManagerTaskRestore.getRestoreCheckpointId()); + localStateStore.retrieveLocalState(restoreCheckpointId); + + localStateStore.pruneMatchingCheckpoints((long checkpointId) -> checkpointId != restoreCheckpointId); List alternativesByPriority = Collections.emptyList(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java index 16416762a525b5..618320ebc4f395 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskLocalStateStoreImplTest.java @@ -36,28 +36,30 @@ import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; import static org.powermock.api.mockito.PowerMockito.spy; public class TaskLocalStateStoreImplTest { + private SortedMap internalSnapshotMap; + private Object internalLock; private TemporaryFolder temporaryFolder; - private JobID jobID; - private AllocationID allocationID; - private JobVertexID jobVertexID; - private int subtaskIdx; private File[] allocationBaseDirs; private TaskLocalStateStoreImpl taskLocalStateStore; @Before public void before() throws Exception { + JobID jobID = new JobID(); + AllocationID allocationID = new AllocationID(); + JobVertexID jobVertexID = new JobVertexID(); + int subtaskIdx = 0; this.temporaryFolder = new TemporaryFolder(); this.temporaryFolder.create(); - this.jobID = new JobID(); - this.allocationID = new AllocationID(); - this.jobVertexID = new JobVertexID(); - this.subtaskIdx = 0; this.allocationBaseDirs = new File[]{temporaryFolder.newFolder(), temporaryFolder.newFolder()}; + this.internalSnapshotMap = new TreeMap<>(); + this.internalLock = new Object(); LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(allocationBaseDirs, jobID, jobVertexID, subtaskIdx); @@ -71,7 +73,9 @@ public void before() throws Exception { jobVertexID, subtaskIdx, localRecoveryConfig, - Executors.directExecutor()); + Executors.directExecutor(), + internalSnapshotMap, + internalLock); } @After @@ -116,6 +120,26 @@ public void storeAndRetrieve() throws Exception { Assert.assertNull(taskLocalStateStore.retrieveLocalState(chkCount + 1)); } + /** + * Test checkpoint pruning. + */ + @Test + public void pruneCheckpoints() throws Exception { + + final int chkCount = 3; + + List taskStateSnapshots = storeStates(chkCount); + + // test retrieve with pruning + taskLocalStateStore.pruneMatchingCheckpoints((long chk) -> chk != chkCount - 1); + + for (int i = 0; i < chkCount - 1; ++i) { + Assert.assertNull(taskLocalStateStore.retrieveLocalState(i)); + } + + checkStoredAsExpected(taskStateSnapshots, chkCount - 1, chkCount); + } + /** * Tests pruning of previous checkpoints if a new checkpoint is confirmed. */ diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java index 12c07ddd595425..2ade3e6d142629 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TestTaskLocalStateStore.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import java.util.function.LongPredicate; /** * Test implementation of a {@link TaskLocalStateStore}. @@ -103,6 +104,11 @@ public void confirmCheckpoint(long confirmedCheckpointId) { } } + @Override + public void pruneMatchingCheckpoints(LongPredicate matcher) { + taskStateSnapshotsByCheckpointID.keySet().removeIf(matcher::test); + } + public boolean isDisposed() { return disposed; } From f57793082f92b8052e4fdccc0aaf16f0c55777a8 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 28 Feb 2018 15:35:13 +0100 Subject: [PATCH 0053/2294] [hotfix] Enable FILESYTEM_DEFAULT_OVERRIDE in FLIP-6 MiniClusterResource --- .../java/org/apache/flink/test/util/MiniClusterResource.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java index 9bb1ae9ec7211b..1c5da62b2e0000 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.CoreOptions; import org.apache.flink.configuration.RestOptions; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.minicluster.JobExecutorService; @@ -144,6 +145,10 @@ private JobExecutorService startOldMiniCluster() throws Exception { private JobExecutorService startFlip6MiniCluster() throws Exception { final Configuration configuration = miniClusterResourceConfiguration.getConfiguration(); + // we need to set this since a lot of test expect this because TestBaseUtils.startCluster() + // enabled this by default + configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); + // set rest port to 0 to avoid clashes with concurrent MiniClusters configuration.setInteger(RestOptions.REST_PORT, 0); From af8efe92c340ad21284d775ce74b15b774b67bf7 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Wed, 28 Feb 2018 14:25:55 +0100 Subject: [PATCH 0054/2294] [FLINK-8557][checkpointing] Remove illegal characters from operator description text before using it to construct the instance directory in RocksDB This closes #5598. --- .../contrib/streaming/state/RocksDBStateBackend.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java index f60cb2cd82deca..93892952f88b60 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java @@ -396,10 +396,14 @@ public AbstractKeyedStateBackend createKeyedStateBackend( String tempDir = env.getTaskManagerInfo().getTmpDirectories()[0]; ensureRocksDBIsLoaded(tempDir); - lazyInitializeForJob(env, operatorIdentifier); + // replace all characters that are not legal for filenames with underscore + String fileCompatibleIdentifier = operatorIdentifier.replaceAll("[^a-zA-Z0-9\\-]", "_"); - File instanceBasePath = - new File(getNextStoragePath(), "job-" + jobId + "_op-" + operatorIdentifier + "_uuid-" + UUID.randomUUID()); + lazyInitializeForJob(env, fileCompatibleIdentifier); + + File instanceBasePath = new File( + getNextStoragePath(), + "job_" + jobId + "_op_" + fileCompatibleIdentifier + "_uuid_" + UUID.randomUUID()); LocalRecoveryConfig localRecoveryConfig = env.getTaskStateManager().createLocalRecoveryConfig(); From 67a547ad438c33d3fbcbe23cd03f009fdc8dd021 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Fri, 23 Feb 2018 11:37:37 +0100 Subject: [PATCH 0055/2294] [hotfix][tests] Deduplicate code in SingleInputGateTest --- .../consumer/SingleInputGateTest.java | 66 +++++++------------ 1 file changed, 25 insertions(+), 41 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java index 0dd08756032135..e94411dac3d155 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java @@ -77,14 +77,7 @@ public class SingleInputGateTest { @Test(timeout = 120 * 1000) public void testBasicGetNextLogic() throws Exception { // Setup - final SingleInputGate inputGate = new SingleInputGate( - "Test Task Name", new JobID(), - new IntermediateDataSetID(), ResultPartitionType.PIPELINED, - 0, 2, - mock(TaskActions.class), - UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup()); - - assertEquals(ResultPartitionType.PIPELINED, inputGate.getConsumedPartitionType()); + final SingleInputGate inputGate = createInputGate(); final TestInputChannel[] inputChannels = new TestInputChannel[]{ new TestInputChannel(inputGate, 0), @@ -135,14 +128,8 @@ public void testBackwardsEventWithUninitializedChannel() throws Exception { any(BufferAvailabilityListener.class))).thenReturn(iterator); // Setup reader with one local and one unknown input channel - final IntermediateDataSetID resultId = new IntermediateDataSetID(); - final SingleInputGate inputGate = new SingleInputGate( - "Test Task Name", new JobID(), - resultId, ResultPartitionType.PIPELINED, - 0, 2, - mock(TaskActions.class), - UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup()); + final SingleInputGate inputGate = createInputGate(); final BufferPool bufferPool = mock(BufferPool.class); when(bufferPool.getNumberOfRequiredMemorySegments()).thenReturn(2); @@ -190,14 +177,7 @@ public void testBackwardsEventWithUninitializedChannel() throws Exception { */ @Test public void testUpdateChannelBeforeRequest() throws Exception { - SingleInputGate inputGate = new SingleInputGate( - "t1", - new JobID(), - new IntermediateDataSetID(), - ResultPartitionType.PIPELINED, - 0, - 1, - mock(TaskActions.class), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup()); + SingleInputGate inputGate = createInputGate(1); ResultPartitionManager partitionManager = mock(ResultPartitionManager.class); @@ -230,15 +210,7 @@ public void testReleaseWhilePollingChannel() throws Exception { final AtomicReference asyncException = new AtomicReference<>(); // Setup the input gate with a single channel that does nothing - final SingleInputGate inputGate = new SingleInputGate( - "InputGate", - new JobID(), - new IntermediateDataSetID(), - ResultPartitionType.PIPELINED, - 0, - 1, - mock(TaskActions.class), - UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup()); + final SingleInputGate inputGate = createInputGate(1); InputChannel unknown = new UnknownInputChannel( inputGate, @@ -410,15 +382,7 @@ public void testRequestBuffersWithRemoteInputChannel() throws Exception { */ @Test public void testRequestBuffersWithUnknownInputChannel() throws Exception { - final SingleInputGate inputGate = new SingleInputGate( - "t1", - new JobID(), - new IntermediateDataSetID(), - ResultPartitionType.PIPELINED_BOUNDED, - 0, - 1, - mock(TaskActions.class), - UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup()); + final SingleInputGate inputGate = createInputGate(1); UnknownInputChannel unknown = mock(UnknownInputChannel.class); final ResultPartitionID resultPartitionId = new ResultPartitionID(); @@ -443,6 +407,26 @@ public void testRequestBuffersWithUnknownInputChannel() throws Exception { // --------------------------------------------------------------------------------------------- + private static SingleInputGate createInputGate() { + return createInputGate(2); + } + + private static SingleInputGate createInputGate(int numberOfInputChannels) { + SingleInputGate inputGate = new SingleInputGate( + "Test Task Name", + new JobID(), + new IntermediateDataSetID(), + ResultPartitionType.PIPELINED, + 0, + numberOfInputChannels, + mock(TaskActions.class), + UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup()); + + assertEquals(ResultPartitionType.PIPELINED, inputGate.getConsumedPartitionType()); + + return inputGate; + } + static void verifyBufferOrEvent( InputGate inputGate, boolean isBuffer, From 42f71f61c1ae683cd39887cb3d921c0d0cb67619 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Fri, 23 Feb 2018 12:11:14 +0100 Subject: [PATCH 0056/2294] [hotfix][runtime] Remove duplicated check --- .../io/network/partition/consumer/UnionInputGate.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java index 5a547ea53b246e..481599ce36ec42 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java @@ -177,13 +177,6 @@ public Optional getNextBufferOrEvent() throws IOException, Interr } } - if (bufferOrEvent.moreAvailable()) { - // this buffer or event was now removed from the non-empty gates queue - // we re-add it in case it has more data, because in that case no "non-empty" notification - // will come for that gate - queueInputGate(inputGate); - } - // Set the channel index to identify the input channel (across all unioned input gates) final int channelIndexOffset = inputGateToIndexOffsetMap.get(inputGate); From 6c9e26713502a5256520ca3c1619e1da952666d9 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Fri, 23 Feb 2018 11:20:21 +0100 Subject: [PATCH 0057/2294] [FLINK-8760][runtime] Correctly propagate moreAvailable flag through SingleInputGate Previously if we SingleInputGate was re-eqnqueuing an input channel, isMoreAvailable might incorrectly return false. This might caused some dead locks. --- .../partition/consumer/BufferOrEvent.java | 6 +- .../partition/consumer/SingleInputGate.java | 1 + .../partition/consumer/UnionInputGate.java | 9 ++- .../consumer/SingleInputGateTest.java | 62 +++++++++++++++---- .../partition/consumer/TestInputChannel.java | 14 ++++- .../consumer/UnionInputGateTest.java | 33 +++++----- 6 files changed, 92 insertions(+), 33 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferOrEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferOrEvent.java index 3e93ae6e6880c2..d1da4388c1b2be 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferOrEvent.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferOrEvent.java @@ -39,7 +39,7 @@ public class BufferOrEvent { * This is not needed outside of the input gate unioning logic and cannot * be set outside of the consumer package. */ - private final boolean moreAvailable; + private boolean moreAvailable; private int channelIndex; @@ -99,4 +99,8 @@ public String toString() { return String.format("BufferOrEvent [%s, channelIndex = %d]", isBuffer() ? buffer : event, channelIndex); } + + public void setMoreAvailable(boolean moreAvailable) { + this.moreAvailable = moreAvailable; + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java index a1f3cdcc5b4b7c..be4035c55b1735 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java @@ -540,6 +540,7 @@ private Optional getNextBufferOrEvent(boolean blocking) throws IO // will come for that channel if (result.get().moreAvailable()) { queueChannel(currentChannel); + moreAvailable = true; } final Buffer buffer = result.get().buffer(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java index 481599ce36ec42..393e08775141a6 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java @@ -181,6 +181,7 @@ public Optional getNextBufferOrEvent() throws IOException, Interr final int channelIndexOffset = inputGateToIndexOffsetMap.get(inputGate); bufferOrEvent.setChannelIndex(channelIndexOffset + bufferOrEvent.getChannelIndex()); + bufferOrEvent.setMoreAvailable(bufferOrEvent.moreAvailable() || inputGateWithData.moreInputGatesAvailable); return Optional.ofNullable(bufferOrEvent); } @@ -193,18 +194,20 @@ public Optional pollNextBufferOrEvent() throws IOException, Inter private InputGateWithData waitAndGetNextInputGate() throws IOException, InterruptedException { while (true) { InputGate inputGate; + boolean moreInputGatesAvailable; synchronized (inputGatesWithData) { while (inputGatesWithData.size() == 0) { inputGatesWithData.wait(); } inputGate = inputGatesWithData.remove(); enqueuedInputGatesWithData.remove(inputGate); + moreInputGatesAvailable = enqueuedInputGatesWithData.size() > 0; } // In case of inputGatesWithData being inaccurate do not block on an empty inputGate, but just poll the data. Optional bufferOrEvent = inputGate.pollNextBufferOrEvent(); if (bufferOrEvent.isPresent()) { - return new InputGateWithData(inputGate, bufferOrEvent.get()); + return new InputGateWithData(inputGate, bufferOrEvent.get(), moreInputGatesAvailable); } } } @@ -212,10 +215,12 @@ private InputGateWithData waitAndGetNextInputGate() throws IOException, Interrup private static class InputGateWithData { private final InputGate inputGate; private final BufferOrEvent bufferOrEvent; + private final boolean moreInputGatesAvailable; - public InputGateWithData(InputGate inputGate, BufferOrEvent bufferOrEvent) { + public InputGateWithData(InputGate inputGate, BufferOrEvent bufferOrEvent, boolean moreInputGatesAvailable) { this.inputGate = checkNotNull(inputGate); this.bufferOrEvent = checkNotNull(bufferOrEvent); + this.moreInputGatesAvailable = moreInputGatesAvailable; } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java index e94411dac3d155..8c54c1f6ad6a56 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java @@ -100,15 +100,40 @@ public void testBasicGetNextLogic() throws Exception { inputGate.notifyChannelNonEmpty(inputChannels[0].getInputChannel()); inputGate.notifyChannelNonEmpty(inputChannels[1].getInputChannel()); - verifyBufferOrEvent(inputGate, true, 0); - verifyBufferOrEvent(inputGate, true, 1); - verifyBufferOrEvent(inputGate, true, 0); - verifyBufferOrEvent(inputGate, false, 1); - verifyBufferOrEvent(inputGate, false, 0); + verifyBufferOrEvent(inputGate, true, 0, true); + verifyBufferOrEvent(inputGate, true, 1, true); + verifyBufferOrEvent(inputGate, true, 0, true); + verifyBufferOrEvent(inputGate, false, 1, true); + verifyBufferOrEvent(inputGate, false, 0, false); // Return null when the input gate has received all end-of-partition events assertTrue(inputGate.isFinished()); - assertFalse(inputGate.getNextBufferOrEvent().isPresent()); + } + + @Test(timeout = 120 * 1000) + public void testIsMoreAvailableReadingFromSingleInputChannel() throws Exception { + // Setup + final SingleInputGate inputGate = createInputGate(); + + final TestInputChannel[] inputChannels = new TestInputChannel[]{ + new TestInputChannel(inputGate, 0), + new TestInputChannel(inputGate, 1) + }; + + inputGate.setInputChannel( + new IntermediateResultPartitionID(), inputChannels[0].getInputChannel()); + + inputGate.setInputChannel( + new IntermediateResultPartitionID(), inputChannels[1].getInputChannel()); + + // Test + inputChannels[0].readBuffer(); + inputChannels[0].readBuffer(false); + + inputGate.notifyChannelNonEmpty(inputChannels[0].getInputChannel()); + + verifyBufferOrEvent(inputGate, true, 0, true); + verifyBufferOrEvent(inputGate, true, 0, false); } @Test @@ -428,13 +453,28 @@ private static SingleInputGate createInputGate(int numberOfInputChannels) { } static void verifyBufferOrEvent( - InputGate inputGate, - boolean isBuffer, - int channelIndex) throws IOException, InterruptedException { + InputGate inputGate, + boolean expectedIsBuffer, + int expectedChannelIndex, + boolean expectedMoreAvailable) throws IOException, InterruptedException { final Optional bufferOrEvent = inputGate.getNextBufferOrEvent(); assertTrue(bufferOrEvent.isPresent()); - assertEquals(isBuffer, bufferOrEvent.get().isBuffer()); - assertEquals(channelIndex, bufferOrEvent.get().getChannelIndex()); + assertEquals(expectedIsBuffer, bufferOrEvent.get().isBuffer()); + assertEquals(expectedChannelIndex, bufferOrEvent.get().getChannelIndex()); + assertEquals(expectedMoreAvailable, bufferOrEvent.get().moreAvailable()); + if (!expectedMoreAvailable) { + try { + assertFalse(inputGate.pollNextBufferOrEvent().isPresent()); + } + catch (UnsupportedOperationException ex) { + /** + * {@link UnionInputGate#pollNextBufferOrEvent()} is unsupported at the moment. + */ + if (!(inputGate instanceof UnionInputGate)) { + throw ex; + } + } + } } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java index f9060f3689320a..3ae3a8a2f69d6b 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java @@ -56,20 +56,28 @@ public TestInputChannel(SingleInputGate inputGate, int channelIndex) { } public TestInputChannel read(Buffer buffer) throws IOException, InterruptedException { + return read(buffer, true); + } + + public TestInputChannel read(Buffer buffer, boolean moreAvailable) throws IOException, InterruptedException { if (stubbing == null) { - stubbing = when(mock.getNextBuffer()).thenReturn(Optional.of(new BufferAndAvailability(buffer, true, 0))); + stubbing = when(mock.getNextBuffer()).thenReturn(Optional.of(new BufferAndAvailability(buffer, moreAvailable, 0))); } else { - stubbing = stubbing.thenReturn(Optional.of(new BufferAndAvailability(buffer, true, 0))); + stubbing = stubbing.thenReturn(Optional.of(new BufferAndAvailability(buffer, moreAvailable, 0))); } return this; } public TestInputChannel readBuffer() throws IOException, InterruptedException { + return readBuffer(true); + } + + public TestInputChannel readBuffer(boolean moreAvailable) throws IOException, InterruptedException { final Buffer buffer = mock(Buffer.class); when(buffer.isBuffer()).thenReturn(true); - return read(buffer); + return read(buffer, moreAvailable); } public TestInputChannel readEndOfPartitionEvent() throws IOException, InterruptedException { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java index 9b164710afd58c..912cd5b24347fb 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java @@ -26,6 +26,7 @@ import org.junit.Test; +import static org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateTest.verifyBufferOrEvent; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -93,22 +94,22 @@ testTaskName, new JobID(), ig2.notifyChannelNonEmpty(inputChannels[1][3].getInputChannel()); ig2.notifyChannelNonEmpty(inputChannels[1][4].getInputChannel()); - SingleInputGateTest.verifyBufferOrEvent(union, true, 0); // gate 1, channel 0 - SingleInputGateTest.verifyBufferOrEvent(union, true, 3); // gate 2, channel 0 - SingleInputGateTest.verifyBufferOrEvent(union, true, 1); // gate 1, channel 1 - SingleInputGateTest.verifyBufferOrEvent(union, true, 4); // gate 2, channel 1 - SingleInputGateTest.verifyBufferOrEvent(union, true, 2); // gate 1, channel 2 - SingleInputGateTest.verifyBufferOrEvent(union, true, 5); // gate 2, channel 1 - SingleInputGateTest.verifyBufferOrEvent(union, false, 0); // gate 1, channel 0 - SingleInputGateTest.verifyBufferOrEvent(union, true, 6); // gate 2, channel 1 - SingleInputGateTest.verifyBufferOrEvent(union, false, 1); // gate 1, channel 1 - SingleInputGateTest.verifyBufferOrEvent(union, true, 7); // gate 2, channel 1 - SingleInputGateTest.verifyBufferOrEvent(union, false, 2); // gate 1, channel 2 - SingleInputGateTest.verifyBufferOrEvent(union, false, 3); // gate 2, channel 0 - SingleInputGateTest.verifyBufferOrEvent(union, false, 4); // gate 2, channel 1 - SingleInputGateTest.verifyBufferOrEvent(union, false, 5); // gate 2, channel 2 - SingleInputGateTest.verifyBufferOrEvent(union, false, 6); // gate 2, channel 3 - SingleInputGateTest.verifyBufferOrEvent(union, false, 7); // gate 2, channel 4 + verifyBufferOrEvent(union, true, 0, true); // gate 1, channel 0 + verifyBufferOrEvent(union, true, 3, true); // gate 2, channel 0 + verifyBufferOrEvent(union, true, 1, true); // gate 1, channel 1 + verifyBufferOrEvent(union, true, 4, true); // gate 2, channel 1 + verifyBufferOrEvent(union, true, 2, true); // gate 1, channel 2 + verifyBufferOrEvent(union, true, 5, true); // gate 2, channel 1 + verifyBufferOrEvent(union, false, 0, true); // gate 1, channel 0 + verifyBufferOrEvent(union, true, 6, true); // gate 2, channel 1 + verifyBufferOrEvent(union, false, 1, true); // gate 1, channel 1 + verifyBufferOrEvent(union, true, 7, true); // gate 2, channel 1 + verifyBufferOrEvent(union, false, 2, true); // gate 1, channel 2 + verifyBufferOrEvent(union, false, 3, true); // gate 2, channel 0 + verifyBufferOrEvent(union, false, 4, true); // gate 2, channel 1 + verifyBufferOrEvent(union, false, 5, true); // gate 2, channel 2 + verifyBufferOrEvent(union, false, 6, true); // gate 2, channel 3 + verifyBufferOrEvent(union, false, 7, false); // gate 2, channel 4 // Return null when the input gate has received all end-of-partition events assertTrue(union.isFinished()); From 2c2e1896bba7805e6afa96cfd9040729c35c2742 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Fri, 23 Feb 2018 11:27:54 +0100 Subject: [PATCH 0058/2294] [hotfix][tests] Do not hide original exception in SuccessAfterNetworkBuffersFailureITCase --- ...ccessAfterNetworkBuffersFailureITCase.java | 40 +++++-------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java index dc19ad1d64712e..dbd0f7976d6b6b 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java @@ -65,38 +65,20 @@ private static Configuration getConfiguration() { } @Test - public void testSuccessfulProgramAfterFailure() { + public void testSuccessfulProgramAfterFailure() throws Exception { + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + + runConnectedComponents(env); + try { - ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - - try { - runConnectedComponents(env); - } - catch (Exception e) { - e.printStackTrace(); - fail("Program Execution should have succeeded."); - } - - try { - runKMeans(env); - fail("This program execution should have failed."); - } - catch (JobExecutionException e) { - assertTrue(e.getCause().getMessage().contains("Insufficient number of network buffers")); - } - - try { - runConnectedComponents(env); - } - catch (Exception e) { - e.printStackTrace(); - fail("Program Execution should have succeeded."); - } + runKMeans(env); + fail("This program execution should have failed."); } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); + catch (JobExecutionException e) { + assertTrue(e.getCause().getMessage().contains("Insufficient number of network buffers")); } + + runConnectedComponents(env); } private static void runConnectedComponents(ExecutionEnvironment env) throws Exception { From ebd39f3244a7c56f0d9a3cc4dba4d3f50efb36ad Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Fri, 23 Feb 2018 11:28:20 +0100 Subject: [PATCH 0059/2294] [FLINK-8694][runtime] Fix notifyDataAvailable race condition Before there was a race condition that might resulted in igonoring some notifyDataAvailable calls. This fixes the problem by moving buffersAvailable handling to Supartitions and adds stress test for flushAlways (without this fix this test is dead locking). --- ...reditBasedSequenceNumberingViewReader.java | 10 +--- .../netty/SequenceNumberingViewReader.java | 7 +-- .../partition/PipelinedSubpartition.java | 37 +++++++++++-- .../partition/PipelinedSubpartitionView.java | 5 ++ .../partition/ResultSubpartitionView.java | 2 + .../partition/SpillableSubpartition.java | 1 - .../partition/SpillableSubpartitionView.java | 28 +++++++--- .../partition/SpilledSubpartitionView.java | 8 +++ .../buffer/BufferBuilderTestUtils.java | 4 ++ .../netty/CancelPartitionRequestTest.java | 5 ++ .../netty/PartitionRequestQueueTest.java | 26 +++++++-- .../partition/PipelinedSubpartitionTest.java | 53 +++++++++++++++++++ .../partition/SpillableSubpartitionTest.java | 9 ++-- .../partition/SubpartitionTestBase.java | 5 ++ ...StreamNetworkThroughputBenchmarkTests.java | 8 +++ 15 files changed, 173 insertions(+), 35 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java index d02b2bf1ad5adc..9acbbacf2735ba 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java @@ -30,7 +30,6 @@ import org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel; import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; /** * Simple wrapper for the subpartition view used in the new network credit-based mode. @@ -44,8 +43,6 @@ class CreditBasedSequenceNumberingViewReader implements BufferAvailabilityListen private final InputChannelID receiverId; - private final AtomicBoolean buffersAvailable = new AtomicBoolean(); - private final PartitionRequestQueue requestQueue; private volatile ResultSubpartitionView subpartitionView; @@ -118,7 +115,7 @@ public boolean isRegisteredAsAvailable() { @Override public boolean isAvailable() { // BEWARE: this must be in sync with #isAvailable()! - return buffersAvailable.get() && + return hasBuffersAvailable() && (numCreditsAvailable > 0 || subpartitionView.nextBufferIsEvent()); } @@ -154,14 +151,13 @@ int getNumCreditsAvailable() { @VisibleForTesting boolean hasBuffersAvailable() { - return buffersAvailable.get(); + return subpartitionView.isAvailable(); } @Override public BufferAndAvailability getNextBuffer() throws IOException, InterruptedException { BufferAndBacklog next = subpartitionView.getNextBuffer(); if (next != null) { - buffersAvailable.set(next.isMoreAvailable()); sequenceNumber++; if (next.buffer().isBuffer() && --numCreditsAvailable < 0) { @@ -197,7 +193,6 @@ public void releaseAllResources() throws IOException { @Override public void notifyDataAvailable() { - buffersAvailable.set(true); requestQueue.notifyReaderNonEmpty(this); } @@ -206,7 +201,6 @@ public String toString() { return "CreditBasedSequenceNumberingViewReader{" + "requestLock=" + requestLock + ", receiverId=" + receiverId + - ", buffersAvailable=" + buffersAvailable.get() + ", sequenceNumber=" + sequenceNumber + ", numCreditsAvailable=" + numCreditsAvailable + ", isRegisteredAsAvailable=" + isRegisteredAsAvailable + diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java index 2d9635ceaeb65c..6a83af13837827 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java @@ -29,7 +29,6 @@ import org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel; import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; /** * Simple wrapper for the subpartition view used in the old network mode. @@ -43,8 +42,6 @@ class SequenceNumberingViewReader implements BufferAvailabilityListener, Network private final InputChannelID receiverId; - private final AtomicBoolean buffersAvailable = new AtomicBoolean(); - private final PartitionRequestQueue requestQueue; private volatile ResultSubpartitionView subpartitionView; @@ -96,7 +93,7 @@ public boolean isRegisteredAsAvailable() { @Override public boolean isAvailable() { - return buffersAvailable.get(); + return subpartitionView.isAvailable(); } @Override @@ -113,7 +110,6 @@ public int getSequenceNumber() { public BufferAndAvailability getNextBuffer() throws IOException, InterruptedException { BufferAndBacklog next = subpartitionView.getNextBuffer(); if (next != null) { - buffersAvailable.set(next.isMoreAvailable()); sequenceNumber++; return new BufferAndAvailability(next.buffer(), next.isMoreAvailable(), next.buffersInBacklog()); } else { @@ -143,7 +139,6 @@ public void releaseAllResources() throws IOException { @Override public void notifyDataAvailable() { - buffersAvailable.set(true); requestQueue.notifyReaderNonEmpty(this); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java index a9c6e57b96597d..cc7936350479f1 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; import java.io.IOException; @@ -48,6 +49,9 @@ class PipelinedSubpartition extends ResultSubpartition { /** Flag indicating whether the subpartition has been finished. */ private boolean isFinished; + @GuardedBy("buffers") + private boolean flushRequested; + /** Flag indicating whether the subpartition has been released. */ private volatile boolean isReleased; @@ -65,9 +69,11 @@ public boolean add(BufferConsumer bufferConsumer) { @Override public void flush() { synchronized (buffers) { - if (readView != null) { - readView.notifyDataAvailable(); + if (buffers.isEmpty()) { + return; } + flushRequested = !buffers.isEmpty(); + notifyDataAvailable(); } } @@ -93,7 +99,7 @@ private boolean add(BufferConsumer bufferConsumer, boolean finish) { if (finish) { isFinished = true; - notifyDataAvailable(); + flush(); } else { maybeNotifyDataAvailable(); @@ -138,17 +144,28 @@ BufferAndBacklog pollBuffer() { synchronized (buffers) { Buffer buffer = null; + if (buffers.isEmpty()) { + flushRequested = false; + } + while (!buffers.isEmpty()) { BufferConsumer bufferConsumer = buffers.peek(); buffer = bufferConsumer.build(); + checkState(bufferConsumer.isFinished() || buffers.size() == 1, "When there are multiple buffers, an unfinished bufferConsumer can not be at the head of the buffers queue."); + if (buffers.size() == 1) { + // turn off flushRequested flag if we drained all of the available data + flushRequested = false; + } + if (bufferConsumer.isFinished()) { buffers.pop().close(); decreaseBuffersInBacklogUnsafe(bufferConsumer.isBuffer()); } + if (buffer.readableBytes() > 0) { break; } @@ -169,7 +186,7 @@ BufferAndBacklog pollBuffer() { // will be 2 or more. return new BufferAndBacklog( buffer, - getNumberOfFinishedBuffers() > 0, + isAvailableUnsafe(), getBuffersInBacklog(), _nextBufferIsEvent()); } @@ -211,13 +228,23 @@ public PipelinedSubpartitionView createReadView(BufferAvailabilityListener avail readView = new PipelinedSubpartitionView(this, availabilityListener); if (!buffers.isEmpty()) { - readView.notifyDataAvailable(); + notifyDataAvailable(); } } return readView; } + public boolean isAvailable() { + synchronized (buffers) { + return isAvailableUnsafe(); + } + } + + private boolean isAvailableUnsafe() { + return flushRequested || getNumberOfFinishedBuffers() > 0; + } + // ------------------------------------------------------------------------ int getCurrentNumberOfBuffers() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionView.java index c60a604f0d7b93..9d083585b82730 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionView.java @@ -80,6 +80,11 @@ public boolean nextBufferIsEvent() { return parent.nextBufferIsEvent(); } + @Override + public boolean isAvailable() { + return parent.isAvailable(); + } + @Override public Throwable getFailureCause() { return parent.getFailureCause(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartitionView.java index 41fbb0a63c6691..b1ccd634704757 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultSubpartitionView.java @@ -57,4 +57,6 @@ public interface ResultSubpartitionView { * Returns whether the next buffer is an event or not. */ boolean nextBufferIsEvent(); + + boolean isAvailable(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java index 6ac493e7e2752f..6b731d42da933a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartition.java @@ -209,7 +209,6 @@ public ResultSubpartitionView createReadView(BufferAvailabilityListener availabi parent.getBufferProvider().getMemorySegmentSize(), availabilityListener); } - return readView; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java index b821dcf6afe8de..3c73e43d8cb9ff 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java @@ -153,18 +153,16 @@ public BufferAndBacklog getNextBuffer() throws IOException, InterruptedException return null; } else if (nextBuffer != null) { current = nextBuffer.build(); + checkState(nextBuffer.isFinished(), + "We can only read from SpillableSubpartition after it was finished"); - if (nextBuffer.isFinished()) { - newBacklog = parent.decreaseBuffersInBacklogUnsafe(nextBuffer.isBuffer()); - nextBuffer.close(); - nextBuffer = buffers.poll(); - } + newBacklog = parent.decreaseBuffersInBacklogUnsafe(nextBuffer.isBuffer()); + nextBuffer.close(); + nextBuffer = buffers.poll(); - isMoreAvailable = buffers.size() > 0; if (nextBuffer != null) { - isMoreAvailable = true; - listener.notifyDataAvailable(); nextBufferIsEvent = !nextBuffer.isBuffer(); + isMoreAvailable = true; } parent.updateStatistics(current); @@ -245,6 +243,20 @@ public boolean nextBufferIsEvent() { return spilledView.nextBufferIsEvent(); } + @Override + public boolean isAvailable() { + synchronized (buffers) { + if (nextBuffer != null) { + return true; + } + else if (spilledView == null) { + return false; + } + } // else: spilled + + return spilledView.isAvailable(); + } + @Override public Throwable getFailureCause() { SpilledSubpartitionView spilled = spilledView; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java index 4c5cd2e0e49ceb..378b0867d6feec 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java @@ -219,6 +219,14 @@ public boolean nextBufferIsEvent() { } } + @Override + public synchronized boolean isAvailable() { + if (nextBuffer != null) { + return true; + } + return !fileReader.hasReachedEndOfFile(); + } + @Override public Throwable getFailureCause() { return parent.getFailureCause(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java index a6e9fdcd0b34d9..7beb18fd21fca4 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java @@ -38,6 +38,10 @@ public static BufferBuilder createBufferBuilder(int size) { return createFilledBufferBuilder(size, 0); } + public static BufferBuilder createFilledBufferBuilder(int dataSize) { + return createFilledBufferBuilder(BUFFER_SIZE, dataSize); + } + public static BufferBuilder createFilledBufferBuilder(int size, int dataSize) { checkArgument(size >= dataSize); BufferBuilder bufferBuilder = new BufferBuilder( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java index 56abff1a614639..eca8263eabd9ce 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java @@ -219,6 +219,11 @@ public boolean nextBufferIsEvent() { return false; } + @Override + public boolean isAvailable() { + return true; + } + @Override public Throwable getFailureCause() { return null; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java index 16418ff9959381..f614c18902be16 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java @@ -66,7 +66,7 @@ public void testNotifyReaderNonEmptyOnEmptyReaders() throws Exception { CreditBasedSequenceNumberingViewReader reader1 = new CreditBasedSequenceNumberingViewReader(new InputChannelID(0, 0), 10, queue); CreditBasedSequenceNumberingViewReader reader2 = new CreditBasedSequenceNumberingViewReader(new InputChannelID(1, 1), 10, queue); - reader1.requestSubpartitionView((partitionId, index, availabilityListener) -> new NotReleasedResultSubpartitionView(), new ResultPartitionID(), 0); + reader1.requestSubpartitionView((partitionId, index, availabilityListener) -> new EmptyAlwaysAvailableResultSubpartitionView(), new ResultPartitionID(), 0); reader1.notifyDataAvailable(); assertTrue(reader1.isAvailable()); assertFalse(reader1.isRegisteredAsAvailable()); @@ -178,6 +178,11 @@ public BufferAndBacklog getNextBuffer() { buffers, false); } + + @Override + public boolean isAvailable() { + return buffersInBacklog.get() > 0; + } } private static class ReadOnlyBufferResultSubpartitionView extends DefaultBufferResultSubpartitionView { @@ -197,14 +202,19 @@ public BufferAndBacklog getNextBuffer() { } } - private static class NotReleasedResultSubpartitionView extends NoOpResultSubpartitionView { + private static class EmptyAlwaysAvailableResultSubpartitionView extends NoOpResultSubpartitionView { @Override public boolean isReleased() { return false; } + + @Override + public boolean isAvailable() { + return true; + } } - private static class ReleasedResultSubpartitionView extends NoOpResultSubpartitionView { + private static class ReleasedResultSubpartitionView extends EmptyAlwaysAvailableResultSubpartitionView { @Override public boolean isReleased() { return true; @@ -263,6 +273,11 @@ private static class NextIsEventResultSubpartitionView extends NoOpResultSubpart public boolean nextBufferIsEvent() { return true; } + + @Override + public boolean isAvailable() { + return true; + } } /** @@ -387,5 +402,10 @@ public Throwable getFailureCause() { public boolean nextBufferIsEvent() { return false; } + + @Override + public boolean isAvailable() { + return false; + } } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java index 528f0e296d341d..ee678abc4ccc4c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java @@ -43,6 +43,7 @@ import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createBufferBuilder; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createEventBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferBuilder; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; import static org.apache.flink.runtime.io.network.util.TestBufferFactory.BUFFER_SIZE; import static org.apache.flink.util.FutureUtil.waitForAll; @@ -142,6 +143,48 @@ public void testAddNonEmptyNotFinishedBuffer() throws Exception { } } + /** + * Normally moreAvailable flag from InputChannel should ignore non finished BufferConsumers, otherwise we would + * busy loop on the unfinished BufferConsumers. + */ + @Test + public void testUnfinishedBufferBehindFinished() throws Exception { + final ResultSubpartition subpartition = createSubpartition(); + AwaitableBufferAvailablityListener availablityListener = new AwaitableBufferAvailablityListener(); + ResultSubpartitionView readView = subpartition.createReadView(availablityListener); + + try { + subpartition.add(createFilledBufferConsumer(1025)); // finished + subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished + + assertNextBuffer(readView, 1025, false, 1); + } finally { + subpartition.release(); + } + } + + /** + * After flush call unfinished BufferConsumers should be reported as available, otherwise we might not flush some + * of the data. + */ + @Test + public void testFlushWithUnfinishedBufferBehindFinished() throws Exception { + final ResultSubpartition subpartition = createSubpartition(); + AwaitableBufferAvailablityListener availablityListener = new AwaitableBufferAvailablityListener(); + ResultSubpartitionView readView = subpartition.createReadView(availablityListener); + + try { + subpartition.add(createFilledBufferConsumer(1025)); // finished + subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished + subpartition.flush(); + + assertNextBuffer(readView, 1025, true, 1); + assertNextBuffer(readView, 1024, false, 1); + } finally { + subpartition.release(); + } + } + @Test public void testMultipleEmptyBuffers() throws Exception { final ResultSubpartition subpartition = createSubpartition(); @@ -187,6 +230,16 @@ public void testIllegalReadViewRequest() throws Exception { } } + @Test + public void testEmptyFlush() throws Exception { + final PipelinedSubpartition subpartition = createSubpartition(); + + AwaitableBufferAvailablityListener listener = new AwaitableBufferAvailablityListener(); + subpartition.createReadView(listener); + subpartition.flush(); + assertEquals(0, listener.getNumNotifications()); + } + @Test public void testBasicPipelinedProduceConsumeLogic() throws Exception { final PipelinedSubpartition subpartition = createSubpartition(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java index a6be748194ba7a..e41a85c5207b44 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java @@ -53,7 +53,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -319,7 +318,8 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception assertEquals(2, partition.getBuffersInBacklog()); assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); read.buffer().recycleBuffer(); - assertEquals(2, listener.getNumNotifications()); + assertTrue(read.isMoreAvailable()); + assertEquals(1, listener.getNumNotifications()); // since isMoreAvailable is set to true, no need for notification assertFalse(bufferConsumer.isRecycled()); assertFalse(read.nextBufferIsEvent()); @@ -332,8 +332,9 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception // only updated when getting/spilling the buffers but without the nextBuffer (kept in memory) assertEquals(BUFFER_DATA_SIZE * 3 + 4, partition.getTotalNumberOfBytes()); - listener.awaitNotifications(3, 30_000); - assertEquals(3, listener.getNumNotifications()); + listener.awaitNotifications(2, 30_000); + // Spiller finished + assertEquals(2, listener.getNumNotifications()); assertFalse(reader.nextBufferIsEvent()); // second buffer (retained in SpillableSubpartition#nextBuffer) read = reader.getNextBuffer(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java index 1b861dfab51db2..215726b3b5ad1f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java @@ -30,6 +30,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -143,4 +144,8 @@ protected void assertNextBuffer( assertEquals(expectedIsMoreAvailable, bufferAndBacklog.isMoreAvailable()); assertEquals(expectedBuffersInBacklog, bufferAndBacklog.buffersInBacklog()); } + + protected void assertNoNextBuffer(ResultSubpartitionView readView) throws IOException, InterruptedException { + assertNull(readView.getNextBuffer()); + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmarkTests.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmarkTests.java index a8251a80da61a4..a60fa3c4988ba7 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmarkTests.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmarkTests.java @@ -52,6 +52,14 @@ public void largeRemoteMode() throws Exception { env.tearDown(); } + @Test + public void largeRemoteAlwaysFlush() throws Exception { + StreamNetworkThroughputBenchmark env = new StreamNetworkThroughputBenchmark(); + env.setUp(1, 1, 0, false); + env.executeBenchmark(1_000_000); + env.tearDown(); + } + @Test public void pointToMultiPointBenchmark() throws Exception { StreamNetworkThroughputBenchmark benchmark = new StreamNetworkThroughputBenchmark(); From 767027ff6f388f808842760b7cf8bc807f8e6913 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Mon, 26 Feb 2018 16:13:06 +0100 Subject: [PATCH 0060/2294] [FLINK-8805][runtime] Optimize EvenSerializer.isEvent method For example, previously if the method was used to check for EndOfPartitionEvent and the Buffer contained huge custom event, the even had to be deserialized before performing the actual check. Now we are quickly entering the correct if/else branch and doing full costly deserialization only if we have to. Other calls to isEvent() then checking against EndOfPartitionEvent were not used. --- .../api/serialization/EventSerializer.java | 57 +++++-------------- .../network/netty/PartitionRequestQueue.java | 3 +- .../serialization/EventSerializerTest.java | 39 ++++++++----- 3 files changed, 42 insertions(+), 57 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java index d7fb7e851c150a..8d76bb26837f38 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java @@ -101,16 +101,15 @@ else if (eventClass == CancelCheckpointMarker.class) { } /** - * Identifies whether the given buffer encodes the given event. + * Identifies whether the given buffer encodes the given event. Custom events are not supported. * *

    Pre-condition: This buffer must encode some event!

    * * @param buffer the buffer to peak into * @param eventClass the expected class of the event type - * @param classLoader the class loader to use for custom event classes * @return whether the event class of the buffer matches the given eventClass */ - private static boolean isEvent(ByteBuffer buffer, Class eventClass, ClassLoader classLoader) throws IOException { + private static boolean isEvent(ByteBuffer buffer, Class eventClass) throws IOException { if (buffer.remaining() < 4) { throw new IOException("Incomplete event"); } @@ -122,38 +121,16 @@ private static boolean isEvent(ByteBuffer buffer, Class eventClass, ClassLoad try { int type = buffer.getInt(); - switch (type) { - case END_OF_PARTITION_EVENT: - return eventClass.equals(EndOfPartitionEvent.class); - case CHECKPOINT_BARRIER_EVENT: - return eventClass.equals(CheckpointBarrier.class); - case END_OF_SUPERSTEP_EVENT: - return eventClass.equals(EndOfSuperstepEvent.class); - case CANCEL_CHECKPOINT_MARKER_EVENT: - return eventClass.equals(CancelCheckpointMarker.class); - case OTHER_EVENT: - try { - final DataInputDeserializer deserializer = new DataInputDeserializer(buffer); - final String className = deserializer.readUTF(); - - final Class clazz; - try { - clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class); - } - catch (ClassNotFoundException e) { - throw new IOException("Could not load event class '" + className + "'.", e); - } - catch (ClassCastException e) { - throw new IOException("The class '" + className + "' is not a valid subclass of '" - + AbstractEvent.class.getName() + "'.", e); - } - return eventClass.equals(clazz); - } - catch (Exception e) { - throw new IOException("Error while deserializing or instantiating event.", e); - } - default: - throw new IOException("Corrupt byte stream for event"); + if (eventClass.equals(EndOfPartitionEvent.class)) { + return type == END_OF_PARTITION_EVENT; + } else if (eventClass.equals(CheckpointBarrier.class)) { + return type == CHECKPOINT_BARRIER_EVENT; + } else if (eventClass.equals(EndOfSuperstepEvent.class)) { + return type == END_OF_SUPERSTEP_EVENT; + } else if (eventClass.equals(CancelCheckpointMarker.class)) { + return type == CANCEL_CHECKPOINT_MARKER_EVENT; + } else { + throw new UnsupportedOperationException("Unsupported eventClass = " + eventClass); } } finally { @@ -314,17 +291,13 @@ public static AbstractEvent fromBuffer(Buffer buffer, ClassLoader classLoader) t } /** - * Identifies whether the given buffer encodes the given event. + * Identifies whether the given buffer encodes the given event. Custom events are not supported. * * @param buffer the buffer to peak into * @param eventClass the expected class of the event type - * @param classLoader the class loader to use for custom event classes * @return whether the event class of the buffer matches the given eventClass */ - public static boolean isEvent(final Buffer buffer, - final Class eventClass, - final ClassLoader classLoader) throws IOException { - return !buffer.isBuffer() && - isEvent(buffer.getNioBufferReadable(), eventClass, classLoader); + public static boolean isEvent(Buffer buffer, Class eventClass) throws IOException { + return !buffer.isBuffer() && isEvent(buffer.getNioBufferReadable(), eventClass); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java index 8d43815ada67ae..d63a88e718276e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java @@ -287,8 +287,7 @@ private NetworkSequenceViewReader pollAvailableReader() { } private boolean isEndOfPartitionEvent(Buffer buffer) throws IOException { - return EventSerializer.isEvent(buffer, EndOfPartitionEvent.class, - getClass().getClassLoader()); + return EventSerializer.isEvent(buffer, EndOfPartitionEvent.class); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializerTest.java index de5f4a82a7a933..c00fea7ebe1931 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializerTest.java @@ -33,11 +33,13 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests for the {@link EventSerializer}. @@ -95,7 +97,7 @@ public void testSerializeDeserializeEvent() throws Exception { } /** - * Tests {@link EventSerializer#isEvent(Buffer, Class, ClassLoader)} + * Tests {@link EventSerializer#isEvent(Buffer, Class)} * whether it peaks into the buffer only, i.e. after the call, the buffer * is still de-serializable. */ @@ -106,8 +108,7 @@ public void testIsEventPeakOnly() throws Exception { try { final ClassLoader cl = getClass().getClassLoader(); assertTrue( - EventSerializer - .isEvent(serializedEvent, EndOfPartitionEvent.class, cl)); + EventSerializer.isEvent(serializedEvent, EndOfPartitionEvent.class)); EndOfPartitionEvent event = (EndOfPartitionEvent) EventSerializer .fromBuffer(serializedEvent, cl); assertEquals(EndOfPartitionEvent.INSTANCE, event); @@ -117,7 +118,7 @@ public void testIsEventPeakOnly() throws Exception { } /** - * Tests {@link EventSerializer#isEvent(Buffer, Class, ClassLoader)} returns + * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns * the correct answer for various encoded event buffers. */ @Test @@ -130,12 +131,25 @@ public void testIsEvent() throws Exception { new CancelCheckpointMarker(287087987329842L) }; + Class[] expectedClasses = Arrays.stream(events) + .map(AbstractEvent::getClass) + .toArray(Class[]::new); + for (AbstractEvent evt : events) { - for (AbstractEvent evt2 : events) { - if (evt == evt2) { - assertTrue(checkIsEvent(evt, evt2.getClass())); + for (Class expectedClass: expectedClasses) { + if (expectedClass.equals(TestTaskEvent.class)) { + try { + checkIsEvent(evt, expectedClass); + fail("This should fail"); + } + catch (UnsupportedOperationException ex) { + // expected + } + } + else if (evt.getClass().equals(expectedClass)) { + assertTrue(checkIsEvent(evt, expectedClass)); } else { - assertFalse(checkIsEvent(evt, evt2.getClass())); + assertFalse(checkIsEvent(evt, expectedClass)); } } } @@ -143,23 +157,22 @@ public void testIsEvent() throws Exception { /** * Returns the result of - * {@link EventSerializer#isEvent(Buffer, Class, ClassLoader)} on a buffer + * {@link EventSerializer#isEvent(Buffer, Class)} on a buffer * that encodes the given event. * * @param event the event to encode * @param eventClass the event class to check against * - * @return whether {@link EventSerializer#isEvent(ByteBuffer, Class, ClassLoader)} + * @return whether {@link EventSerializer#isEvent(ByteBuffer, Class)} * thinks the encoded buffer matches the class */ private boolean checkIsEvent( AbstractEvent event, - Class eventClass) throws IOException { + Class eventClass) throws IOException { final Buffer serializedEvent = EventSerializer.toBuffer(event); try { - final ClassLoader cl = getClass().getClassLoader(); - return EventSerializer.isEvent(serializedEvent, eventClass, cl); + return EventSerializer.isEvent(serializedEvent, eventClass); } finally { serializedEvent.recycleBuffer(); } From b9b7416f4d6a708d22029a5e971af5b1f67e3296 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Tue, 27 Feb 2018 10:39:00 +0100 Subject: [PATCH 0061/2294] [FLINK-8750][runtime] Improve detection of no remaining data after EndOfPartitionEvent Because of race condition between: 1. releasing inputChannelsWithData lock in this method and reaching this place 2. empty data notification that re-enqueues a channel we can end up with moreAvailable flag set to true, while we expect no more data. This commit detects such situation, makes a correct assertion and turn off moreAvailable flag. This closes #5588. --- .../io/network/partition/consumer/SingleInputGate.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java index be4035c55b1735..b9091b2e52d15d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java @@ -554,6 +554,12 @@ private Optional getNextBufferOrEvent(boolean blocking) throws IO channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex()); if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) { + // Because of race condition between: + // 1. releasing inputChannelsWithData lock in this method and reaching this place + // 2. empty data notification that re-enqueues a channel + // we can end up with moreAvailable flag set to true, while we expect no more data. + checkState(!moreAvailable || !pollNextBufferOrEvent().isPresent()); + moreAvailable = false; hasReceivedAllEndOfPartitionEvents = true; } From 6e9e0dd6eb3c4fdb1168cc4e294d9fa52641ddb0 Mon Sep 17 00:00:00 2001 From: Zhijiang Date: Thu, 22 Feb 2018 22:41:38 +0800 Subject: [PATCH 0062/2294] [FLINK-8747][bugfix] The tag of waiting for floating buffers in RemoteInputChannel should be updated properly This closes #5558. --- .../consumer/RemoteInputChannel.java | 6 +++ .../consumer/RemoteInputChannelTest.java | 42 ++++++++++++++----- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java index 8174359db02762..990166f0b1effc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java @@ -337,6 +337,11 @@ public int getSenderBacklog() { return numRequiredBuffers - initialCredit; } + @VisibleForTesting + public boolean isWaitingForFloatingBuffers() { + return isWaitingForFloatingBuffers; + } + /** * The Buffer pool notifies this channel of an available floating buffer. If the channel is released or * currently does not need extra buffers, the buffer should be recycled to the buffer pool. Otherwise, @@ -362,6 +367,7 @@ public boolean notifyBufferAvailable(Buffer buffer) { // Important: double check the isReleased state inside synchronized block, so there is no // race condition when notifyBufferAvailable and releaseAllResources running in parallel. if (isReleased.get() || bufferQueue.getAvailableBufferSize() >= numRequiredBuffers) { + isWaitingForFloatingBuffers = false; buffer.recycleBuffer(); return false; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java index 7c8ed18d94937c..e3e6623c929ad6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java @@ -357,6 +357,7 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 16, inputChannel.getNumberOfRequiredBuffers()); assertEquals("There should be 0 buffer available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); + assertTrue(inputChannel.isWaitingForFloatingBuffers()); // Increase the backlog inputChannel.onSenderBacklog(16); @@ -370,11 +371,12 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 18, inputChannel.getNumberOfRequiredBuffers()); assertEquals("There should be 0 buffer available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); + assertTrue(inputChannel.isWaitingForFloatingBuffers()); - // Recycle one floating buffer - floatingBufferQueue.poll().recycleBuffer(); + // Recycle one exclusive buffer + exclusiveBuffer.recycleBuffer(); - // Assign the floating buffer to the listener and the channel is still waiting for more floating buffers + // The exclusive buffer is returned to the channel directly verify(bufferPool, times(15)).requestBuffer(); verify(bufferPool, times(1)).addBufferListener(inputChannel); assertEquals("There should be 14 buffers available in the channel", @@ -383,8 +385,9 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 18, inputChannel.getNumberOfRequiredBuffers()); assertEquals("There should be 0 buffer available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); + assertTrue(inputChannel.isWaitingForFloatingBuffers()); - // Recycle one more floating buffer + // Recycle one floating buffer floatingBufferQueue.poll().recycleBuffer(); // Assign the floating buffer to the listener and the channel is still waiting for more floating buffers @@ -396,32 +399,49 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 18, inputChannel.getNumberOfRequiredBuffers()); assertEquals("There should be 0 buffer available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); + assertTrue(inputChannel.isWaitingForFloatingBuffers()); // Decrease the backlog - inputChannel.onSenderBacklog(15); + inputChannel.onSenderBacklog(13); // Only the number of required buffers is changed by (backlog + numExclusiveBuffers) verify(bufferPool, times(15)).requestBuffer(); verify(bufferPool, times(1)).addBufferListener(inputChannel); assertEquals("There should be 15 buffers available in the channel", 15, inputChannel.getNumberOfAvailableBuffers()); - assertEquals("There should be 17 buffers required in the channel", - 17, inputChannel.getNumberOfRequiredBuffers()); + assertEquals("There should be 15 buffers required in the channel", + 15, inputChannel.getNumberOfRequiredBuffers()); assertEquals("There should be 0 buffer available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); + assertTrue(inputChannel.isWaitingForFloatingBuffers()); - // Recycle one exclusive buffer - exclusiveBuffer.recycleBuffer(); + // Recycle one more floating buffer + floatingBufferQueue.poll().recycleBuffer(); - // The exclusive buffer is returned to the channel directly + // Return the floating buffer to the buffer pool and the channel is not waiting for more floating buffers verify(bufferPool, times(15)).requestBuffer(); verify(bufferPool, times(1)).addBufferListener(inputChannel); + assertEquals("There should be 15 buffers available in the channel", + 15, inputChannel.getNumberOfAvailableBuffers()); + assertEquals("There should be 15 buffers required in the channel", + 15, inputChannel.getNumberOfRequiredBuffers()); + assertEquals("There should be 1 buffers available in local pool", + 1, bufferPool.getNumberOfAvailableMemorySegments()); + assertFalse(inputChannel.isWaitingForFloatingBuffers()); + + // Increase the backlog again + inputChannel.onSenderBacklog(15); + + // The floating buffer is requested from the buffer pool and the channel is registered as listener again. + verify(bufferPool, times(17)).requestBuffer(); + verify(bufferPool, times(2)).addBufferListener(inputChannel); assertEquals("There should be 16 buffers available in the channel", 16, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 17 buffers required in the channel", 17, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffers available in local pool", + assertEquals("There should be 0 buffer available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); + assertTrue(inputChannel.isWaitingForFloatingBuffers()); } finally { // Release all the buffer resources From 6165b3db5587170bed1a40bb1e5f2f3613f24e3f Mon Sep 17 00:00:00 2001 From: Zhijiang Date: Fri, 23 Feb 2018 09:55:57 +0800 Subject: [PATCH 0063/2294] [hotfix] Fix package private and comments --- .../partition/consumer/RemoteInputChannel.java | 2 +- .../consumer/RemoteInputChannelTest.java | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java index 990166f0b1effc..0f70d448020a31 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java @@ -338,7 +338,7 @@ public int getSenderBacklog() { } @VisibleForTesting - public boolean isWaitingForFloatingBuffers() { + boolean isWaitingForFloatingBuffers() { return isWaitingForFloatingBuffers; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java index e3e6623c929ad6..97a56887739891 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java @@ -355,7 +355,7 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 13, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 16 buffers required in the channel", 16, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); assertTrue(inputChannel.isWaitingForFloatingBuffers()); @@ -369,7 +369,7 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 13, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 18 buffers required in the channel", 18, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); assertTrue(inputChannel.isWaitingForFloatingBuffers()); @@ -383,7 +383,7 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 14, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 18 buffers required in the channel", 18, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); assertTrue(inputChannel.isWaitingForFloatingBuffers()); @@ -397,7 +397,7 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 15, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 18 buffers required in the channel", 18, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); assertTrue(inputChannel.isWaitingForFloatingBuffers()); @@ -411,7 +411,7 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 15, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 15 buffers required in the channel", 15, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); assertTrue(inputChannel.isWaitingForFloatingBuffers()); @@ -439,7 +439,7 @@ public void testAvailableBuffersLessThanRequiredBuffers() throws Exception { 16, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 17 buffers required in the channel", 17, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); assertTrue(inputChannel.isWaitingForFloatingBuffers()); @@ -490,7 +490,7 @@ public void testAvailableBuffersEqualToRequiredBuffers() throws Exception { 14, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 14 buffers required in the channel", 14, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); // Recycle one floating buffer @@ -569,7 +569,7 @@ public void testAvailableBuffersMoreThanRequiredBuffers() throws Exception { 14, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 14 buffers required in the channel", 14, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); // Decrease the backlog to make the number of available buffers more than required buffers @@ -582,7 +582,7 @@ public void testAvailableBuffersMoreThanRequiredBuffers() throws Exception { 14, inputChannel.getNumberOfAvailableBuffers()); assertEquals("There should be 12 buffers required in the channel", 12, inputChannel.getNumberOfRequiredBuffers()); - assertEquals("There should be 0 buffer available in local pool", + assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments()); // Recycle one exclusive buffer From 420355715e43d5c870bd7f89808430fc959cedc4 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 26 Feb 2018 17:50:10 +0100 Subject: [PATCH 0064/2294] [hotfix][network] minor improvements in UnionInputGate --- .../io/network/partition/consumer/UnionInputGate.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java index 393e08775141a6..44cdd526c5364a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java @@ -38,7 +38,7 @@ /** * Input gate wrapper to union the input from multiple input gates. * - *

    Each input gate has input channels attached from which it reads data. At each input gate, the + *

    Each input gate has input channels attached from which it reads data. At each input gate, the * input channels have unique IDs from 0 (inclusive) to the number of input channels (exclusive). * * 

    @@ -49,7 +49,7 @@
  * +--------------+--------------+
  *
    * - * The union input gate maps these IDs from 0 to the *total* number of input channels across all + *

    The union input gate maps these IDs from 0 to the *total* number of input channels across all * unioned input gates, e.g. the channels of input gate 0 keep their original indexes and the * channel indexes of input gate 1 are set off by 2 to 2--4. * @@ -183,11 +183,11 @@ public Optional getNextBufferOrEvent() throws IOException, Interr bufferOrEvent.setChannelIndex(channelIndexOffset + bufferOrEvent.getChannelIndex()); bufferOrEvent.setMoreAvailable(bufferOrEvent.moreAvailable() || inputGateWithData.moreInputGatesAvailable); - return Optional.ofNullable(bufferOrEvent); + return Optional.of(bufferOrEvent); } @Override - public Optional pollNextBufferOrEvent() throws IOException, InterruptedException { + public Optional pollNextBufferOrEvent() throws UnsupportedOperationException { throw new UnsupportedOperationException(); } @@ -217,7 +217,7 @@ private static class InputGateWithData { private final BufferOrEvent bufferOrEvent; private final boolean moreInputGatesAvailable; - public InputGateWithData(InputGate inputGate, BufferOrEvent bufferOrEvent, boolean moreInputGatesAvailable) { + InputGateWithData(InputGate inputGate, BufferOrEvent bufferOrEvent, boolean moreInputGatesAvailable) { this.inputGate = checkNotNull(inputGate); this.bufferOrEvent = checkNotNull(bufferOrEvent); this.moreInputGatesAvailable = moreInputGatesAvailable; From e8de53817c60917776c7264623cf8adfd7886f93 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 26 Feb 2018 17:52:37 +0100 Subject: [PATCH 0065/2294] [FLINK-8737][network] disallow creating a union of UnionInputGate instances Recently, the pollNextBufferOrEvent() was added but not implemented but this is used in getNextBufferOrEvent() and thus any UnionInputGate containing a UnionInputGate would have failed already. There should be no use case for wiring up inputs this way. Therefore, fail early when trying to construct this. This closes #5583. --- .../io/network/partition/consumer/UnionInputGate.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java index 44cdd526c5364a..742592a93f0b35 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java @@ -61,7 +61,7 @@ * +--------------------+ * * - * It is possible to recursively union union input gates. + * It is NOT possible to recursively union union input gates. */ public class UnionInputGate implements InputGate, InputGateListener { @@ -103,6 +103,11 @@ public UnionInputGate(InputGate... inputGates) { int currentNumberOfInputChannels = 0; for (InputGate inputGate : inputGates) { + if (inputGate instanceof UnionInputGate) { + // if we want to add support for this, we need to implement pollNextBufferOrEvent() + throw new UnsupportedOperationException("Cannot union a union of input gates."); + } + // The offset to use for buffer or event instances received from this input gate. inputGateToIndexOffsetMap.put(checkNotNull(inputGate), currentNumberOfInputChannels); inputGatesWithRemainingData.add(inputGate); From 273fea4aa36123c0501845e4bc3f8777f203e596 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Wed, 28 Feb 2018 12:15:30 +0100 Subject: [PATCH 0066/2294] [hotfix] [tests] Fix SelfConnectionITCase The test previously did not fail on failed execution, and thus evaluated incomplete results from a failed execution with th expected results. This cleans up serialization warnings and uses lambdas where possible, to make the code more readable. --- .../runtime/SelfConnectionITCase.java | 63 ++++--------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java index b302513cd78674..a8023a01a594bc 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/SelfConnectionITCase.java @@ -18,8 +18,6 @@ package org.apache.flink.test.streaming.runtime; import org.apache.flink.api.common.functions.FlatMapFunction; -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.co.CoMapFunction; @@ -38,6 +36,7 @@ /** * Integration tests for connected streams. */ +@SuppressWarnings("serial") public class SelfConnectionITCase extends AbstractTestBase { /** @@ -46,26 +45,17 @@ public class SelfConnectionITCase extends AbstractTestBase { @Test public void differentDataStreamSameChain() throws Exception { - TestListResultSink resultSink = new TestListResultSink(); + TestListResultSink resultSink = new TestListResultSink<>(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DataStream src = env.fromElements(1, 3, 5); - DataStream stringMap = src.map(new MapFunction() { - private static final long serialVersionUID = 1L; - - @Override - public String map(Integer value) throws Exception { - return "x " + value; - } - }); + DataStream stringMap = src.map(value -> "x " + value); stringMap.connect(src).map(new CoMapFunction() { - private static final long serialVersionUID = 1L; - @Override public String map1(String value) { return value; @@ -94,55 +84,30 @@ public String map2(Integer value) { * (This is not actually self-connect.) */ @Test - public void differentDataStreamDifferentChain() { + public void differentDataStreamDifferentChain() throws Exception { - TestListResultSink resultSink = new TestListResultSink(); + TestListResultSink resultSink = new TestListResultSink<>(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); DataStream src = env.fromElements(1, 3, 5).disableChaining(); - DataStream stringMap = src.flatMap(new FlatMapFunction() { - - private static final long serialVersionUID = 1L; + DataStream stringMap = src + .flatMap(new FlatMapFunction() { @Override public void flatMap(Integer value, Collector out) throws Exception { out.collect("x " + value); } - }).keyBy(new KeySelector() { - - private static final long serialVersionUID = 1L; - - @Override - public Integer getKey(String value) throws Exception { - return value.length(); - } - }); - - DataStream longMap = src.map(new MapFunction() { + }).keyBy(String::length); - private static final long serialVersionUID = 1L; - - @Override - public Long map(Integer value) throws Exception { - return (long) (value + 1); - } - }).keyBy(new KeySelector() { - - private static final long serialVersionUID = 1L; - - @Override - public Integer getKey(Long value) throws Exception { - return value.intValue(); - } - }); + DataStream longMap = src + .map(value -> (long) (value + 1)) + .keyBy(Long::intValue); stringMap.connect(longMap).map(new CoMapFunction() { - private static final long serialVersionUID = 1L; - @Override public String map1(String value) { return value; @@ -154,11 +119,7 @@ public String map2(Long value) { } }).addSink(resultSink); - try { - env.execute(); - } catch (Exception e) { - e.printStackTrace(); - } + env.execute(); List expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6"); List result = resultSink.getResult(); From b19769db7f916b2eb40a9a68c0ee60d0b27da0a2 Mon Sep 17 00:00:00 2001 From: gyao Date: Wed, 28 Feb 2018 13:04:19 +0100 Subject: [PATCH 0067/2294] [hotfix] Add missing space to log message in ZooKeeperLeaderElectionService --- .../runtime/leaderelection/ZooKeeperLeaderElectionService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java index 59d359218060c9..dc0f3aefc68460 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java @@ -199,7 +199,7 @@ public void confirmLeaderSessionID(UUID leaderSessionID) { } } } else { - LOG.warn("The leader session ID {} was confirmed even though the" + + LOG.warn("The leader session ID {} was confirmed even though the " + "corresponding JobManager was not elected as the leader.", leaderSessionID); } } From 19874459d5fc750158eea9dcc54be81d5d4e08c9 Mon Sep 17 00:00:00 2001 From: gyao Date: Wed, 28 Feb 2018 13:06:00 +0100 Subject: [PATCH 0068/2294] [hotfix][Javadoc] Fix typo in YARN Utils: teh -> the --- flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java index 9ae5b543296b53..ff2478ede1415d 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java @@ -320,7 +320,7 @@ public static Map getEnvironmentVariables(String envPrefix, org. * * @return The launch context for the TaskManager processes. * - * @throws Exception Thrown if teh launch context could not be created, for example if + * @throws Exception Thrown if the launch context could not be created, for example if * the resources could not be copied. */ static ContainerLaunchContext createTaskExecutorContext( From f409f9208f340a6b741818a56a9a07b236fb0a7f Mon Sep 17 00:00:00 2001 From: gyao Date: Wed, 28 Feb 2018 13:07:04 +0100 Subject: [PATCH 0069/2294] [hotfix][Javadoc] Fix typo in YarnTestBase: teh -> the --- .../src/test/java/org/apache/flink/yarn/YarnTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index c863e145f05e92..7bca32192489b1 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -102,7 +102,7 @@ public abstract class YarnTestBase extends TestLogger { "Started SelectChannelConnector@0.0.0.0:8081" // Jetty should start on a random port in YARN mode. }; - /** These strings are white-listed, overriding teh prohibited strings. */ + /** These strings are white-listed, overriding the prohibited strings. */ protected static final String[] WHITELISTED_STRINGS = { "akka.remote.RemoteTransportExceptionNoStackTrace", // workaround for annoying InterruptedException logging: From 035257e425133de9368bbef268c38f179bdc68c2 Mon Sep 17 00:00:00 2001 From: gyao Date: Wed, 28 Feb 2018 13:08:25 +0100 Subject: [PATCH 0070/2294] [hotfix][tests] Fix wrong assertEquals in YARNSessionCapacitySchedulerITCase Test swapped actual and expected arguments. Remove catching Throwable in test; instead propagate all exceptions. --- .../yarn/YARNSessionCapacitySchedulerITCase.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java index 3a674ad0829895..d00a9c447d1353 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java @@ -416,7 +416,7 @@ public void perJobYarnClusterWithParallelism() throws IOException { * Test a fire-and-forget job submission to a YARN cluster. */ @Test(timeout = 60000) - public void testDetachedPerJobYarnCluster() throws IOException { + public void testDetachedPerJobYarnCluster() throws Exception { LOG.info("Starting testDetachedPerJobYarnCluster()"); File exampleJarLocation = new File("target/programs/BatchWordCount.jar"); @@ -432,7 +432,7 @@ public void testDetachedPerJobYarnCluster() throws IOException { * Test a fire-and-forget job submission to a YARN cluster. */ @Test(timeout = 60000) - public void testDetachedPerJobYarnClusterWithStreamingJob() throws IOException { + public void testDetachedPerJobYarnClusterWithStreamingJob() throws Exception { LOG.info("Starting testDetachedPerJobYarnClusterWithStreamingJob()"); File exampleJarLocation = new File("target/programs/StreamingWordCount.jar"); @@ -444,7 +444,7 @@ public void testDetachedPerJobYarnClusterWithStreamingJob() throws IOException { LOG.info("Finished testDetachedPerJobYarnClusterWithStreamingJob()"); } - private void testDetachedPerJobYarnClusterInternal(String job) throws IOException { + private void testDetachedPerJobYarnClusterInternal(String job) throws Exception { YarnClient yc = YarnClient.createYarnClient(); yc.init(YARN_CONFIGURATION); yc.start(); @@ -575,9 +575,6 @@ public boolean accept(File dir, String name) { } while (rep.getYarnApplicationState() == YarnApplicationState.RUNNING); verifyApplicationTags(rep); - } catch (Throwable t) { - LOG.warn("Error while detached yarn session was running", t); - Assert.fail(t.getMessage()); } finally { //cleanup the yarn-properties file @@ -625,7 +622,7 @@ private void verifyApplicationTags(final ApplicationReport report) throws Invoca @SuppressWarnings("unchecked") Set applicationTags = (Set) applicationTagsMethod.invoke(report); - Assert.assertEquals(applicationTags, Collections.singleton("test-tag")); + Assert.assertEquals(Collections.singleton("test-tag"), applicationTags); } @After From 45397fe974e1390cd39a34fc2eb216f3771ddf06 Mon Sep 17 00:00:00 2001 From: gyao Date: Wed, 28 Feb 2018 13:20:23 +0100 Subject: [PATCH 0071/2294] [FLINK-7805][flip6] Recover YARN containers after AM restart. Recover previously running containers after a restart of the ApplicationMaster. This is a port of a feature that was already implemented prior to FLIP-6. Extract RegisterApplicationMasterResponseReflector class into separate file. This closes #5597. --- ...terApplicationMasterResponseReflector.java | 102 +++++++++++++++ .../flink/yarn/YarnFlinkResourceManager.java | 52 -------- .../flink/yarn/YarnResourceManager.java | 17 ++- ...pplicationMasterResponseReflectorTest.java | 117 ++++++++++++++++++ 4 files changed, 235 insertions(+), 53 deletions(-) create mode 100644 flink-yarn/src/main/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflector.java create mode 100644 flink-yarn/src/test/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflectorTest.java diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflector.java b/flink-yarn/src/main/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflector.java new file mode 100644 index 00000000000000..13b5745d0cbc35 --- /dev/null +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflector.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.annotation.VisibleForTesting; + +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.Container; +import org.slf4j.Logger; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Looks up the method {@link RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()} + * once and saves the method. This saves computation time on subsequent calls. + */ +class RegisterApplicationMasterResponseReflector { + + private final Logger logger; + + /** + * Reflected method {@link RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()}. + */ + private Method method; + + RegisterApplicationMasterResponseReflector(final Logger log) { + this(log, RegisterApplicationMasterResponse.class); + } + + @VisibleForTesting + RegisterApplicationMasterResponseReflector(final Logger log, final Class clazz) { + this.logger = requireNonNull(log); + requireNonNull(clazz); + + try { + method = clazz.getMethod("getContainersFromPreviousAttempts"); + } catch (NoSuchMethodException e) { + // that happens in earlier Hadoop versions (pre 2.2) + logger.info("Cannot reconnect to previously allocated containers. " + + "This YARN version does not support 'getContainersFromPreviousAttempts()'"); + } + } + + /** + * Checks if a YARN application still has registered containers. If the application master + * registered at the ResourceManager for the first time, this list will be empty. If the + * application master registered a repeated time (after a failure and recovery), this list + * will contain the containers that were previously allocated. + * + * @param response The response object from the registration at the ResourceManager. + * @return A list with containers from previous application attempt. + */ + List getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse response) { + return getContainersFromPreviousAttemptsUnsafe(response); + } + + /** + * Same as {@link #getContainersFromPreviousAttempts(RegisterApplicationMasterResponse)} but + * allows to pass objects that are not of type {@link RegisterApplicationMasterResponse}. + */ + @VisibleForTesting + List getContainersFromPreviousAttemptsUnsafe(final Object response) { + if (method != null && response != null) { + try { + @SuppressWarnings("unchecked") + final List containers = (List) method.invoke(response); + if (containers != null && !containers.isEmpty()) { + return containers; + } + } catch (Exception t) { + logger.error("Error invoking 'getContainersFromPreviousAttempts()'", t); + } + } + + return Collections.emptyList(); + } + + @VisibleForTesting + Method getMethod() { + return method; + } +} diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java index 4d8142f7b830ac..8e686bbbe34c10 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnFlinkResourceManager.java @@ -47,10 +47,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; -import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -616,56 +614,6 @@ private FinalApplicationStatus getYarnStatus(ApplicationStatus status) { } } - /** - * Looks up the getContainersFromPreviousAttempts method on RegisterApplicationMasterResponse - * once and saves the method. This saves computation time on the sequent calls. - */ - private static class RegisterApplicationMasterResponseReflector { - - private Logger logger; - private Method method; - - public RegisterApplicationMasterResponseReflector(Logger log) { - this.logger = log; - - try { - method = RegisterApplicationMasterResponse.class - .getMethod("getContainersFromPreviousAttempts"); - - } catch (NoSuchMethodException e) { - // that happens in earlier Hadoop versions - logger.info("Cannot reconnect to previously allocated containers. " + - "This YARN version does not support 'getContainersFromPreviousAttempts()'"); - } - } - - /** - * Checks if a YARN application still has registered containers. If the application master - * registered at the ResourceManager for the first time, this list will be empty. If the - * application master registered a repeated time (after a failure and recovery), this list - * will contain the containers that were previously allocated. - * - * @param response The response object from the registration at the ResourceManager. - * @return A list with containers from previous application attempt. - */ - private List getContainersFromPreviousAttempts(RegisterApplicationMasterResponse response) { - if (method != null && response != null) { - try { - @SuppressWarnings("unchecked") - List list = (List) method.invoke(response); - if (list != null && !list.isEmpty()) { - return list; - } - } catch (Throwable t) { - logger.error("Error invoking 'getContainersFromPreviousAttempts()'", t); - } - } - - return Collections.emptyList(); - } - - } - // ------------------------------------------------------------------------ // Actor props factory // ------------------------------------------------------------------------ diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 5380356de79655..f3ec04bf1e6e0f 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -41,6 +41,7 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -192,11 +193,24 @@ protected AMRMClientAsync createAndStartResourceMan restPort = -1; } - resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl); + final RegisterApplicationMasterResponse registerApplicationMasterResponse = + resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl); + getContainersFromPreviousAttempts(registerApplicationMasterResponse); return resourceManagerClient; } + private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) { + final List containersFromPreviousAttempts = + new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse); + + log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts); + + for (final Container container : containersFromPreviousAttempts) { + workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container)); + } + } + protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) { // create the client to communicate with the node managers NMClient nodeManagerClient = NMClient.createNMClient(); @@ -315,6 +329,7 @@ public void onContainersCompleted(List list) { closeTaskManagerConnection(new ResourceID( container.getContainerId().toString()), new Exception(container.getDiagnostics())); } + workerNodeMap.remove(new ResourceID(container.getContainerId().toString())); } } diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflectorTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflectorTest.java new file mode 100644 index 00000000000000..af33e65c27a2e1 --- /dev/null +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/RegisterApplicationMasterResponseReflectorTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.util.TestLogger; + +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.Container; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for {@link RegisterApplicationMasterResponseReflector}. + */ +public class RegisterApplicationMasterResponseReflectorTest extends TestLogger { + + private static final Logger LOG = LoggerFactory.getLogger(RegisterApplicationMasterResponseReflectorTest.class); + + @Mock + private Container mockContainer; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testCallsMethodIfPresent() { + final RegisterApplicationMasterResponseReflector registerApplicationMasterResponseReflector = + new RegisterApplicationMasterResponseReflector(LOG, HasMethod.class); + + final List containersFromPreviousAttemptsUnsafe = + registerApplicationMasterResponseReflector.getContainersFromPreviousAttemptsUnsafe(new + HasMethod()); + + assertThat(containersFromPreviousAttemptsUnsafe, hasSize(1)); + } + + @Test + public void testDoesntCallMethodIfAbsent() { + final RegisterApplicationMasterResponseReflector registerApplicationMasterResponseReflector = + new RegisterApplicationMasterResponseReflector(LOG, HasMethod.class); + + final List containersFromPreviousAttemptsUnsafe = + registerApplicationMasterResponseReflector.getContainersFromPreviousAttemptsUnsafe(new + Object()); + + assertThat(containersFromPreviousAttemptsUnsafe, empty()); + } + + @Test + public void testGetMethodReflectiveHadoop22() { + assumeTrue( + "Method getContainersFromPreviousAttempts is not supported by Hadoop: " + + VersionInfo.getVersion(), + isHadoopVersionGreaterThanOrEquals(2, 2)); + + final RegisterApplicationMasterResponseReflector registerApplicationMasterResponseReflector = + new RegisterApplicationMasterResponseReflector(LOG); + + final Method method = registerApplicationMasterResponseReflector.getMethod(); + assertThat(method, notNullValue()); + } + + private static boolean isHadoopVersionGreaterThanOrEquals(final int major, final int minor) { + final String[] splitVersion = VersionInfo.getVersion().split("\\."); + final int[] versions = Arrays.stream(splitVersion).mapToInt(Integer::parseInt).toArray(); + return versions[0] >= major && versions[1] >= minor; + } + + /** + * Class which has a method with the same signature as + * {@link RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()}. + */ + private class HasMethod { + + /** + * Called from {@link #testCallsMethodIfPresent()}. + */ + @SuppressWarnings("unused") + public List getContainersFromPreviousAttempts() { + return Collections.singletonList(mockContainer); + } + } +} From 6ad626ae51a157306ddf4165f13ff5eb5b4d5e8b Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Thu, 1 Mar 2018 09:43:56 +0100 Subject: [PATCH 0072/2294] [FLINK-4387][QS] don't wait and process requests at Netty servers after shutdown request There is a race condition on an assertion in Netty's event loop that may cause tests to fail when finished early. This was fixed in 4.0.33.Final, see https://github.com/netty/netty/issues/4357. This closes #5606. --- .../org/apache/flink/queryablestate/network/ClientTest.java | 3 ++- .../flink/queryablestate/network/KvStateServerTest.java | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/ClientTest.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/ClientTest.java index 8638efa680fc06..6aa4710942e6d1 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/ClientTest.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/ClientTest.java @@ -111,7 +111,8 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { if (nioGroup != null) { - nioGroup.shutdownGracefully(); + // note: no "quiet period" to not trigger Netty#4357 + nioGroup.shutdownGracefully(0, 10, TimeUnit.SECONDS); } } diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerTest.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerTest.java index 8af9cf58851fab..79c23ad2a2d95e 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerTest.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/KvStateServerTest.java @@ -79,7 +79,8 @@ public class KvStateServerTest { @AfterClass public static void tearDown() throws Exception { if (NIO_GROUP != null) { - NIO_GROUP.shutdownGracefully(); + // note: no "quiet period" to not trigger Netty#4357 + NIO_GROUP.shutdownGracefully(0, 10, TimeUnit.SECONDS); } } @@ -191,7 +192,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (bootstrap != null) { EventLoopGroup group = bootstrap.group(); if (group != null) { - group.shutdownGracefully(); + // note: no "quiet period" to not trigger Netty#4357 + group.shutdownGracefully(0, 10, TimeUnit.SECONDS); } } } From b2a1b49566cc9dda3e393887371c678b48349fde Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Thu, 1 Mar 2018 10:58:14 +0100 Subject: [PATCH 0073/2294] [hotfix] [formats] Make ObjectMapper final in JsonNodeDeserializationSchema --- .../flink/formats/json/JsonNodeDeserializationSchema.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonNodeDeserializationSchema.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonNodeDeserializationSchema.java index 7501cc3a836192..e20c83977343a5 100644 --- a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonNodeDeserializationSchema.java +++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonNodeDeserializationSchema.java @@ -35,13 +35,10 @@ public class JsonNodeDeserializationSchema extends AbstractDeserializationSchema private static final long serialVersionUID = -1699854177598621044L; - private ObjectMapper mapper; + private final ObjectMapper mapper = new ObjectMapper(); @Override public ObjectNode deserialize(byte[] message) throws IOException { - if (mapper == null) { - mapper = new ObjectMapper(); - } return mapper.readValue(message, ObjectNode.class); } } From 53345c841a9398f2e36a30b648301d535b0563cc Mon Sep 17 00:00:00 2001 From: zentol Date: Thu, 1 Mar 2018 11:52:28 +0100 Subject: [PATCH 0074/2294] [hotfix][build] Add missing shade-plugin execution id's 2 metric modules weren't setting the execution id to "shade-flink" causing them to not pick up the default shade-plugin configuration. --- flink-metrics/flink-metrics-datadog/pom.xml | 1 + flink-metrics/flink-metrics-prometheus/pom.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/flink-metrics/flink-metrics-datadog/pom.xml b/flink-metrics/flink-metrics-datadog/pom.xml index 995dafc91fa02f..86141459d929e5 100644 --- a/flink-metrics/flink-metrics-datadog/pom.xml +++ b/flink-metrics/flink-metrics-datadog/pom.xml @@ -60,6 +60,7 @@ under the License. maven-shade-plugin + shade-flink package shade diff --git a/flink-metrics/flink-metrics-prometheus/pom.xml b/flink-metrics/flink-metrics-prometheus/pom.xml index cb983edc1312ee..ddaeae031766f1 100644 --- a/flink-metrics/flink-metrics-prometheus/pom.xml +++ b/flink-metrics/flink-metrics-prometheus/pom.xml @@ -105,6 +105,7 @@ under the License. maven-shade-plugin + shade-flink package shade From f152542468b37783932fc2c7725a3a5871b7a701 Mon Sep 17 00:00:00 2001 From: Jelmer Kuperus Date: Wed, 28 Feb 2018 21:34:08 +0100 Subject: [PATCH 0075/2294] [FLINK-8814] [file system sinks] Control over the extension of part files created by BucketingSink. --- .../fs/bucketing/BucketingSink.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java index 6e7f460a6dc636..faf3c566803e8c 100644 --- a/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java +++ b/flink-connectors/flink-connector-filesystem/src/main/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSink.java @@ -226,7 +226,12 @@ public class BucketingSink /** * The default prefix for part files. */ - private static final String DEFAULT_PART_REFIX = "part"; + private static final String DEFAULT_PART_PREFIX = "part"; + + /** + * The default suffix for part files. + */ + private static final String DEFAULT_PART_SUFFIX = null; /** * The default timeout for asynchronous operations such as recoverLease and truncate (in {@code ms}). @@ -263,7 +268,8 @@ public class BucketingSink private String validLengthSuffix = DEFAULT_VALID_SUFFIX; private String validLengthPrefix = DEFAULT_VALID_PREFIX; - private String partPrefix = DEFAULT_PART_REFIX; + private String partPrefix = DEFAULT_PART_PREFIX; + private String partSuffix = DEFAULT_PART_SUFFIX; private boolean useTruncate = true; @@ -530,6 +536,10 @@ private void openNewPartFile(Path bucketPath, BucketState bucketState) throws partPath = new Path(bucketPath, partPrefix + "-" + subtaskIndex + "-" + bucketState.partCounter); } + if (partSuffix != null) { + partPath = partPath.suffix(partSuffix); + } + // increase, so we don't have to check for this name next time bucketState.partCounter++; @@ -986,6 +996,14 @@ public BucketingSink setValidLengthPrefix(String validLengthPrefix) { return this; } + /** + * Sets the prefix of part files. The default is no suffix. + */ + public BucketingSink setPartSuffix(String partSuffix) { + this.partSuffix = partSuffix; + return this; + } + /** * Sets the prefix of part files. The default is {@code "part"}. */ From a0336f2e822b738e7843bd2fb69cd0347d9fb757 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 28 Feb 2018 18:02:40 +0100 Subject: [PATCH 0076/2294] [FLINK-8810] Move end-to-end test scripts to end-to-end module This also makes the tests executable by calling $ flink-end-to-end-tests/run-pre-commit-tests.sh --- flink-end-to-end-tests/README.md | 24 +++++ .../run-pre-commit-tests.sh | 98 +++++++++++++++++++ .../test-scripts}/common.sh | 0 .../test-scripts}/test-data/words | 0 .../test-scripts}/test_batch_wordcount.sh | 0 .../test-scripts}/test_hadoop_free.sh | 0 .../test-scripts}/test_shaded_hadoop_s3a.sh | 0 .../test-scripts}/test_shaded_presto_s3.sh | 0 .../test_streaming_classloader.sh | 0 .../test-scripts}/test_streaming_kafka010.sh | 0 .../test_streaming_python_wordcount.sh | 0 pom.xml | 2 +- tools/travis_mvn_watchdog.sh | 58 +---------- 13 files changed, 126 insertions(+), 56 deletions(-) create mode 100644 flink-end-to-end-tests/README.md create mode 100755 flink-end-to-end-tests/run-pre-commit-tests.sh rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/common.sh (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test-data/words (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test_batch_wordcount.sh (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test_hadoop_free.sh (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test_shaded_hadoop_s3a.sh (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test_shaded_presto_s3.sh (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test_streaming_classloader.sh (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test_streaming_kafka010.sh (100%) rename {test-infra/end-to-end-test => flink-end-to-end-tests/test-scripts}/test_streaming_python_wordcount.sh (100%) diff --git a/flink-end-to-end-tests/README.md b/flink-end-to-end-tests/README.md new file mode 100644 index 00000000000000..1c8aadcc6177b8 --- /dev/null +++ b/flink-end-to-end-tests/README.md @@ -0,0 +1,24 @@ +# Flink End-to-End Tests + +This module contains tests that verify end-to-end behaviour of Flink. + +## Running Tests +You can run all tests by executing + +``` +$ FLINK_DIR= flink-end-to-end-tests/run-pre-commit-tests.sh +``` + +where is a Flink distribution directory. + +You can also run tests individually via + +``` +$ FLINK_DIR= flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh +``` + +## Writing Tests + +Have a look at test_batch_wordcount.sh for a very basic test and +test_streaming_kafka010.sh for a more involved example. Whenever possible, try +to put new functionality in common.sh so that it can be reused by other tests. diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh new file mode 100755 index 00000000000000..2c1810b91c8bf8 --- /dev/null +++ b/flink-end-to-end-tests/run-pre-commit-tests.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd )`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +EXIT_CODE=0 + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Wordcount end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_batch_wordcount.sh + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Kafka end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_streaming_kafka010.sh + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running class loading end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_streaming_classloader.sh + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Shaded Hadoop S3A end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_shaded_hadoop_s3a.sh + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Shaded Presto S3 end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Hadoop-free Wordcount end-to-end test\n" + printf "==============================================================================\n" + CLUSTER_MODE=cluster $END_TO_END_DIR/test-scripts/test_hadoop_free.sh + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Streaming Python Wordcount end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_streaming_python_wordcount.sh + EXIT_CODE=$? +fi + + +# Exit code for Travis build success/failure +exit $EXIT_CODE diff --git a/test-infra/end-to-end-test/common.sh b/flink-end-to-end-tests/test-scripts/common.sh similarity index 100% rename from test-infra/end-to-end-test/common.sh rename to flink-end-to-end-tests/test-scripts/common.sh diff --git a/test-infra/end-to-end-test/test-data/words b/flink-end-to-end-tests/test-scripts/test-data/words similarity index 100% rename from test-infra/end-to-end-test/test-data/words rename to flink-end-to-end-tests/test-scripts/test-data/words diff --git a/test-infra/end-to-end-test/test_batch_wordcount.sh b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh similarity index 100% rename from test-infra/end-to-end-test/test_batch_wordcount.sh rename to flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh diff --git a/test-infra/end-to-end-test/test_hadoop_free.sh b/flink-end-to-end-tests/test-scripts/test_hadoop_free.sh similarity index 100% rename from test-infra/end-to-end-test/test_hadoop_free.sh rename to flink-end-to-end-tests/test-scripts/test_hadoop_free.sh diff --git a/test-infra/end-to-end-test/test_shaded_hadoop_s3a.sh b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh similarity index 100% rename from test-infra/end-to-end-test/test_shaded_hadoop_s3a.sh rename to flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh diff --git a/test-infra/end-to-end-test/test_shaded_presto_s3.sh b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh similarity index 100% rename from test-infra/end-to-end-test/test_shaded_presto_s3.sh rename to flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh diff --git a/test-infra/end-to-end-test/test_streaming_classloader.sh b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh similarity index 100% rename from test-infra/end-to-end-test/test_streaming_classloader.sh rename to flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh diff --git a/test-infra/end-to-end-test/test_streaming_kafka010.sh b/flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh similarity index 100% rename from test-infra/end-to-end-test/test_streaming_kafka010.sh rename to flink-end-to-end-tests/test-scripts/test_streaming_kafka010.sh diff --git a/test-infra/end-to-end-test/test_streaming_python_wordcount.sh b/flink-end-to-end-tests/test-scripts/test_streaming_python_wordcount.sh similarity index 100% rename from test-infra/end-to-end-test/test_streaming_python_wordcount.sh rename to flink-end-to-end-tests/test-scripts/test_streaming_python_wordcount.sh diff --git a/pom.xml b/pom.xml index a140e6e3687a1a..40c0e2447d93d0 100644 --- a/pom.xml +++ b/pom.xml @@ -1017,7 +1017,7 @@ under the License. out/test/flink-avro/avro/user.avsc flink-libraries/flink-table/src/test/scala/resources/*.out flink-yarn/src/test/resources/krb5.keytab - test-infra/end-to-end-test/test-data/* + flink-end-to-end-tests/test-scripts/test-data/* **/src/test/resources/*-snapshot diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh index 4b1c2e31ad5da5..dc1125d5296853 100755 --- a/tools/travis_mvn_watchdog.sh +++ b/tools/travis_mvn_watchdog.sh @@ -576,61 +576,9 @@ case $TEST in printf "Running end-to-end tests\n" printf "==============================================================================\n" - if [ $EXIT_CODE == 0 ]; then - printf "\n==============================================================================\n" - printf "Running Wordcount end-to-end test\n" - printf "==============================================================================\n" - FLINK_DIR=build-target test-infra/end-to-end-test/test_batch_wordcount.sh - EXIT_CODE=$? - fi - - if [ $EXIT_CODE == 0 ]; then - printf "\n==============================================================================\n" - printf "Running Kafka end-to-end test\n" - printf "==============================================================================\n" - FLINK_DIR=build-target test-infra/end-to-end-test/test_streaming_kafka010.sh - EXIT_CODE=$? - fi - - if [ $EXIT_CODE == 0 ]; then - printf "\n==============================================================================\n" - printf "Running class loading end-to-end test\n" - printf "==============================================================================\n" - FLINK_DIR=build-target test-infra/end-to-end-test/test_streaming_classloader.sh - EXIT_CODE=$? - fi - - if [ $EXIT_CODE == 0 ]; then - printf "\n==============================================================================\n" - printf "Running Shaded Hadoop S3A end-to-end test\n" - printf "==============================================================================\n" - FLINK_DIR=build-target test-infra/end-to-end-test/test_shaded_hadoop_s3a.sh - EXIT_CODE=$? - fi - - if [ $EXIT_CODE == 0 ]; then - printf "\n==============================================================================\n" - printf "Running Shaded Presto S3 end-to-end test\n" - printf "==============================================================================\n" - FLINK_DIR=build-target test-infra/end-to-end-test/test_shaded_presto_s3.sh - EXIT_CODE=$? - fi - - if [ $EXIT_CODE == 0 ]; then - printf "\n==============================================================================\n" - printf "Running Hadoop-free Wordcount end-to-end test\n" - printf "==============================================================================\n" - FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_hadoop_free.sh - EXIT_CODE=$? - fi - - if [ $EXIT_CODE == 0 ]; then - printf "\n==============================================================================\n" - printf "Running Streaming Python Wordcount end-to-end test\n" - printf "==============================================================================\n" - FLINK_DIR=build-target test-infra/end-to-end-test/test_streaming_python_wordcount.sh - EXIT_CODE=$? - fi + FLINK_DIR=build-target flink-end-to-end-tests/run-pre-commit-tests.sh + + EXIT_CODE=$? else printf "\n==============================================================================\n" printf "Previous build failure detected, skipping end-to-end tests.\n" From 57ffddedd0e4edf3b7c55405aca7130f5a32fe4a Mon Sep 17 00:00:00 2001 From: zjureel Date: Thu, 21 Dec 2017 17:49:11 +0800 Subject: [PATCH 0077/2294] [FLINK-6352] [kafka] Support to set offset of Kafka with specific date --- .../kafka/FlinkKafkaConsumer010.java | 33 +- .../kafka/internal/Kafka010Fetcher.java | 9 +- .../internal/KafkaConsumerCallBridge010.java | 25 ++ .../connectors/kafka/Kafka010ITCase.java | 35 +- .../kafka/internal/Kafka010FetcherTest.java | 9 +- .../connectors/kafka/Kafka011ITCase.java | 34 ++ .../connectors/kafka/Kafka08ITCase.java | 4 +- .../kafka/FlinkKafkaConsumer09.java | 3 +- .../kafka/internal/Kafka09Fetcher.java | 8 +- .../internal/KafkaConsumerCallBridge.java | 3 + .../kafka/internal/KafkaConsumerThread.java | 3 + .../kafka/internal/Kafka09FetcherTest.java | 9 +- .../kafka/FlinkKafkaConsumerBase.java | 4 +- .../connectors/kafka/config/StartupMode.java | 3 + .../KafkaTopicPartitionStateSentinel.java | 3 + .../kafka/KafkaConsumerTestBase.java | 317 ++++++++++++++++-- 16 files changed, 466 insertions(+), 36 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java index 394d1239a92f8a..d5bd1653dc3f58 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java @@ -24,6 +24,7 @@ import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode; +import org.apache.flink.streaming.connectors.kafka.config.StartupMode; import org.apache.flink.streaming.connectors.kafka.internal.Kafka010Fetcher; import org.apache.flink.streaming.connectors.kafka.internal.Kafka010PartitionDiscoverer; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; @@ -37,11 +38,14 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.regex.Pattern; +import static org.apache.flink.util.Preconditions.checkArgument; + /** * The Flink Kafka Consumer is a streaming data source that pulls a parallel data stream from * Apache Kafka 0.10.x. The consumer can run in multiple parallel instances, each of which will pull @@ -64,6 +68,8 @@ public class FlinkKafkaConsumer010 extends FlinkKafkaConsumer09 { private static final long serialVersionUID = 2324564345203409112L; + private Date specificStartupDate = null; + // ------------------------------------------------------------------------ /** @@ -172,6 +178,30 @@ public FlinkKafkaConsumer010(Pattern subscriptionPattern, KeyedDeserializationSc super(subscriptionPattern, deserializer, props); } + + /** + * Specifies the consumer to start reading partitions from specific date. The specified date must before current timestamp. + * This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers. + * + *

    The consumer will look up the earliest offset whose timestamp is greater than or equal to the specific date from Kafka. + * If there's no such offset, the consumer will use the latest offset to read data from kafka. + * + *

    This method does not effect where partitions are read from when the consumer is restored + * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or + * savepoint, only the offsets in the restored state will be used. + * + * @return The consumer object, to allow function chaining. + */ + public FlinkKafkaConsumer010 setStartFromSpecificDate(Date date) { + Date now = new Date(); + checkArgument(null != date && date.getTime() <= now.getTime(), + "Startup time[" + date + "] must be before current time[" + now + "]."); + this.startupMode = StartupMode.SPECIFIC_TIMESTAMP; + this.specificStartupDate = date; + this.specificStartupOffsets = null; + return this; + } + @Override protected AbstractFetcher createFetcher( SourceContext sourceContext, @@ -203,7 +233,8 @@ public FlinkKafkaConsumer010(Pattern subscriptionPattern, KeyedDeserializationSc pollTimeout, runtimeContext.getMetricGroup(), consumerMetricGroup, - useMetrics); + useMetrics, + specificStartupDate); } @Override diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java index 9b9b217b205f07..9ed15fc272a591 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java @@ -32,6 +32,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.TopicPartition; +import java.util.Date; import java.util.Map; import java.util.Properties; @@ -60,7 +61,8 @@ public Kafka010Fetcher( long pollTimeout, MetricGroup subtaskMetricGroup, MetricGroup consumerMetricGroup, - boolean useMetrics) throws Exception { + boolean useMetrics, + Date startupDate) throws Exception { super( sourceContext, assignedPartitionsWithInitialOffsets, @@ -75,7 +77,8 @@ public Kafka010Fetcher( pollTimeout, subtaskMetricGroup, consumerMetricGroup, - useMetrics); + useMetrics, + startupDate); } @Override @@ -95,7 +98,7 @@ protected void emitRecord( */ @Override protected KafkaConsumerCallBridge010 createCallBridge() { - return new KafkaConsumerCallBridge010(); + return new KafkaConsumerCallBridge010(startupDate); } @Override diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java index 5815bfade38de0..2f430c5a1a7b15 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java @@ -21,10 +21,14 @@ import org.apache.flink.annotation.Internal; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.common.TopicPartition; import java.util.Collections; +import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * The ConsumerCallBridge simply calls the {@link KafkaConsumer#assign(java.util.Collection)} method. @@ -37,6 +41,13 @@ @Internal public class KafkaConsumerCallBridge010 extends KafkaConsumerCallBridge { + private Date startupDate; + + public KafkaConsumerCallBridge010(Date startupDate) { + super(); + this.startupDate = startupDate; + } + @Override public void assignPartitions(KafkaConsumer consumer, List topicPartitions) throws Exception { consumer.assign(topicPartitions); @@ -51,4 +62,18 @@ public void seekPartitionToBeginning(KafkaConsumer consumer, TopicPartitio public void seekPartitionToEnd(KafkaConsumer consumer, TopicPartition partition) { consumer.seekToEnd(Collections.singletonList(partition)); } + + @Override + public void seekPartitionToDate(KafkaConsumer consumer, TopicPartition partition) { + Map partitionTimestampMap = new HashMap<>(1); + partitionTimestampMap.put(partition, startupDate.getTime()); + + Map topicPartitionOffsetMap = consumer.offsetsForTimes(partitionTimestampMap); + OffsetAndTimestamp offsetAndTimestamp = null == topicPartitionOffsetMap ? null : topicPartitionOffsetMap.get(partition); + if (null == offsetAndTimestamp) { + seekPartitionToEnd(consumer, partition); + } else { + consumer.seek(partition, offsetAndTimestamp.offset()); + } + } } diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java index c2b3dfa1056bd8..f821bf66e193b6 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.serialization.TypeInformationSerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.GenericTypeInfo; import org.apache.flink.api.java.typeutils.TypeInfoParser; import org.apache.flink.core.memory.DataInputView; @@ -34,6 +35,8 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.streaming.api.operators.StreamSink; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.connectors.kafka.config.StartupMode; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; @@ -45,6 +48,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.Date; +import java.util.Map; /** * IT cases for Kafka 0.10 . @@ -155,6 +160,11 @@ public void testStartFromSpecificOffsets() throws Exception { runStartFromSpecificOffsets(); } + @Test(timeout = 60000) + public void testStartFromSpecificDate() throws Exception { + runStartFromSpecificDate(); + } + // --- offset committing --- @Test(timeout = 60000) @@ -254,6 +264,30 @@ public long extractTimestamp(Long element, long previousElementTimestamp) { deleteTestTopic(topic); } + @Override + protected void setKafkaConsumerOffset(final StartupMode startupMode, + final FlinkKafkaConsumerBase> consumer, + final Map specificStartupOffsets, + final Date specificStartupDate) { + switch (startupMode) { + case EARLIEST: + consumer.setStartFromEarliest(); + break; + case LATEST: + consumer.setStartFromLatest(); + break; + case SPECIFIC_OFFSETS: + consumer.setStartFromSpecificOffsets(specificStartupOffsets); + break; + case GROUP_OFFSETS: + consumer.setStartFromGroupOffsets(); + break; + case SPECIFIC_TIMESTAMP: + ((FlinkKafkaConsumer010>) consumer).setStartFromSpecificDate(specificStartupDate); + break; + } + } + private static class TimestampValidatingOperator extends StreamSink { private static final long serialVersionUID = 1353168781235526806L; @@ -339,5 +373,4 @@ public boolean isEndOfStream(Long nextElement) { return cnt > 1000L; } } - } diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java index f57fbea67e1472..accf487dc439ef 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java @@ -131,7 +131,8 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false); + false, + null); // ----- run the fetcher ----- @@ -268,7 +269,8 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false); + false, + null); // ----- run the fetcher ----- @@ -383,7 +385,8 @@ public void testCancellationWhenEmitBlocks() throws Exception { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false); + false, + null); // ----- run the fetcher ----- diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java index 99d5b56d94bc16..1bbbdb4ab5a6aa 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java @@ -22,6 +22,7 @@ import org.apache.flink.api.common.serialization.TypeInformationSerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.GenericTypeInfo; import org.apache.flink.api.java.typeutils.TypeInfoParser; import org.apache.flink.core.memory.DataInputView; @@ -34,6 +35,8 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.streaming.api.operators.StreamSink; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.connectors.kafka.config.StartupMode; +import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; @@ -46,6 +49,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.Date; +import java.util.Map; import java.util.Optional; /** @@ -163,6 +168,11 @@ public void testStartFromSpecificOffsets() throws Exception { runStartFromSpecificOffsets(); } + @Test(timeout = 60000) + public void testStartFromSpecificDate() throws Exception { + runStartFromSpecificDate(); + } + // --- offset committing --- @Test(timeout = 60000) @@ -264,6 +274,30 @@ public long extractTimestamp(Long element, long previousElementTimestamp) { deleteTestTopic(topic); } + @Override + protected void setKafkaConsumerOffset(final StartupMode startupMode, + final FlinkKafkaConsumerBase> consumer, + final Map specificStartupOffsets, + final Date specificStartupDate) { + switch (startupMode) { + case EARLIEST: + consumer.setStartFromEarliest(); + break; + case LATEST: + consumer.setStartFromLatest(); + break; + case SPECIFIC_OFFSETS: + consumer.setStartFromSpecificOffsets(specificStartupOffsets); + break; + case GROUP_OFFSETS: + consumer.setStartFromGroupOffsets(); + break; + case SPECIFIC_TIMESTAMP: + ((FlinkKafkaConsumer011>) consumer).setStartFromSpecificDate(specificStartupDate); + break; + } + } + private static class TimestampValidatingOperator extends StreamSink { private static final long serialVersionUID = 1353168781235526806L; diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java index b3afa579aab6e3..6abccde9b4b833 100644 --- a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08ITCase.java @@ -99,7 +99,7 @@ public void testInvalidOffset() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); - readSequence(env, StartupMode.GROUP_OFFSETS, null, standardProps, parallelism, topic, valuesCount, startFrom); + readSequence(env, StartupMode.GROUP_OFFSETS, null, null, standardProps, parallelism, topic, valuesCount, startFrom); deleteTestTopic(topic); } @@ -212,7 +212,7 @@ public void testOffsetAutocommitTest() throws Exception { readProps.setProperty("auto.commit.interval.ms", "500"); // read so that the offset can be committed to ZK - readSequence(env, StartupMode.GROUP_OFFSETS, null, readProps, parallelism, topicName, 100, 0); + readSequence(env, StartupMode.GROUP_OFFSETS, null, null, readProps, parallelism, topicName, 100, 0); // get the offset CuratorFramework curatorFramework = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient(); diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java index 6e6051b35e40d5..4c3fa32cd11110 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java @@ -260,7 +260,8 @@ private FlinkKafkaConsumer09( pollTimeout, runtimeContext.getMetricGroup(), consumerMetricGroup, - useMetrics); + useMetrics, + null); } @Override diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java index dcc67d5b4fb140..185760bd680ad2 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java @@ -40,6 +40,7 @@ import javax.annotation.Nonnull; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -71,6 +72,9 @@ public class Kafka09Fetcher extends AbstractFetcher { /** Flag to mark the main work loop as alive. */ private volatile boolean running = true; + /** The specific date used to set the offset of kafka. */ + protected Date startupDate; + // ------------------------------------------------------------------------ public Kafka09Fetcher( @@ -87,7 +91,8 @@ public Kafka09Fetcher( long pollTimeout, MetricGroup subtaskMetricGroup, MetricGroup consumerMetricGroup, - boolean useMetrics) throws Exception { + boolean useMetrics, + Date startupDate) throws Exception { super( sourceContext, assignedPartitionsWithInitialOffsets, @@ -101,6 +106,7 @@ public Kafka09Fetcher( this.deserializer = deserializer; this.handover = new Handover(); + this.startupDate = startupDate; this.consumerThread = new KafkaConsumerThread( LOG, diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java index b789633333df20..ac8ef7a0c93f5e 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java @@ -50,4 +50,7 @@ public void seekPartitionToEnd(KafkaConsumer consumer, TopicPartition part consumer.seekToEnd(partition); } + public void seekPartitionToDate(KafkaConsumer consumer, TopicPartition partition) { + throw new RuntimeException("Seek offset from a specific date is only supported for version 0.10 and later."); + } } diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java index 38e8a41d474cfd..6f8c699f4fd8cc 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java @@ -423,6 +423,9 @@ void reassignPartitions(List> newPartit } else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.LATEST_OFFSET) { consumerCallBridge.seekPartitionToEnd(consumerTmp, newPartitionState.getKafkaPartitionHandle()); newPartitionState.setOffset(consumerTmp.position(newPartitionState.getKafkaPartitionHandle()) - 1); + } else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.TIMESTAMP) { + consumerCallBridge.seekPartitionToDate(consumerTmp, newPartitionState.getKafkaPartitionHandle()); + newPartitionState.setOffset(consumerTmp.position(newPartitionState.getKafkaPartitionHandle()) - 1); } else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.GROUP_OFFSET) { // the KafkaConsumer by default will automatically seek the consumer position // to the committed group offset, so we do not need to do it. diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java index 27b67f10984138..749c2dbdc4132f 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java @@ -131,7 +131,8 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false); + false, + null); // ----- run the fetcher ----- @@ -267,7 +268,8 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false); + false, + null); // ----- run the fetcher ----- @@ -382,7 +384,8 @@ public void testCancellationWhenEmitBlocks() throws Exception { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false); + false, + null); // ----- run the fetcher ----- diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index df35de61f03c52..94230172deec1a 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -147,10 +147,10 @@ public abstract class FlinkKafkaConsumerBase extends RichParallelSourceFuncti private final long discoveryIntervalMillis; /** The startup mode for the consumer (default is {@link StartupMode#GROUP_OFFSETS}). */ - private StartupMode startupMode = StartupMode.GROUP_OFFSETS; + protected StartupMode startupMode = StartupMode.GROUP_OFFSETS; /** Specific startup offsets; only relevant when startup mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ - private Map specificStartupOffsets; + protected Map specificStartupOffsets; // ------------------------------------------------------------------------ // runtime state (used individually by each parallel subtask) diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java index f984c8254abd6b..ec0a7e8dd0e300 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java @@ -35,6 +35,9 @@ public enum StartupMode { /** Start from the latest offset. */ LATEST(KafkaTopicPartitionStateSentinel.LATEST_OFFSET), + /** Start from specific timestamp. */ + SPECIFIC_TIMESTAMP(KafkaTopicPartitionStateSentinel.TIMESTAMP), + /** * Start from user-supplied specific offsets for each partition. * Since this mode will have specific offsets to start with, we do not need a sentinel value; diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java index 68f842ae9c522a..1503a99a233450 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java @@ -28,6 +28,9 @@ @Internal public class KafkaTopicPartitionStateSentinel { + /** Magic number that defines the partition should start from specify timestamp. */ + public static final long TIMESTAMP = -915623761777L; + /** Magic number that defines an unset offset. */ public static final long OFFSET_NOT_SET = -915623761776L; diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java index 55a9c4d3b14923..1e26d1f85aacbb 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java @@ -375,7 +375,7 @@ public void runStartFromEarliestOffsets() throws Exception { kafkaOffsetHandler.setCommittedOffset(topicName, 1, 31); kafkaOffsetHandler.setCommittedOffset(topicName, 2, 43); - readSequence(env, StartupMode.EARLIEST, null, readProps, parallelism, topicName, recordsInEachPartition, 0); + readSequence(env, StartupMode.EARLIEST, null, null, readProps, parallelism, topicName, recordsInEachPartition, 0); kafkaOffsetHandler.close(); deleteTestTopic(topicName); @@ -557,7 +557,7 @@ public void runStartFromGroupOffsets() throws Exception { partitionsToValueCountAndStartOffsets.put(1, new Tuple2<>(50, 0)); // partition 1 should read offset 0-49 partitionsToValueCountAndStartOffsets.put(2, new Tuple2<>(7, 43)); // partition 2 should read offset 43-49 - readSequence(env, StartupMode.GROUP_OFFSETS, null, readProps, topicName, partitionsToValueCountAndStartOffsets); + readSequence(env, StartupMode.GROUP_OFFSETS, null, null, readProps, topicName, partitionsToValueCountAndStartOffsets); kafkaOffsetHandler.close(); deleteTestTopic(topicName); @@ -621,12 +621,40 @@ public void runStartFromSpecificOffsets() throws Exception { partitionsToValueCountAndStartOffsets.put(2, new Tuple2<>(28, 22)); // partition 2 should read offset 22-49 partitionsToValueCountAndStartOffsets.put(3, new Tuple2<>(50, 0)); // partition 3 should read offset 0-49 - readSequence(env, StartupMode.SPECIFIC_OFFSETS, specificStartupOffsets, readProps, topicName, partitionsToValueCountAndStartOffsets); + readSequence(env, StartupMode.SPECIFIC_OFFSETS, specificStartupOffsets, null, readProps, topicName, partitionsToValueCountAndStartOffsets); kafkaOffsetHandler.close(); deleteTestTopic(topicName); } + /** + * This test ensures that the consumer correctly uses user-supplied specific date when explicitly configured to + * start from specific date. + * + *

    When configured to start from first start date, each partition should start from offset 0 and read 100 records. + * And when configured to start from second start date, each partition should start from 50 and read 50 records. + */ + protected void runStartFromSpecificDate() throws Exception { + // 4 partitions with 50 records each + final int parallelism = 4; + final int recordsInEachPartition = 50; + TopicWithStartDate topicWithStartDate = writeAppendSequence("runStartFromSpecificDate", recordsInEachPartition, parallelism, 1); + + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.getConfig().disableSysoutLogging(); + env.setParallelism(parallelism); + + Properties readProps = new Properties(); + readProps.putAll(standardProps); + + readSequence(env, StartupMode.SPECIFIC_TIMESTAMP, null, topicWithStartDate.getFirstStartDate(), + readProps, parallelism, topicWithStartDate.getTopicName(), recordsInEachPartition * 2, 0); + readSequence(env, StartupMode.SPECIFIC_TIMESTAMP, null, topicWithStartDate.getSecondStartDate(), + readProps, parallelism, topicWithStartDate.getTopicName(), recordsInEachPartition, recordsInEachPartition); + + deleteTestTopic(topicWithStartDate.getTopicName()); + } + /** * Ensure Kafka is working on both producer and consumer side. * This executes a job that contains two Flink pipelines. @@ -1712,6 +1740,7 @@ public TypeInformation> getProducedType() { protected void readSequence(final StreamExecutionEnvironment env, final StartupMode startupMode, final Map specificStartupOffsets, + final Date specificStartupDate, final Properties cc, final String topicName, final Map> partitionsToValuesCountAndStartOffset) throws Exception { @@ -1731,20 +1760,7 @@ protected void readSequence(final StreamExecutionEnvironment env, // create the consumer cc.putAll(secureProps); FlinkKafkaConsumerBase> consumer = kafkaServer.getConsumer(topicName, deser, cc); - switch (startupMode) { - case EARLIEST: - consumer.setStartFromEarliest(); - break; - case LATEST: - consumer.setStartFromLatest(); - break; - case SPECIFIC_OFFSETS: - consumer.setStartFromSpecificOffsets(specificStartupOffsets); - break; - case GROUP_OFFSETS: - consumer.setStartFromGroupOffsets(); - break; - } + setKafkaConsumerOffset(startupMode, consumer, specificStartupOffsets, specificStartupDate); DataStream> source = env .addSource(consumer).setParallelism(sourceParallelism) @@ -1808,12 +1824,13 @@ public void flatMap(Tuple2 value, Collector out) thro } /** - * Variant of {@link KafkaConsumerTestBase#readSequence(StreamExecutionEnvironment, StartupMode, Map, Properties, String, Map)} to + * Variant of {@link KafkaConsumerTestBase#readSequence(StreamExecutionEnvironment, StartupMode, Map, Date, Properties, String, Map)} to * expect reading from the same start offset and the same value count for all partitions of a single Kafka topic. */ protected void readSequence(final StreamExecutionEnvironment env, final StartupMode startupMode, final Map specificStartupOffsets, + final Date specificStartupDate, final Properties cc, final int sourceParallelism, final String topicName, @@ -1823,7 +1840,29 @@ protected void readSequence(final StreamExecutionEnvironment env, for (int i = 0; i < sourceParallelism; i++) { partitionsToValuesCountAndStartOffset.put(i, new Tuple2<>(valuesCount, startFrom)); } - readSequence(env, startupMode, specificStartupOffsets, cc, topicName, partitionsToValuesCountAndStartOffset); + readSequence(env, startupMode, specificStartupOffsets, specificStartupDate, cc, topicName, partitionsToValuesCountAndStartOffset); + } + + protected void setKafkaConsumerOffset(final StartupMode startupMode, + final FlinkKafkaConsumerBase> consumer, + final Map specificStartupOffsets, + final Date specificStartupDate) { + switch (startupMode) { + case EARLIEST: + consumer.setStartFromEarliest(); + break; + case LATEST: + consumer.setStartFromLatest(); + break; + case SPECIFIC_OFFSETS: + consumer.setStartFromSpecificOffsets(specificStartupOffsets); + break; + case GROUP_OFFSETS: + consumer.setStartFromGroupOffsets(); + break; + case SPECIFIC_TIMESTAMP: + throw new RuntimeException("Only support to start from specific start up date for version 0.10 and later"); + } } protected String writeSequence( @@ -1992,6 +2031,246 @@ public void run() { throw new Exception("Could not write a valid sequence to Kafka after " + maxNumAttempts + " attempts"); } + protected TopicWithStartDate writeAppendSequence( + String baseTopicName, + final int numElements, + final int parallelism, + final int replicationFactor) throws Exception { + LOG.info("\n===================================\n" + + "== Writing sequence of " + numElements + " into " + baseTopicName + " with p=" + parallelism + "\n" + + "==================================="); + + final TypeInformation> resultType = + TypeInformation.of(new TypeHint>() {}); + + final KeyedSerializationSchema> serSchema = + new KeyedSerializationSchemaWrapper<>( + new TypeInformationSerializationSchema<>(resultType, new ExecutionConfig())); + + final KeyedDeserializationSchema> deserSchema = + new KeyedDeserializationSchemaWrapper<>( + new TypeInformationSerializationSchema<>(resultType, new ExecutionConfig())); + + final int maxNumAttempts = 10; + + final List> kafkaTupleList = new ArrayList<>(); + for (int attempt = 1; attempt <= maxNumAttempts; attempt++) { + + final String topicName = baseTopicName + '-' + attempt; + + LOG.info("Writing attempt #1"); + + // -------- Write the Sequence -------- + + createTestTopic(topicName, parallelism, replicationFactor); + + Date firstStartDate = new Date(); + StreamExecutionEnvironment writeEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + writeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + writeEnv.getConfig().disableSysoutLogging(); + + DataStream> stream = writeEnv.addSource(new RichParallelSourceFunction>() { + + private boolean running = true; + + @Override + public void run(SourceContext> ctx) throws Exception { + int cnt = 0; + int partition = getRuntimeContext().getIndexOfThisSubtask(); + + while (running && cnt < numElements) { + ctx.collect(new Tuple2<>(partition, cnt)); + cnt++; + } + } + + @Override + public void cancel() { + running = false; + } + }).setParallelism(parallelism); + + // the producer must not produce duplicates + Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); + producerProperties.setProperty("retries", "0"); + producerProperties.putAll(secureProps); + + kafkaServer.produceIntoKafka(stream, topicName, serSchema, producerProperties, new Tuple2FlinkPartitioner(parallelism)) + .setParallelism(parallelism); + + try { + writeEnv.execute("Write sequence"); + } + catch (Exception e) { + LOG.error("Write attempt failed, trying again", e); + deleteTestTopic(topicName); + JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); + continue; + } + + Thread.sleep(10); + Date secondStartDate = new Date(); + writeEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + writeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + writeEnv.getConfig().disableSysoutLogging(); + + stream = writeEnv.addSource(new RichParallelSourceFunction>() { + + private boolean running = true; + + @Override + public void run(SourceContext> ctx) throws Exception { + int cnt = numElements; + int partition = getRuntimeContext().getIndexOfThisSubtask(); + + while (running && cnt < numElements + numElements) { + ctx.collect(new Tuple2<>(partition, cnt)); + cnt++; + } + } + + @Override + public void cancel() { + running = false; + } + }).setParallelism(parallelism); + + // the producer must not produce duplicates + producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); + producerProperties.setProperty("retries", "0"); + producerProperties.putAll(secureProps); + + kafkaServer.produceIntoKafka(stream, topicName, serSchema, producerProperties, new Tuple2FlinkPartitioner(parallelism)) + .setParallelism(parallelism); + + try { + writeEnv.execute("Write sequence"); + } + catch (Exception e) { + LOG.error("Write attempt failed, trying again", e); + deleteTestTopic(topicName); + JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); + continue; + } + + LOG.info("Finished writing sequence"); + + // -------- Validate the Sequence -------- + + // we need to validate the sequence, because kafka's producers are not exactly once + LOG.info("Validating sequence"); + + JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); + + final StreamExecutionEnvironment readEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + readEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + readEnv.getConfig().disableSysoutLogging(); + readEnv.setParallelism(parallelism); + + Properties readProps = (Properties) standardProps.clone(); + readProps.setProperty("group.id", "flink-tests-validator"); + readProps.putAll(secureProps); + FlinkKafkaConsumerBase> consumer = kafkaServer.getConsumer(topicName, deserSchema, readProps); + + readEnv + .addSource(consumer) + .map(new RichMapFunction, Tuple2>() { + + private final int totalCount = parallelism * (numElements + 1); + private int count = 0; + + @Override + public Tuple2 map(Tuple2 value) throws Exception { + if (++count == totalCount) { + throw new SuccessException(); + } else { + return value; + } + } + }).setParallelism(1) + .addSink(new DiscardingSink>()).setParallelism(1); + + final AtomicReference errorRef = new AtomicReference<>(); + + Thread runner = new Thread() { + @Override + public void run() { + try { + tryExecute(readEnv, "sequence validation"); + } catch (Throwable t) { + errorRef.set(t); + } + } + }; + runner.start(); + + final long deadline = System.nanoTime() + 10_000_000_000L; + long delay; + while (runner.isAlive() && (delay = deadline - System.nanoTime()) > 0) { + runner.join(delay / 1_000_000L); + } + + boolean success; + + if (runner.isAlive()) { + // did not finish in time, maybe the producer dropped one or more records and + // the validation did not reach the exit point + success = false; + JobManagerCommunicationUtils.cancelCurrentJob(flink.getLeaderGateway(timeout)); + } + else { + Throwable error = errorRef.get(); + if (error != null) { + success = false; + LOG.info("Attempt " + attempt + " failed with exception", error); + } + else { + success = true; + } + } + + JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); + + if (success) { + // everything is good! + return new TopicWithStartDate(topicName, firstStartDate, secondStartDate); + } + else { + deleteTestTopic(topicName); + // fall through the loop + } + } + + throw new Exception("Could not write a valid sequence to Kafka after " + maxNumAttempts + " attempts"); + } + + /** + * Pojo class for consumer with date. + */ + public static class TopicWithStartDate { + private final String topicName; + private final Date firstStartDate; + private final Date secondStartDate; + + public TopicWithStartDate(String topicName, Date firstStartDate, Date secondStartDate) { + this.topicName = topicName; + this.firstStartDate = firstStartDate; + this.secondStartDate = secondStartDate; + } + + public String getTopicName() { + return topicName; + } + + public Date getFirstStartDate() { + return firstStartDate; + } + + public Date getSecondStartDate() { + return secondStartDate; + } + } + // ------------------------------------------------------------------------ // Debugging utilities // ------------------------------------------------------------------------ From f8ca273549aded00c7cd12699cebc1f5bba83153 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 11 Jan 2018 14:26:37 +0800 Subject: [PATCH 0078/2294] [FLINK-6352] [kafka] Further improvements for timestamped-based startup mode 1) Eagerly deterrmin startup offsets when startup mode is TIMESTAMP 2) Remove usage of java Date in API to specify timestamp 3) Make tests more robust and flexible 4) Add documentation for the feature This closes #5282. --- docs/dev/connectors/kafka.md | 7 + .../kafka/FlinkKafkaConsumer010.java | 78 ++-- .../kafka/internal/Kafka010Fetcher.java | 9 +- .../internal/KafkaConsumerCallBridge010.java | 25 -- .../connectors/kafka/Kafka010ITCase.java | 33 +- .../kafka/internal/Kafka010FetcherTest.java | 9 +- .../connectors/kafka/Kafka011ITCase.java | 33 +- .../kafka/FlinkKafkaConsumer08.java | 8 + .../kafka/FlinkKafkaConsumer09.java | 11 +- .../kafka/internal/Kafka09Fetcher.java | 8 +- .../internal/KafkaConsumerCallBridge.java | 3 - .../kafka/internal/KafkaConsumerThread.java | 3 - .../kafka/internal/Kafka09FetcherTest.java | 9 +- .../kafka/FlinkKafkaConsumerBase.java | 128 +++++- .../connectors/kafka/config/StartupMode.java | 8 +- .../KafkaTopicPartitionStateSentinel.java | 3 - .../FlinkKafkaConsumerBaseMigrationTest.java | 7 + .../kafka/FlinkKafkaConsumerBaseTest.java | 10 +- .../kafka/KafkaConsumerTestBase.java | 420 ++++++------------ 19 files changed, 351 insertions(+), 461 deletions(-) diff --git a/docs/dev/connectors/kafka.md b/docs/dev/connectors/kafka.md index f28195c190f7c0..27fca7a2cd9165 100644 --- a/docs/dev/connectors/kafka.md +++ b/docs/dev/connectors/kafka.md @@ -191,6 +191,7 @@ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEn FlinkKafkaConsumer08 myConsumer = new FlinkKafkaConsumer08<>(...); myConsumer.setStartFromEarliest(); // start from the earliest record possible myConsumer.setStartFromLatest(); // start from the latest record +myConsumer.setStartFromTimestamp(...); // start from specified epoch timestamp (milliseconds) myConsumer.setStartFromGroupOffsets(); // the default behaviour DataStream stream = env.addSource(myConsumer); @@ -204,6 +205,7 @@ val env = StreamExecutionEnvironment.getExecutionEnvironment() val myConsumer = new FlinkKafkaConsumer08[String](...) myConsumer.setStartFromEarliest() // start from the earliest record possible myConsumer.setStartFromLatest() // start from the latest record +myConsumer.setStartFromTimestamp(...) // start from specified epoch timestamp (milliseconds) myConsumer.setStartFromGroupOffsets() // the default behaviour val stream = env.addSource(myConsumer) @@ -221,6 +223,11 @@ All versions of the Flink Kafka Consumer have the above explicit configuration m * `setStartFromEarliest()` / `setStartFromLatest()`: Start from the earliest / latest record. Under these modes, committed offsets in Kafka will be ignored and not used as starting positions. + * `setStartFromTimestamp(long)`: Start from the specified timestamp. For each partition, the record + whose timestamp is larger than or equal to the specified timestamp will be used as the start position. + If a partition's latest record is earlier than the timestamp, the partition will simply be read + from the latest record. Under this mode, committed offsets in Kafka will be ignored and not used as + starting positions. You can also specify the exact offsets the consumer should start from for each partition: diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java index d5bd1653dc3f58..3508d6de37efca 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java @@ -24,7 +24,6 @@ import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode; -import org.apache.flink.streaming.connectors.kafka.config.StartupMode; import org.apache.flink.streaming.connectors.kafka.internal.Kafka010Fetcher; import org.apache.flink.streaming.connectors.kafka.internal.Kafka010PartitionDiscoverer; import org.apache.flink.streaming.connectors.kafka.internals.AbstractFetcher; @@ -36,16 +35,18 @@ import org.apache.flink.util.SerializedValue; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.common.TopicPartition; +import java.util.Collection; import java.util.Collections; -import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.regex.Pattern; -import static org.apache.flink.util.Preconditions.checkArgument; - /** * The Flink Kafka Consumer is a streaming data source that pulls a parallel data stream from * Apache Kafka 0.10.x. The consumer can run in multiple parallel instances, each of which will pull @@ -68,8 +69,6 @@ public class FlinkKafkaConsumer010 extends FlinkKafkaConsumer09 { private static final long serialVersionUID = 2324564345203409112L; - private Date specificStartupDate = null; - // ------------------------------------------------------------------------ /** @@ -178,30 +177,6 @@ public FlinkKafkaConsumer010(Pattern subscriptionPattern, KeyedDeserializationSc super(subscriptionPattern, deserializer, props); } - - /** - * Specifies the consumer to start reading partitions from specific date. The specified date must before current timestamp. - * This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers. - * - *

    The consumer will look up the earliest offset whose timestamp is greater than or equal to the specific date from Kafka. - * If there's no such offset, the consumer will use the latest offset to read data from kafka. - * - *

    This method does not effect where partitions are read from when the consumer is restored - * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or - * savepoint, only the offsets in the restored state will be used. - * - * @return The consumer object, to allow function chaining. - */ - public FlinkKafkaConsumer010 setStartFromSpecificDate(Date date) { - Date now = new Date(); - checkArgument(null != date && date.getTime() <= now.getTime(), - "Startup time[" + date + "] must be before current time[" + now + "]."); - this.startupMode = StartupMode.SPECIFIC_TIMESTAMP; - this.specificStartupDate = date; - this.specificStartupOffsets = null; - return this; - } - @Override protected AbstractFetcher createFetcher( SourceContext sourceContext, @@ -233,8 +208,7 @@ public FlinkKafkaConsumer010 setStartFromSpecificDate(Date date) { pollTimeout, runtimeContext.getMetricGroup(), consumerMetricGroup, - useMetrics, - specificStartupDate); + useMetrics); } @Override @@ -245,4 +219,44 @@ protected AbstractPartitionDiscoverer createPartitionDiscoverer( return new Kafka010PartitionDiscoverer(topicsDescriptor, indexOfThisSubtask, numParallelSubtasks, properties); } + + // ------------------------------------------------------------------------ + // Timestamp-based startup + // ------------------------------------------------------------------------ + + @Override + public FlinkKafkaConsumerBase setStartFromTimestamp(long startupOffsetsTimestamp) { + // the purpose of this override is just to publicly expose the method for Kafka 0.10+; + // the base class doesn't publicly expose it since not all Kafka versions support the functionality + return super.setStartFromTimestamp(startupOffsetsTimestamp); + } + + @Override + protected Map fetchOffsetsWithTimestamp( + Collection partitions, + long timestamp) { + + Map partitionOffsetsRequest = new HashMap<>(partitions.size()); + for (KafkaTopicPartition partition : partitions) { + partitionOffsetsRequest.put( + new TopicPartition(partition.getTopic(), partition.getPartition()), + timestamp); + } + + // use a short-lived consumer to fetch the offsets; + // this is ok because this is a one-time operation that happens only on startup + KafkaConsumer consumer = new KafkaConsumer(properties); + + Map result = new HashMap<>(partitions.size()); + for (Map.Entry partitionToOffset : + consumer.offsetsForTimes(partitionOffsetsRequest).entrySet()) { + + result.put( + new KafkaTopicPartition(partitionToOffset.getKey().topic(), partitionToOffset.getKey().partition()), + (partitionToOffset.getValue() == null) ? null : partitionToOffset.getValue().offset()); + } + + consumer.close(); + return result; + } } diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java index 9ed15fc272a591..9b9b217b205f07 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java @@ -32,7 +32,6 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.TopicPartition; -import java.util.Date; import java.util.Map; import java.util.Properties; @@ -61,8 +60,7 @@ public Kafka010Fetcher( long pollTimeout, MetricGroup subtaskMetricGroup, MetricGroup consumerMetricGroup, - boolean useMetrics, - Date startupDate) throws Exception { + boolean useMetrics) throws Exception { super( sourceContext, assignedPartitionsWithInitialOffsets, @@ -77,8 +75,7 @@ public Kafka010Fetcher( pollTimeout, subtaskMetricGroup, consumerMetricGroup, - useMetrics, - startupDate); + useMetrics); } @Override @@ -98,7 +95,7 @@ protected void emitRecord( */ @Override protected KafkaConsumerCallBridge010 createCallBridge() { - return new KafkaConsumerCallBridge010(startupDate); + return new KafkaConsumerCallBridge010(); } @Override diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java index 2f430c5a1a7b15..5815bfade38de0 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge010.java @@ -21,14 +21,10 @@ import org.apache.flink.annotation.Internal; import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.common.TopicPartition; import java.util.Collections; -import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * The ConsumerCallBridge simply calls the {@link KafkaConsumer#assign(java.util.Collection)} method. @@ -41,13 +37,6 @@ @Internal public class KafkaConsumerCallBridge010 extends KafkaConsumerCallBridge { - private Date startupDate; - - public KafkaConsumerCallBridge010(Date startupDate) { - super(); - this.startupDate = startupDate; - } - @Override public void assignPartitions(KafkaConsumer consumer, List topicPartitions) throws Exception { consumer.assign(topicPartitions); @@ -62,18 +51,4 @@ public void seekPartitionToBeginning(KafkaConsumer consumer, TopicPartitio public void seekPartitionToEnd(KafkaConsumer consumer, TopicPartition partition) { consumer.seekToEnd(Collections.singletonList(partition)); } - - @Override - public void seekPartitionToDate(KafkaConsumer consumer, TopicPartition partition) { - Map partitionTimestampMap = new HashMap<>(1); - partitionTimestampMap.put(partition, startupDate.getTime()); - - Map topicPartitionOffsetMap = consumer.offsetsForTimes(partitionTimestampMap); - OffsetAndTimestamp offsetAndTimestamp = null == topicPartitionOffsetMap ? null : topicPartitionOffsetMap.get(partition); - if (null == offsetAndTimestamp) { - seekPartitionToEnd(consumer, partition); - } else { - consumer.seek(partition, offsetAndTimestamp.offset()); - } - } } diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java index f821bf66e193b6..06f627d7f425a1 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010ITCase.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.serialization.TypeInformationSerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.GenericTypeInfo; import org.apache.flink.api.java.typeutils.TypeInfoParser; import org.apache.flink.core.memory.DataInputView; @@ -35,8 +34,6 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.streaming.api.operators.StreamSink; import org.apache.flink.streaming.api.watermark.Watermark; -import org.apache.flink.streaming.connectors.kafka.config.StartupMode; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; @@ -48,8 +45,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.Date; -import java.util.Map; /** * IT cases for Kafka 0.10 . @@ -161,8 +156,8 @@ public void testStartFromSpecificOffsets() throws Exception { } @Test(timeout = 60000) - public void testStartFromSpecificDate() throws Exception { - runStartFromSpecificDate(); + public void testStartFromTimestamp() throws Exception { + runStartFromTimestamp(); } // --- offset committing --- @@ -264,30 +259,6 @@ public long extractTimestamp(Long element, long previousElementTimestamp) { deleteTestTopic(topic); } - @Override - protected void setKafkaConsumerOffset(final StartupMode startupMode, - final FlinkKafkaConsumerBase> consumer, - final Map specificStartupOffsets, - final Date specificStartupDate) { - switch (startupMode) { - case EARLIEST: - consumer.setStartFromEarliest(); - break; - case LATEST: - consumer.setStartFromLatest(); - break; - case SPECIFIC_OFFSETS: - consumer.setStartFromSpecificOffsets(specificStartupOffsets); - break; - case GROUP_OFFSETS: - consumer.setStartFromGroupOffsets(); - break; - case SPECIFIC_TIMESTAMP: - ((FlinkKafkaConsumer010>) consumer).setStartFromSpecificDate(specificStartupDate); - break; - } - } - private static class TimestampValidatingOperator extends StreamSink { private static final long serialVersionUID = 1353168781235526806L; diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java index accf487dc439ef..f57fbea67e1472 100644 --- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java +++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java @@ -131,8 +131,7 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false, - null); + false); // ----- run the fetcher ----- @@ -269,8 +268,7 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false, - null); + false); // ----- run the fetcher ----- @@ -385,8 +383,7 @@ public void testCancellationWhenEmitBlocks() throws Exception { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false, - null); + false); // ----- run the fetcher ----- diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java index 1bbbdb4ab5a6aa..fd6eb617a0df30 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011ITCase.java @@ -22,7 +22,6 @@ import org.apache.flink.api.common.serialization.TypeInformationSerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.GenericTypeInfo; import org.apache.flink.api.java.typeutils.TypeInfoParser; import org.apache.flink.core.memory.DataInputView; @@ -35,8 +34,6 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.streaming.api.operators.StreamSink; import org.apache.flink.streaming.api.watermark.Watermark; -import org.apache.flink.streaming.connectors.kafka.config.StartupMode; -import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; @@ -49,8 +46,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.Date; -import java.util.Map; import java.util.Optional; /** @@ -169,8 +164,8 @@ public void testStartFromSpecificOffsets() throws Exception { } @Test(timeout = 60000) - public void testStartFromSpecificDate() throws Exception { - runStartFromSpecificDate(); + public void testStartFromTimestamp() throws Exception { + runStartFromTimestamp(); } // --- offset committing --- @@ -274,30 +269,6 @@ public long extractTimestamp(Long element, long previousElementTimestamp) { deleteTestTopic(topic); } - @Override - protected void setKafkaConsumerOffset(final StartupMode startupMode, - final FlinkKafkaConsumerBase> consumer, - final Map specificStartupOffsets, - final Date specificStartupDate) { - switch (startupMode) { - case EARLIEST: - consumer.setStartFromEarliest(); - break; - case LATEST: - consumer.setStartFromLatest(); - break; - case SPECIFIC_OFFSETS: - consumer.setStartFromSpecificOffsets(specificStartupOffsets); - break; - case GROUP_OFFSETS: - consumer.setStartFromGroupOffsets(); - break; - case SPECIFIC_TIMESTAMP: - ((FlinkKafkaConsumer011>) consumer).setStartFromSpecificDate(specificStartupDate); - break; - } - } - private static class TimestampValidatingOperator extends StreamSink { private static final long serialVersionUID = 1353168781235526806L; diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java index 312c8c46403ab7..86f69cd3f6e2b4 100644 --- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java +++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java @@ -37,6 +37,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -274,6 +275,13 @@ protected boolean getIsAutoCommitEnabled() { PropertiesUtil.getLong(kafkaProperties, "auto.commit.interval.ms", 60000) > 0; } + @Override + protected Map fetchOffsetsWithTimestamp(Collection partitions, long timestamp) { + // this should not be reached, since we do not expose the timestamp-based startup feature in version 0.8. + throw new UnsupportedOperationException( + "Fetching partition offsets using timestamps is only supported in Kafka versions 0.10 and above."); + } + // ------------------------------------------------------------------------ // Kafka / ZooKeeper configuration utilities // ------------------------------------------------------------------------ diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java index 4c3fa32cd11110..497003293415f4 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java @@ -40,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -260,8 +261,7 @@ private FlinkKafkaConsumer09( pollTimeout, runtimeContext.getMetricGroup(), consumerMetricGroup, - useMetrics, - null); + useMetrics); } @Override @@ -279,6 +279,13 @@ protected boolean getIsAutoCommitEnabled() { PropertiesUtil.getLong(properties, ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 5000) > 0; } + @Override + protected Map fetchOffsetsWithTimestamp(Collection partitions, long timestamp) { + // this should not be reached, since we do not expose the timestamp-based startup feature in version 0.9. + throw new UnsupportedOperationException( + "Fetching partition offsets using timestamps is only supported in Kafka versions 0.10 and above."); + } + // ------------------------------------------------------------------------ // Utilities // ------------------------------------------------------------------------ diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java index 185760bd680ad2..dcc67d5b4fb140 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java @@ -40,7 +40,6 @@ import javax.annotation.Nonnull; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -72,9 +71,6 @@ public class Kafka09Fetcher extends AbstractFetcher { /** Flag to mark the main work loop as alive. */ private volatile boolean running = true; - /** The specific date used to set the offset of kafka. */ - protected Date startupDate; - // ------------------------------------------------------------------------ public Kafka09Fetcher( @@ -91,8 +87,7 @@ public Kafka09Fetcher( long pollTimeout, MetricGroup subtaskMetricGroup, MetricGroup consumerMetricGroup, - boolean useMetrics, - Date startupDate) throws Exception { + boolean useMetrics) throws Exception { super( sourceContext, assignedPartitionsWithInitialOffsets, @@ -106,7 +101,6 @@ public Kafka09Fetcher( this.deserializer = deserializer; this.handover = new Handover(); - this.startupDate = startupDate; this.consumerThread = new KafkaConsumerThread( LOG, diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java index ac8ef7a0c93f5e..b789633333df20 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerCallBridge.java @@ -50,7 +50,4 @@ public void seekPartitionToEnd(KafkaConsumer consumer, TopicPartition part consumer.seekToEnd(partition); } - public void seekPartitionToDate(KafkaConsumer consumer, TopicPartition partition) { - throw new RuntimeException("Seek offset from a specific date is only supported for version 0.10 and later."); - } } diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java index 6f8c699f4fd8cc..38e8a41d474cfd 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java @@ -423,9 +423,6 @@ void reassignPartitions(List> newPartit } else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.LATEST_OFFSET) { consumerCallBridge.seekPartitionToEnd(consumerTmp, newPartitionState.getKafkaPartitionHandle()); newPartitionState.setOffset(consumerTmp.position(newPartitionState.getKafkaPartitionHandle()) - 1); - } else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.TIMESTAMP) { - consumerCallBridge.seekPartitionToDate(consumerTmp, newPartitionState.getKafkaPartitionHandle()); - newPartitionState.setOffset(consumerTmp.position(newPartitionState.getKafkaPartitionHandle()) - 1); } else if (newPartitionState.getOffset() == KafkaTopicPartitionStateSentinel.GROUP_OFFSET) { // the KafkaConsumer by default will automatically seek the consumer position // to the committed group offset, so we do not need to do it. diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java index 749c2dbdc4132f..27b67f10984138 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java @@ -131,8 +131,7 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false, - null); + false); // ----- run the fetcher ----- @@ -268,8 +267,7 @@ public Void answer(InvocationOnMock invocation) { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false, - null); + false); // ----- run the fetcher ----- @@ -384,8 +382,7 @@ public void testCancellationWhenEmitBlocks() throws Exception { 0L, new UnregisteredMetricsGroup(), new UnregisteredMetricsGroup(), - false, - null); + false); // ----- run the fetcher ----- diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index 94230172deec1a..e19772a1f801c9 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -57,6 +57,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -147,10 +148,13 @@ public abstract class FlinkKafkaConsumerBase extends RichParallelSourceFuncti private final long discoveryIntervalMillis; /** The startup mode for the consumer (default is {@link StartupMode#GROUP_OFFSETS}). */ - protected StartupMode startupMode = StartupMode.GROUP_OFFSETS; + private StartupMode startupMode = StartupMode.GROUP_OFFSETS; /** Specific startup offsets; only relevant when startup mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ - protected Map specificStartupOffsets; + private Map specificStartupOffsets; + + /** Timestamp to determine startup offsets; only relevant when startup mode is {@link StartupMode#TIMESTAMP}. */ + private Long startupOffsetsTimestamp; // ------------------------------------------------------------------------ // runtime state (used individually by each parallel subtask) @@ -335,7 +339,7 @@ public FlinkKafkaConsumerBase setCommitOffsetsOnCheckpoints(boolean commitOnC * Specifies the consumer to start reading from the earliest offset for all partitions. * This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers. * - *

    This method does not effect where partitions are read from when the consumer is restored + *

    This method does not affect where partitions are read from when the consumer is restored * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or * savepoint, only the offsets in the restored state will be used. * @@ -343,6 +347,7 @@ public FlinkKafkaConsumerBase setCommitOffsetsOnCheckpoints(boolean commitOnC */ public FlinkKafkaConsumerBase setStartFromEarliest() { this.startupMode = StartupMode.EARLIEST; + this.startupOffsetsTimestamp = null; this.specificStartupOffsets = null; return this; } @@ -351,7 +356,7 @@ public FlinkKafkaConsumerBase setStartFromEarliest() { * Specifies the consumer to start reading from the latest offset for all partitions. * This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers. * - *

    This method does not effect where partitions are read from when the consumer is restored + *

    This method does not affect where partitions are read from when the consumer is restored * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or * savepoint, only the offsets in the restored state will be used. * @@ -359,6 +364,41 @@ public FlinkKafkaConsumerBase setStartFromEarliest() { */ public FlinkKafkaConsumerBase setStartFromLatest() { this.startupMode = StartupMode.LATEST; + this.startupOffsetsTimestamp = null; + this.specificStartupOffsets = null; + return this; + } + + /** + * Specifies the consumer to start reading partitions from a specified timestamp. + * The specified timestamp must be before the current timestamp. + * This lets the consumer ignore any committed group offsets in Zookeeper / Kafka brokers. + * + *

    The consumer will look up the earliest offset whose timestamp is greater than or equal + * to the specific timestamp from Kafka. If there's no such offset, the consumer will use the + * latest offset to read data from kafka. + * + *

    This method does not affect where partitions are read from when the consumer is restored + * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or + * savepoint, only the offsets in the restored state will be used. + * + * @param startupOffsetsTimestamp timestamp for the startup offsets, as milliseconds from epoch. + * + * @return The consumer object, to allow function chaining. + */ + // NOTE - + // This method is implemented in the base class because this is where the startup logging and verifications live. + // However, it is not publicly exposed since only newer Kafka versions support the functionality. + // Version-specific subclasses which can expose the functionality should override and allow public access. + protected FlinkKafkaConsumerBase setStartFromTimestamp(long startupOffsetsTimestamp) { + checkArgument(startupOffsetsTimestamp >= 0, "The provided value for the startup offsets timestamp is invalid."); + + long currentTimestamp = System.currentTimeMillis(); + checkArgument(startupOffsetsTimestamp <= currentTimestamp, + "Startup time[%s] must be before current time[%s].", startupOffsetsTimestamp, currentTimestamp); + + this.startupMode = StartupMode.TIMESTAMP; + this.startupOffsetsTimestamp = startupOffsetsTimestamp; this.specificStartupOffsets = null; return this; } @@ -369,7 +409,7 @@ public FlinkKafkaConsumerBase setStartFromLatest() { * properties. If no offset can be found for a partition, the behaviour in "auto.offset.reset" * set in the configuration properties will be used for the partition. * - *

    This method does not effect where partitions are read from when the consumer is restored + *

    This method does not affect where partitions are read from when the consumer is restored * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or * savepoint, only the offsets in the restored state will be used. * @@ -377,6 +417,7 @@ public FlinkKafkaConsumerBase setStartFromLatest() { */ public FlinkKafkaConsumerBase setStartFromGroupOffsets() { this.startupMode = StartupMode.GROUP_OFFSETS; + this.startupOffsetsTimestamp = null; this.specificStartupOffsets = null; return this; } @@ -395,7 +436,7 @@ public FlinkKafkaConsumerBase setStartFromGroupOffsets() { * offsets but still no group offset could be found for it, then the "auto.offset.reset" behaviour set in the * configuration properties will be used for the partition * - *

    This method does not effect where partitions are read from when the consumer is restored + *

    This method does not affect where partitions are read from when the consumer is restored * from a checkpoint or savepoint. When the consumer is restored from a checkpoint or * savepoint, only the offsets in the restored state will be used. * @@ -403,6 +444,7 @@ public FlinkKafkaConsumerBase setStartFromGroupOffsets() { */ public FlinkKafkaConsumerBase setStartFromSpecificOffsets(Map specificStartupOffsets) { this.startupMode = StartupMode.SPECIFIC_OFFSETS; + this.startupOffsetsTimestamp = null; this.specificStartupOffsets = checkNotNull(specificStartupOffsets); return this; } @@ -457,28 +499,57 @@ public void open(Configuration configuration) throws Exception { getRuntimeContext().getIndexOfThisSubtask(), subscribedPartitionsToStartOffsets.size(), subscribedPartitionsToStartOffsets); } else { // use the partition discoverer to fetch the initial seed partitions, - // and set their initial offsets depending on the startup mode - for (KafkaTopicPartition seedPartition : allPartitions) { - if (startupMode != StartupMode.SPECIFIC_OFFSETS) { - subscribedPartitionsToStartOffsets.put(seedPartition, startupMode.getStateSentinel()); - } else { + // and set their initial offsets depending on the startup mode. + // for SPECIFIC_OFFSETS and TIMESTAMP modes, we set the specific offsets now; + // for other modes (EARLIEST, LATEST, and GROUP_OFFSETS), the offset is lazily determined + // when the partition is actually read. + switch (startupMode) { + case SPECIFIC_OFFSETS: if (specificStartupOffsets == null) { - throw new IllegalArgumentException( + throw new IllegalStateException( "Startup mode for the consumer set to " + StartupMode.SPECIFIC_OFFSETS + - ", but no specific offsets were specified"); + ", but no specific offsets were specified."); } - Long specificOffset = specificStartupOffsets.get(seedPartition); - if (specificOffset != null) { - // since the specified offsets represent the next record to read, we subtract - // it by one so that the initial state of the consumer will be correct - subscribedPartitionsToStartOffsets.put(seedPartition, specificOffset - 1); - } else { - // default to group offset behaviour if the user-provided specific offsets - // do not contain a value for this partition - subscribedPartitionsToStartOffsets.put(seedPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET); + for (KafkaTopicPartition seedPartition : allPartitions) { + Long specificOffset = specificStartupOffsets.get(seedPartition); + if (specificOffset != null) { + // since the specified offsets represent the next record to read, we subtract + // it by one so that the initial state of the consumer will be correct + subscribedPartitionsToStartOffsets.put(seedPartition, specificOffset - 1); + } else { + // default to group offset behaviour if the user-provided specific offsets + // do not contain a value for this partition + subscribedPartitionsToStartOffsets.put(seedPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET); + } + } + + break; + case TIMESTAMP: + if (startupOffsetsTimestamp == null) { + throw new IllegalStateException( + "Startup mode for the consumer set to " + StartupMode.TIMESTAMP + + ", but no startup timestamp was specified."); + } + + for (Map.Entry partitionToOffset + : fetchOffsetsWithTimestamp(allPartitions, startupOffsetsTimestamp).entrySet()) { + subscribedPartitionsToStartOffsets.put( + partitionToOffset.getKey(), + (partitionToOffset.getValue() == null) + // if an offset cannot be retrieved for a partition with the given timestamp, + // we default to using the latest offset for the partition + ? KafkaTopicPartitionStateSentinel.LATEST_OFFSET + // since the specified offsets represent the next record to read, we subtract + // it by one so that the initial state of the consumer will be correct + : partitionToOffset.getValue() - 1); + } + + break; + default: + for (KafkaTopicPartition seedPartition : allPartitions) { + subscribedPartitionsToStartOffsets.put(seedPartition, startupMode.getStateSentinel()); } - } } if (!subscribedPartitionsToStartOffsets.isEmpty()) { @@ -495,6 +566,13 @@ public void open(Configuration configuration) throws Exception { subscribedPartitionsToStartOffsets.size(), subscribedPartitionsToStartOffsets.keySet()); break; + case TIMESTAMP: + LOG.info("Consumer subtask {} will start reading the following {} partitions from timestamp {}: {}", + getRuntimeContext().getIndexOfThisSubtask(), + subscribedPartitionsToStartOffsets.size(), + startupOffsetsTimestamp, + subscribedPartitionsToStartOffsets.keySet()); + break; case SPECIFIC_OFFSETS: LOG.info("Consumer subtask {} will start reading the following {} partitions from the specified startup offsets {}: {}", getRuntimeContext().getIndexOfThisSubtask(), @@ -873,6 +951,10 @@ protected abstract AbstractPartitionDiscoverer createPartitionDiscoverer( protected abstract boolean getIsAutoCommitEnabled(); + protected abstract Map fetchOffsetsWithTimestamp( + Collection partitions, + long timestamp); + // ------------------------------------------------------------------------ // ResultTypeQueryable methods // ------------------------------------------------------------------------ diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java index ec0a7e8dd0e300..d417fdcfdf4553 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/config/StartupMode.java @@ -35,8 +35,12 @@ public enum StartupMode { /** Start from the latest offset. */ LATEST(KafkaTopicPartitionStateSentinel.LATEST_OFFSET), - /** Start from specific timestamp. */ - SPECIFIC_TIMESTAMP(KafkaTopicPartitionStateSentinel.TIMESTAMP), + /** + * Start from user-supplied timestamp for each partition. + * Since this mode will have specific offsets to start with, we do not need a sentinel value; + * using Long.MIN_VALUE as a placeholder. + */ + TIMESTAMP(Long.MIN_VALUE), /** * Start from user-supplied specific offsets for each partition. diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java index 1503a99a233450..68f842ae9c522a 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/KafkaTopicPartitionStateSentinel.java @@ -28,9 +28,6 @@ @Internal public class KafkaTopicPartitionStateSentinel { - /** Magic number that defines the partition should start from specify timestamp. */ - public static final long TIMESTAMP = -915623761777L; - /** Magic number that defines an unset offset. */ public static final long OFFSET_NOT_SET = -915623761776L; diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java index d9a9e41cdbc3c1..768ac16547c422 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java @@ -419,6 +419,13 @@ protected AbstractPartitionDiscoverer createPartitionDiscoverer( protected boolean getIsAutoCommitEnabled() { return false; } + + @Override + protected Map fetchOffsetsWithTimestamp( + Collection partitions, + long timestamp) { + throw new UnsupportedOperationException(); + } } private abstract static class DummySourceContext diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java index 263ed8eca7c66a..b226ff1360a600 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java @@ -67,6 +67,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -674,7 +675,14 @@ protected AbstractPartitionDiscoverer createPartitionDiscoverer( @Override protected boolean getIsAutoCommitEnabled() { - return this.isAutoCommitEnabled; + return isAutoCommitEnabled; + } + + @Override + protected Map fetchOffsetsWithTimestamp( + Collection partitions, + long timestamp) { + throw new UnsupportedOperationException(); } } diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java index 1e26d1f85aacbb..f07c0bb48a562a 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerTestBase.java @@ -628,17 +628,32 @@ public void runStartFromSpecificOffsets() throws Exception { } /** - * This test ensures that the consumer correctly uses user-supplied specific date when explicitly configured to - * start from specific date. + * This test ensures that the consumer correctly uses user-supplied timestamp when explicitly configured to + * start from timestamp. * - *

    When configured to start from first start date, each partition should start from offset 0 and read 100 records. - * And when configured to start from second start date, each partition should start from 50 and read 50 records. + *

    The validated Kafka data is written in 2 steps: first, an initial 50 records is written to each partition. + * After that, another 30 records is appended to each partition. Before each step, a timestamp is recorded. + * For the validation, when the read job is configured to start from the first timestamp, each partition should start + * from offset 0 and read a total of 80 records. When configured to start from the second timestamp, + * each partition should start from offset 50 and read on the remaining 30 appended records. */ - protected void runStartFromSpecificDate() throws Exception { + public void runStartFromTimestamp() throws Exception { // 4 partitions with 50 records each final int parallelism = 4; - final int recordsInEachPartition = 50; - TopicWithStartDate topicWithStartDate = writeAppendSequence("runStartFromSpecificDate", recordsInEachPartition, parallelism, 1); + final int initialRecordsInEachPartition = 50; + final int appendRecordsInEachPartition = 30; + + // attempt to create an appended test sequence, where the timestamp of writing the appended sequence + // is assured to be larger than the timestamp of the original sequence. + long firstTimestamp = System.currentTimeMillis(); + String topic = writeSequence("runStartFromTimestamp", initialRecordsInEachPartition, parallelism, 1); + + long secondTimestamp = 0; + while (secondTimestamp <= firstTimestamp) { + Thread.sleep(1000); + secondTimestamp = System.currentTimeMillis(); + } + writeAppendSequence(topic, initialRecordsInEachPartition, appendRecordsInEachPartition, parallelism); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); @@ -647,12 +662,12 @@ protected void runStartFromSpecificDate() throws Exception { Properties readProps = new Properties(); readProps.putAll(standardProps); - readSequence(env, StartupMode.SPECIFIC_TIMESTAMP, null, topicWithStartDate.getFirstStartDate(), - readProps, parallelism, topicWithStartDate.getTopicName(), recordsInEachPartition * 2, 0); - readSequence(env, StartupMode.SPECIFIC_TIMESTAMP, null, topicWithStartDate.getSecondStartDate(), - readProps, parallelism, topicWithStartDate.getTopicName(), recordsInEachPartition, recordsInEachPartition); + readSequence(env, StartupMode.TIMESTAMP, null, firstTimestamp, + readProps, parallelism, topic, initialRecordsInEachPartition + appendRecordsInEachPartition, 0); + readSequence(env, StartupMode.TIMESTAMP, null, secondTimestamp, + readProps, parallelism, topic, appendRecordsInEachPartition, initialRecordsInEachPartition); - deleteTestTopic(topicWithStartDate.getTopicName()); + deleteTestTopic(topic); } /** @@ -1740,7 +1755,7 @@ public TypeInformation> getProducedType() { protected void readSequence(final StreamExecutionEnvironment env, final StartupMode startupMode, final Map specificStartupOffsets, - final Date specificStartupDate, + final Long startupTimestamp, final Properties cc, final String topicName, final Map> partitionsToValuesCountAndStartOffset) throws Exception { @@ -1760,7 +1775,7 @@ protected void readSequence(final StreamExecutionEnvironment env, // create the consumer cc.putAll(secureProps); FlinkKafkaConsumerBase> consumer = kafkaServer.getConsumer(topicName, deser, cc); - setKafkaConsumerOffset(startupMode, consumer, specificStartupOffsets, specificStartupDate); + setKafkaConsumerOffset(startupMode, consumer, specificStartupOffsets, startupTimestamp); DataStream> source = env .addSource(consumer).setParallelism(sourceParallelism) @@ -1824,13 +1839,13 @@ public void flatMap(Tuple2 value, Collector out) thro } /** - * Variant of {@link KafkaConsumerTestBase#readSequence(StreamExecutionEnvironment, StartupMode, Map, Date, Properties, String, Map)} to + * Variant of {@link KafkaConsumerTestBase#readSequence(StreamExecutionEnvironment, StartupMode, Map, Long, Properties, String, Map)} to * expect reading from the same start offset and the same value count for all partitions of a single Kafka topic. */ protected void readSequence(final StreamExecutionEnvironment env, final StartupMode startupMode, final Map specificStartupOffsets, - final Date specificStartupDate, + final Long startupTimestamp, final Properties cc, final int sourceParallelism, final String topicName, @@ -1840,13 +1855,13 @@ protected void readSequence(final StreamExecutionEnvironment env, for (int i = 0; i < sourceParallelism; i++) { partitionsToValuesCountAndStartOffset.put(i, new Tuple2<>(valuesCount, startFrom)); } - readSequence(env, startupMode, specificStartupOffsets, specificStartupDate, cc, topicName, partitionsToValuesCountAndStartOffset); + readSequence(env, startupMode, specificStartupOffsets, startupTimestamp, cc, topicName, partitionsToValuesCountAndStartOffset); } protected void setKafkaConsumerOffset(final StartupMode startupMode, final FlinkKafkaConsumerBase> consumer, final Map specificStartupOffsets, - final Date specificStartupDate) { + final Long startupTimestamp) { switch (startupMode) { case EARLIEST: consumer.setStartFromEarliest(); @@ -1860,8 +1875,9 @@ protected void setKafkaConsumerOffset(final StartupMode startupMode, case GROUP_OFFSETS: consumer.setStartFromGroupOffsets(); break; - case SPECIFIC_TIMESTAMP: - throw new RuntimeException("Only support to start from specific start up date for version 0.10 and later"); + case TIMESTAMP: + consumer.setStartFromTimestamp(startupTimestamp); + break; } } @@ -1891,7 +1907,7 @@ protected String writeSequence( final String topicName = baseTopicName + '-' + attempt; - LOG.info("Writing attempt #1"); + LOG.info("Writing attempt #" + attempt); // -------- Write the Sequence -------- @@ -1949,76 +1965,7 @@ public void cancel() { JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); - final StreamExecutionEnvironment readEnv = StreamExecutionEnvironment.getExecutionEnvironment(); - readEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); - readEnv.getConfig().disableSysoutLogging(); - readEnv.setParallelism(parallelism); - - Properties readProps = (Properties) standardProps.clone(); - readProps.setProperty("group.id", "flink-tests-validator"); - readProps.putAll(secureProps); - FlinkKafkaConsumerBase> consumer = kafkaServer.getConsumer(topicName, deserSchema, readProps); - - readEnv - .addSource(consumer) - .map(new RichMapFunction, Tuple2>() { - - private final int totalCount = parallelism * numElements; - private int count = 0; - - @Override - public Tuple2 map(Tuple2 value) throws Exception { - if (++count == totalCount) { - throw new SuccessException(); - } else { - return value; - } - } - }).setParallelism(1) - .addSink(new DiscardingSink>()).setParallelism(1); - - final AtomicReference errorRef = new AtomicReference<>(); - - Thread runner = new Thread() { - @Override - public void run() { - try { - tryExecute(readEnv, "sequence validation"); - } catch (Throwable t) { - errorRef.set(t); - } - } - }; - runner.start(); - - final long deadline = System.nanoTime() + 10_000_000_000L; - long delay; - while (runner.isAlive() && (delay = deadline - System.nanoTime()) > 0) { - runner.join(delay / 1_000_000L); - } - - boolean success; - - if (runner.isAlive()) { - // did not finish in time, maybe the producer dropped one or more records and - // the validation did not reach the exit point - success = false; - JobManagerCommunicationUtils.cancelCurrentJob(flink.getLeaderGateway(timeout)); - } - else { - Throwable error = errorRef.get(); - if (error != null) { - success = false; - LOG.info("Attempt " + attempt + " failed with exception", error); - } - else { - success = true; - } - } - - JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); - - if (success) { + if (validateSequence(topicName, parallelism, deserSchema, numElements)) { // everything is good! return topicName; } @@ -2031,13 +1978,14 @@ public void run() { throw new Exception("Could not write a valid sequence to Kafka after " + maxNumAttempts + " attempts"); } - protected TopicWithStartDate writeAppendSequence( - String baseTopicName, - final int numElements, - final int parallelism, - final int replicationFactor) throws Exception { + protected void writeAppendSequence( + String topicName, + final int originalNumElements, + final int numElementsToAppend, + final int parallelism) throws Exception { + LOG.info("\n===================================\n" + - "== Writing sequence of " + numElements + " into " + baseTopicName + " with p=" + parallelism + "\n" + + "== Appending sequence of " + numElementsToAppend + " into " + topicName + "==================================="); final TypeInformation> resultType = @@ -2051,224 +1999,136 @@ protected TopicWithStartDate writeAppendSequence( new KeyedDeserializationSchemaWrapper<>( new TypeInformationSerializationSchema<>(resultType, new ExecutionConfig())); - final int maxNumAttempts = 10; - - final List> kafkaTupleList = new ArrayList<>(); - for (int attempt = 1; attempt <= maxNumAttempts; attempt++) { - - final String topicName = baseTopicName + '-' + attempt; - - LOG.info("Writing attempt #1"); - - // -------- Write the Sequence -------- - - createTestTopic(topicName, parallelism, replicationFactor); - - Date firstStartDate = new Date(); - StreamExecutionEnvironment writeEnv = StreamExecutionEnvironment.getExecutionEnvironment(); - writeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); - writeEnv.getConfig().disableSysoutLogging(); + // -------- Write the append sequence -------- - DataStream> stream = writeEnv.addSource(new RichParallelSourceFunction>() { + StreamExecutionEnvironment writeEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + writeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + writeEnv.getConfig().disableSysoutLogging(); - private boolean running = true; + DataStream> stream = writeEnv.addSource(new RichParallelSourceFunction>() { - @Override - public void run(SourceContext> ctx) throws Exception { - int cnt = 0; - int partition = getRuntimeContext().getIndexOfThisSubtask(); + private boolean running = true; - while (running && cnt < numElements) { - ctx.collect(new Tuple2<>(partition, cnt)); - cnt++; - } - } + @Override + public void run(SourceContext> ctx) throws Exception { + int cnt = originalNumElements; + int partition = getRuntimeContext().getIndexOfThisSubtask(); - @Override - public void cancel() { - running = false; + while (running && cnt < numElementsToAppend + originalNumElements) { + ctx.collect(new Tuple2<>(partition, cnt)); + cnt++; } - }).setParallelism(parallelism); - - // the producer must not produce duplicates - Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); - producerProperties.setProperty("retries", "0"); - producerProperties.putAll(secureProps); - - kafkaServer.produceIntoKafka(stream, topicName, serSchema, producerProperties, new Tuple2FlinkPartitioner(parallelism)) - .setParallelism(parallelism); - - try { - writeEnv.execute("Write sequence"); } - catch (Exception e) { - LOG.error("Write attempt failed, trying again", e); - deleteTestTopic(topicName); - JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); - continue; - } - - Thread.sleep(10); - Date secondStartDate = new Date(); - writeEnv = StreamExecutionEnvironment.getExecutionEnvironment(); - writeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); - writeEnv.getConfig().disableSysoutLogging(); - - stream = writeEnv.addSource(new RichParallelSourceFunction>() { - - private boolean running = true; - @Override - public void run(SourceContext> ctx) throws Exception { - int cnt = numElements; - int partition = getRuntimeContext().getIndexOfThisSubtask(); + @Override + public void cancel() { + running = false; + } + }).setParallelism(parallelism); - while (running && cnt < numElements + numElements) { - ctx.collect(new Tuple2<>(partition, cnt)); - cnt++; - } - } + // the producer must not produce duplicates + Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); + producerProperties.setProperty("retries", "0"); + producerProperties.putAll(secureProps); - @Override - public void cancel() { - running = false; - } - }).setParallelism(parallelism); + kafkaServer.produceIntoKafka(stream, topicName, serSchema, producerProperties, new Tuple2FlinkPartitioner(parallelism)) + .setParallelism(parallelism); - // the producer must not produce duplicates - producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); - producerProperties.setProperty("retries", "0"); - producerProperties.putAll(secureProps); + try { + writeEnv.execute("Write sequence"); + } + catch (Exception e) { + throw new Exception("Failed to append sequence to Kafka; append job failed.", e); + } - kafkaServer.produceIntoKafka(stream, topicName, serSchema, producerProperties, new Tuple2FlinkPartitioner(parallelism)) - .setParallelism(parallelism); + LOG.info("Finished writing append sequence"); - try { - writeEnv.execute("Write sequence"); - } - catch (Exception e) { - LOG.error("Write attempt failed, trying again", e); - deleteTestTopic(topicName); - JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); - continue; - } + // we need to validate the sequence, because kafka's producers are not exactly once + LOG.info("Validating sequence"); + JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); - LOG.info("Finished writing sequence"); + if (!validateSequence(topicName, parallelism, deserSchema, originalNumElements + numElementsToAppend)) { + throw new Exception("Could not append a valid sequence to Kafka."); + } + } - // -------- Validate the Sequence -------- + private boolean validateSequence( + final String topic, + final int parallelism, + KeyedDeserializationSchema> deserSchema, + final int totalNumElements) throws Exception { - // we need to validate the sequence, because kafka's producers are not exactly once - LOG.info("Validating sequence"); + final StreamExecutionEnvironment readEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + readEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + readEnv.getConfig().disableSysoutLogging(); + readEnv.setParallelism(parallelism); - JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); + Properties readProps = (Properties) standardProps.clone(); + readProps.setProperty("group.id", "flink-tests-validator"); + readProps.putAll(secureProps); + FlinkKafkaConsumerBase> consumer = kafkaServer.getConsumer(topic, deserSchema, readProps); + consumer.setStartFromEarliest(); - final StreamExecutionEnvironment readEnv = StreamExecutionEnvironment.getExecutionEnvironment(); - readEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); - readEnv.getConfig().disableSysoutLogging(); - readEnv.setParallelism(parallelism); - - Properties readProps = (Properties) standardProps.clone(); - readProps.setProperty("group.id", "flink-tests-validator"); - readProps.putAll(secureProps); - FlinkKafkaConsumerBase> consumer = kafkaServer.getConsumer(topicName, deserSchema, readProps); - - readEnv - .addSource(consumer) - .map(new RichMapFunction, Tuple2>() { - - private final int totalCount = parallelism * (numElements + 1); - private int count = 0; - - @Override - public Tuple2 map(Tuple2 value) throws Exception { - if (++count == totalCount) { - throw new SuccessException(); - } else { - return value; - } - } - }).setParallelism(1) - .addSink(new DiscardingSink>()).setParallelism(1); + readEnv + .addSource(consumer) + .map(new RichMapFunction, Tuple2>() { - final AtomicReference errorRef = new AtomicReference<>(); + private final int totalCount = parallelism * totalNumElements; + private int count = 0; - Thread runner = new Thread() { @Override - public void run() { - try { - tryExecute(readEnv, "sequence validation"); - } catch (Throwable t) { - errorRef.set(t); + public Tuple2 map(Tuple2 value) throws Exception { + if (++count == totalCount) { + throw new SuccessException(); + } else { + return value; } } - }; - runner.start(); - - final long deadline = System.nanoTime() + 10_000_000_000L; - long delay; - while (runner.isAlive() && (delay = deadline - System.nanoTime()) > 0) { - runner.join(delay / 1_000_000L); - } + }).setParallelism(1) + .addSink(new DiscardingSink<>()).setParallelism(1); - boolean success; + final AtomicReference errorRef = new AtomicReference<>(); - if (runner.isAlive()) { - // did not finish in time, maybe the producer dropped one or more records and - // the validation did not reach the exit point - success = false; - JobManagerCommunicationUtils.cancelCurrentJob(flink.getLeaderGateway(timeout)); - } - else { - Throwable error = errorRef.get(); - if (error != null) { - success = false; - LOG.info("Attempt " + attempt + " failed with exception", error); - } - else { - success = true; + Thread runner = new Thread() { + @Override + public void run() { + try { + tryExecute(readEnv, "sequence validation"); + } catch (Throwable t) { + errorRef.set(t); } } + }; + runner.start(); - JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); - - if (success) { - // everything is good! - return new TopicWithStartDate(topicName, firstStartDate, secondStartDate); - } - else { - deleteTestTopic(topicName); - // fall through the loop - } + final long deadline = System.nanoTime() + 10_000_000_000L; + long delay; + while (runner.isAlive() && (delay = deadline - System.nanoTime()) > 0) { + runner.join(delay / 1_000_000L); } - throw new Exception("Could not write a valid sequence to Kafka after " + maxNumAttempts + " attempts"); - } + boolean success; - /** - * Pojo class for consumer with date. - */ - public static class TopicWithStartDate { - private final String topicName; - private final Date firstStartDate; - private final Date secondStartDate; - - public TopicWithStartDate(String topicName, Date firstStartDate, Date secondStartDate) { - this.topicName = topicName; - this.firstStartDate = firstStartDate; - this.secondStartDate = secondStartDate; + if (runner.isAlive()) { + // did not finish in time, maybe the producer dropped one or more records and + // the validation did not reach the exit point + success = false; + JobManagerCommunicationUtils.cancelCurrentJob(flink.getLeaderGateway(timeout)); } - - public String getTopicName() { - return topicName; + else { + Throwable error = errorRef.get(); + if (error != null) { + success = false; + LOG.info("Sequence validation job failed with exception", error); + } + else { + success = true; + } } - public Date getFirstStartDate() { - return firstStartDate; - } + JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); - public Date getSecondStartDate() { - return secondStartDate; - } + return success; } // ------------------------------------------------------------------------ From 63b3563e5404b89ad8fd20f1181b8247e7de3fa2 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 28 Feb 2018 14:07:12 +0100 Subject: [PATCH 0079/2294] [FLINK-8808] [flip6] Allow RestClusterClient to connect to local dispatcher The RestClusterClient resolves a dispatcher address without an explicit host to 'localhost'. That way we allow the RestClusterClient to talk to a Dispatcher which runs in a local ActorSystem. This closes #5599. --- .../program/rest/RestClusterClient.java | 20 ++++-- .../apache/flink/runtime/util/ScalaUtils.java | 46 +++++++++++++ .../flink/runtime/util/ScalaUtilsTest.java | 66 +++++++++++++++++++ 3 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/util/ScalaUtils.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/util/ScalaUtilsTest.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 3a377f3ef49b14..98332968004bcb 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -77,6 +77,7 @@ import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.runtime.util.LeaderRetrievalUtils; +import org.apache.flink.runtime.util.ScalaUtils; import org.apache.flink.runtime.webmonitor.retriever.LeaderRetriever; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.ExecutorUtils; @@ -97,6 +98,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -107,10 +109,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import scala.Option; - -import static org.apache.flink.util.Preconditions.checkArgument; - /** * A {@link ClusterClient} implementation that communicates via HTTP REST requests. */ @@ -636,9 +634,17 @@ private CompletableFuture getDispatcherAddress() { TimeUnit.MILLISECONDS) .thenApplyAsync(leaderAddressSessionId -> { final String address = leaderAddressSessionId.f0; - final Option host = AddressFromURIString.parse(address).host(); - checkArgument(host.isDefined(), "Could not parse host from %s", address); - return host.get(); + final Optional host = ScalaUtils.toJava(AddressFromURIString.parse(address).host()); + + return host.orElseGet(() -> { + // if the dispatcher address does not contain a host part, then assume it's running + // on the same machine as the client + log.info("The dispatcher seems to run without remoting enabled. This indicates that we are " + + "in a test. This can only work if the RestClusterClient runs on the same machine. " + + "Assuming, therefore, 'localhost' as the host."); + + return "localhost"; + }); }, executorService); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ScalaUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ScalaUtils.java new file mode 100644 index 00000000000000..fa4260eded8fa0 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ScalaUtils.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.util; + +import java.util.Optional; + +import scala.Option; + +/** + * Utilities to convert Scala types into Java types. + */ +public class ScalaUtils { + + /** + * Converts a Scala {@link Option} to a {@link Optional}. + * + * @param scalaOption to convert into ta Java {@link Optional} + * @param type of the optional value + * @return Optional of the given option + */ + public static Optional toJava(Option scalaOption) { + if (scalaOption.isEmpty()) { + return Optional.empty(); + } else { + return Optional.ofNullable(scalaOption.get()); + } + } + + private ScalaUtils() {} +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/ScalaUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/ScalaUtilsTest.java new file mode 100644 index 00000000000000..e5e9896240d1ac --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/ScalaUtilsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.util; + +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import java.util.NoSuchElementException; +import java.util.Optional; + +import scala.Option; + +import static org.apache.flink.runtime.util.ScalaUtils.toJava; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Tests for {@link ScalaUtils} convenience methods. + */ +public class ScalaUtilsTest extends TestLogger { + + @Test + public void testOptionToOptional() { + final String value = "foobar"; + final Option option = Option.apply(value); + final Optional optional = toJava(option); + + assertThat(optional.isPresent(), is(true)); + assertThat(optional.get(), is(value)); + + final Option nullOption = Option.apply(null); + final Optional nullOptional = toJava(nullOption); + + assertThat(nullOptional.isPresent(), is(false)); + + try { + nullOptional.get(); + fail("Expected NoSuchElementException"); + } catch (NoSuchElementException ignored) { + // ignored + } + + final Option emptyOption = Option.empty(); + final Optional emptyOptional = toJava(emptyOption); + + assertThat(emptyOptional.isPresent(), is(false)); + } +} From 19a8d2ff361be8797d79074b39afb09d51cca671 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 28 Feb 2018 17:36:39 +0100 Subject: [PATCH 0080/2294] [FLINK-8811] [flip6] Add initial implementation of the MiniClusterClient The MiniClusterClient directly talks to the MiniCluster avoiding polling latencies of th RestClusterClient. This closes #5600. --- .../client/program/MiniClusterClient.java | 171 ++++++++++++++++++ .../runtime/minicluster/MiniCluster.java | 7 + .../flink/test/util/MiniClusterResource.java | 53 ++++-- 3 files changed, 216 insertions(+), 15 deletions(-) create mode 100644 flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java new file mode 100644 index 00000000000000..5baae5b0a3e637 --- /dev/null +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.client.program; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobSubmissionResult; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.runtime.client.JobStatusMessage; +import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; +import org.apache.flink.runtime.messages.Acknowledge; +import org.apache.flink.runtime.minicluster.MiniCluster; +import org.apache.flink.runtime.util.LeaderConnectionInfo; +import org.apache.flink.runtime.util.LeaderRetrievalUtils; +import org.apache.flink.util.FlinkException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import java.net.URL; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * Client to interact with a {@link MiniCluster}. + */ +public class MiniClusterClient extends ClusterClient { + + private final MiniCluster miniCluster; + + public MiniClusterClient(@Nonnull Configuration configuration, @Nonnull MiniCluster miniCluster) throws Exception { + super(configuration, miniCluster.getHighAvailabilityServices()); + + this.miniCluster = miniCluster; + } + + @Override + protected JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { + if (isDetached()) { + try { + miniCluster.runDetached(jobGraph); + } catch (JobExecutionException | InterruptedException e) { + throw new ProgramInvocationException( + String.format("Could not run job %s in detached mode.", jobGraph.getJobID()), + e); + } + + return new JobSubmissionResult(jobGraph.getJobID()); + } else { + try { + return miniCluster.executeJobBlocking(jobGraph); + } catch (JobExecutionException | InterruptedException e) { + throw new ProgramInvocationException( + String.format("Could not run job %s.", jobGraph.getJobID()), + e); + } + } + } + + @Override + public void cancel(JobID jobId) throws Exception { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public String cancelWithSavepoint(JobID jobId, @Nullable String savepointDirectory) throws Exception { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public void stop(JobID jobId) throws Exception { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public CompletableFuture triggerSavepoint(JobID jobId, @Nullable String savepointDirectory) throws FlinkException { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public CompletableFuture disposeSavepoint(String savepointPath, Time timeout) throws FlinkException { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public CompletableFuture> listJobs() throws Exception { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public Map getAccumulators(JobID jobID) throws Exception { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public Map getAccumulators(JobID jobID, ClassLoader loader) throws Exception { + throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + } + + @Override + public MiniClusterClient.MiniClusterId getClusterId() { + return MiniClusterId.INSTANCE; + } + + @Override + public LeaderConnectionInfo getClusterConnectionInfo() throws LeaderRetrievalException { + return LeaderRetrievalUtils.retrieveLeaderConnectionInfo( + highAvailabilityServices.getDispatcherLeaderRetriever(), + timeout); + } + + // ====================================== + // Legacy methods + // ====================================== + + @Override + public void waitForClusterToBeReady() { + // no op + } + + @Override + public String getWebInterfaceURL() { + return miniCluster.getRestAddress().toString(); + } + + @Override + public GetClusterStatusResponse getClusterStatus() { + return null; + } + + @Override + public List getNewMessages() { + return Collections.emptyList(); + } + + @Override + public int getMaxSlots() { + return 0; + } + + @Override + public boolean hasUserJarsInClassPath(List userJarFiles) { + return false; + } + + enum MiniClusterId { + INSTANCE + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 3bd17d1e858240..cbfb266499287f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -178,6 +178,13 @@ public URI getRestAddress() { } } + public HighAvailabilityServices getHighAvailabilityServices() { + synchronized (lock) { + checkState(running, "MiniCluster is not yet running."); + return haServices; + } + } + // ------------------------------------------------------------------------ // life cycle // ------------------------------------------------------------------------ diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java index 1c5da62b2e0000..a1ce64737571a6 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java @@ -19,15 +19,20 @@ package org.apache.flink.test.util; import org.apache.flink.api.common.time.Time; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.client.program.MiniClusterClient; +import org.apache.flink.client.program.StandaloneClusterClient; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; import org.apache.flink.configuration.RestOptions; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.minicluster.JobExecutorService; +import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.runtime.minicluster.MiniCluster; import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.streaming.util.TestStreamEnvironment; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.Preconditions; @@ -35,8 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; - import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -59,6 +62,8 @@ public class MiniClusterResource extends ExternalResource { private JobExecutorService jobExecutorService; + private ClusterClient clusterClient; + private int numberSlots = -1; private TestEnvironment executionEnvironment; @@ -80,6 +85,10 @@ public int getNumberSlots() { return numberSlots; } + public ClusterClient getClusterClient() { + return clusterClient; + } + public TestEnvironment getTestEnvironment() { return executionEnvironment; } @@ -87,7 +96,7 @@ public TestEnvironment getTestEnvironment() { @Override public void before() throws Exception { - jobExecutorService = startJobExecutorService(miniClusterType); + startJobExecutorService(miniClusterType); numberSlots = miniClusterResourceConfiguration.getNumberSlotsPerTaskManager() * miniClusterResourceConfiguration.getNumberTaskManagers(); @@ -102,6 +111,16 @@ public void after() { TestStreamEnvironment.unsetAsContext(); TestEnvironment.unsetAsContext(); + Exception exception = null; + + try { + clusterClient.shutdown(); + } catch (Exception e) { + exception = e; + } + + clusterClient = null; + final CompletableFuture terminationFuture = jobExecutorService.closeAsync(); try { @@ -109,40 +128,43 @@ public void after() { miniClusterResourceConfiguration.getShutdownTimeout().toMilliseconds(), TimeUnit.MILLISECONDS); } catch (Exception e) { - LOG.warn("Could not properly shut down the MiniClusterResource.", e); + exception = ExceptionUtils.firstOrSuppressed(e, exception); } jobExecutorService = null; + + if (exception != null) { + LOG.warn("Could not properly shut down the MiniClusterResource.", exception); + } } - private JobExecutorService startJobExecutorService(MiniClusterType miniClusterType) throws Exception { - final JobExecutorService jobExecutorService; + private void startJobExecutorService(MiniClusterType miniClusterType) throws Exception { switch (miniClusterType) { case OLD: - jobExecutorService = startOldMiniCluster(); + startOldMiniCluster(); break; case FLIP6: - jobExecutorService = startFlip6MiniCluster(); + startFlip6MiniCluster(); break; default: throw new FlinkRuntimeException("Unknown MiniClusterType " + miniClusterType + '.'); } - - return jobExecutorService; } - private JobExecutorService startOldMiniCluster() throws Exception { + private void startOldMiniCluster() throws Exception { final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration()); configuration.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, miniClusterResourceConfiguration.getNumberTaskManagers()); configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, miniClusterResourceConfiguration.getNumberSlotsPerTaskManager()); - return TestBaseUtils.startCluster( + final LocalFlinkMiniCluster flinkMiniCluster = TestBaseUtils.startCluster( configuration, true); + + jobExecutorService = flinkMiniCluster; + clusterClient = new StandaloneClusterClient(configuration, flinkMiniCluster.highAvailabilityServices()); } - @Nonnull - private JobExecutorService startFlip6MiniCluster() throws Exception { + private void startFlip6MiniCluster() throws Exception { final Configuration configuration = miniClusterResourceConfiguration.getConfiguration(); // we need to set this since a lot of test expect this because TestBaseUtils.startCluster() @@ -165,7 +187,8 @@ private JobExecutorService startFlip6MiniCluster() throws Exception { // update the port of the rest endpoint configuration.setInteger(RestOptions.REST_PORT, miniCluster.getRestAddress().getPort()); - return miniCluster; + jobExecutorService = miniCluster; + clusterClient = new MiniClusterClient(configuration, miniCluster); } /** From 96a176ac03bb1d188e173ba3bf14c29259d33377 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 28 Feb 2018 17:49:29 +0100 Subject: [PATCH 0081/2294] [FLINK-8811] [flip6] Implement MiniClusterClient#getJobStatus --- .../client/program/MiniClusterClient.java | 7 ++++++- .../flink/runtime/minicluster/MiniCluster.java | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 5baae5b0a3e637..e99addda9e8131 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -26,6 +26,7 @@ import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.minicluster.MiniCluster; @@ -57,7 +58,7 @@ public MiniClusterClient(@Nonnull Configuration configuration, @Nonnull MiniClus } @Override - protected JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { + public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { if (isDetached()) { try { miniCluster.runDetached(jobGraph); @@ -119,6 +120,10 @@ public Map getAccumulators(JobID jobID, ClassLoader loader) thro throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); } + public CompletableFuture getJobStatus(JobID jobId) { + return miniCluster.getJobStatus(jobId); + } + @Override public MiniClusterClient.MiniClusterId getClusterId() { return MiniClusterId.INSTANCE; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index cbfb266499287f..5b086ca55de204 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.minicluster; import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.io.FileOutputFormat; import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.Configuration; @@ -41,6 +42,7 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; @@ -63,6 +65,7 @@ import org.apache.flink.runtime.webmonitor.retriever.impl.RpcGatewayRetriever; import org.apache.flink.util.AutoCloseableAsync; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FlinkException; import akka.actor.ActorSystem; import com.typesafe.config.Config; @@ -454,6 +457,21 @@ public CompletableFuture closeAsync() { } } + // ------------------------------------------------------------------------ + // Accessing jobs + // ------------------------------------------------------------------------ + + public CompletableFuture getJobStatus(JobID jobId) { + try { + return getDispatcherGateway().requestJobStatus(jobId, rpcTimeout); + } catch (LeaderRetrievalException | InterruptedException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Could not retrieve job status for job %s", jobId), + e)); + } + } + // ------------------------------------------------------------------------ // running jobs // ------------------------------------------------------------------------ From 8039464df8f315b1fd06831e11dfc2ef4466b888 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 28 Feb 2018 17:54:06 +0100 Subject: [PATCH 0082/2294] [FLINK-8811] [flip6] Implement MiniClusterClient#cancel --- .../flink/client/program/MiniClusterClient.java | 2 +- .../flink/runtime/minicluster/MiniCluster.java | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index e99addda9e8131..b98e895c515589 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -82,7 +82,7 @@ public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) @Override public void cancel(JobID jobId) throws Exception { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + miniCluster.cancelJob(jobId); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 5b086ca55de204..2efdb03505b975 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -467,7 +467,18 @@ public CompletableFuture getJobStatus(JobID jobId) { } catch (LeaderRetrievalException | InterruptedException e) { return FutureUtils.completedExceptionally( new FlinkException( - String.format("Could not retrieve job status for job %s", jobId), + String.format("Could not retrieve job status for job %s.", jobId), + e)); + } + } + + public CompletableFuture cancelJob(JobID jobId) { + try { + return getDispatcherGateway().cancelJob(jobId, rpcTimeout); + } catch (LeaderRetrievalException | InterruptedException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Could not cancel job %s.", jobId), e)); } } From 4c849942b2734ed962245c6df23fb6b4d823dc60 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 17:02:29 +0100 Subject: [PATCH 0083/2294] [hotfix] Introduce null checks for SlotManager#suspend --- .../resourcemanager/slotmanager/SlotManager.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java index f078a28fab260c..ca3371945c8fd4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java @@ -208,11 +208,15 @@ public void suspend() { LOG.info("Suspending the SlotManager."); // stop the timeout checks for the TaskManagers and the SlotRequests - taskManagerTimeoutCheck.cancel(false); - slotRequestTimeoutCheck.cancel(false); + if (taskManagerTimeoutCheck != null) { + taskManagerTimeoutCheck.cancel(false); + taskManagerTimeoutCheck = null; + } - taskManagerTimeoutCheck = null; - slotRequestTimeoutCheck = null; + if (slotRequestTimeoutCheck != null) { + slotRequestTimeoutCheck.cancel(false); + slotRequestTimeoutCheck = null; + } for (PendingSlotRequest pendingSlotRequest : pendingSlotRequests.values()) { cancelPendingSlotRequest(pendingSlotRequest); From b25d30074c3f2b9474fc0597973e50a0b7b8f4e7 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 17:05:44 +0100 Subject: [PATCH 0084/2294] [hotfix] Let ClusterClient only shut down own HaServices --- .../apache/flink/client/program/ClusterClient.java | 14 ++++++++++---- .../flink/client/program/MiniClusterClient.java | 2 +- .../client/program/StandaloneClusterClient.java | 4 ++-- .../flink/client/cli/CliFrontendModifyTest.java | 2 +- .../flink/client/cli/CliFrontendSavepointTest.java | 5 +++-- .../flink/client/program/ClientConnectionTest.java | 2 +- .../flink/client/program/ClusterClientTest.java | 4 ++-- .../flink/test/util/MiniClusterResource.java | 2 +- .../checkpointing/AbstractLocalRecoveryITCase.java | 12 ++++++------ .../test/example/client/JobRetrievalITCase.java | 2 +- 10 files changed, 28 insertions(+), 21 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index e2efbac523ceb9..1cf2bc2847c362 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -117,6 +117,8 @@ public abstract class ClusterClient { /** Service factory for high available. */ protected final HighAvailabilityServices highAvailabilityServices; + private final boolean sharedHaServices; + /** Flag indicating whether to sysout print execution updates. */ private boolean printStatusDuringExecution = true; @@ -144,11 +146,13 @@ public abstract class ClusterClient { * @throws Exception we cannot create the high availability services */ public ClusterClient(Configuration flinkConfig) throws Exception { - this(flinkConfig, + this( + flinkConfig, HighAvailabilityServicesUtils.createHighAvailabilityServices( flinkConfig, Executors.directExecutor(), - HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION)); + HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION), + false); } /** @@ -158,8 +162,9 @@ public ClusterClient(Configuration flinkConfig) throws Exception { * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * @param highAvailabilityServices HighAvailabilityServices to use for leader retrieval + * @param sharedHaServices true if the HighAvailabilityServices are shared and must not be shut down */ - public ClusterClient(Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices) { + public ClusterClient(Configuration flinkConfig, HighAvailabilityServices highAvailabilityServices, boolean sharedHaServices) { this.flinkConfig = Preconditions.checkNotNull(flinkConfig); this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); @@ -173,6 +178,7 @@ public ClusterClient(Configuration flinkConfig, HighAvailabilityServices highAva log); this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); + this.sharedHaServices = sharedHaServices; } // ------------------------------------------------------------------------ @@ -265,7 +271,7 @@ public void shutdown() throws Exception { synchronized (this) { actorSystemLoader.shutdown(); - if (highAvailabilityServices != null) { + if (!sharedHaServices && highAvailabilityServices != null) { highAvailabilityServices.close(); } } diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index b98e895c515589..aca75e0c15087c 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -52,7 +52,7 @@ public class MiniClusterClient extends ClusterClient> rescaleJobFuture; public TestingClusterClient(CompletableFuture> rescaleJobFuture) throws Exception { - super(new Configuration(), new TestingHighAvailabilityServices()); + super(new Configuration(), new TestingHighAvailabilityServices(), false); this.rescaleJobFuture = rescaleJobFuture; } diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java index d7303447da72ea..f4c66eb08c2047 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java @@ -138,7 +138,8 @@ public void testTriggerSavepointFailureIllegalJobID() throws Exception { try { CliFrontend frontend = new MockedCliFrontend(new StandaloneClusterClient( new Configuration(), - new TestingHighAvailabilityServices())); + new TestingHighAvailabilityServices(), + false)); String[] parameters = { "invalid job id" }; try { @@ -288,7 +289,7 @@ private static final class DisposeSavepointClusterClient extends StandaloneClust private final BiFunction> disposeSavepointFunction; DisposeSavepointClusterClient(BiFunction> disposeSavepointFunction) throws Exception { - super(new Configuration(), new TestingHighAvailabilityServices()); + super(new Configuration(), new TestingHighAvailabilityServices(), false); this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction); } diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java index 1dd4787c12d2a3..2b8abb1f8ff48d 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/ClientConnectionTest.java @@ -140,7 +140,7 @@ public void testJobManagerRetrievalWithHAServices() throws Exception { highAvailabilityServices.setJobMasterLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID, settableLeaderRetrievalService); - StandaloneClusterClient client = new StandaloneClusterClient(configuration, highAvailabilityServices); + StandaloneClusterClient client = new StandaloneClusterClient(configuration, highAvailabilityServices, true); ActorGateway gateway = client.getJobManagerGateway(); diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java index e2eb88d20d4b16..f30fd192e8388a 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java @@ -71,7 +71,7 @@ public void testClusterClientShutdown() throws Exception { Configuration config = new Configuration(); HighAvailabilityServices highAvailabilityServices = mock(HighAvailabilityServices.class); - StandaloneClusterClient clusterClient = new StandaloneClusterClient(config, highAvailabilityServices); + StandaloneClusterClient clusterClient = new StandaloneClusterClient(config, highAvailabilityServices, false); clusterClient.shutdown(); @@ -333,7 +333,7 @@ private static class TestClusterClient extends StandaloneClusterClient { private final ActorGateway jobmanagerGateway; TestClusterClient(Configuration config, ActorGateway jobmanagerGateway) throws Exception { - super(config, new TestingHighAvailabilityServices()); + super(config, new TestingHighAvailabilityServices(), false); this.jobmanagerGateway = jobmanagerGateway; } diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java index a1ce64737571a6..954b06f65a9ca4 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java @@ -161,7 +161,7 @@ private void startOldMiniCluster() throws Exception { true); jobExecutorService = flinkMiniCluster; - clusterClient = new StandaloneClusterClient(configuration, flinkMiniCluster.highAvailabilityServices()); + clusterClient = new StandaloneClusterClient(configuration, flinkMiniCluster.highAvailabilityServices(), true); } private void startFlip6MiniCluster() throws Exception { diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java index a02e902ab2bfd7..13040c965549f4 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractLocalRecoveryITCase.java @@ -78,20 +78,20 @@ protected Configuration createClusterConfig() throws IOException { private void executeTest(AbstractEventTimeWindowCheckpointingITCase delegate) throws Exception { delegate.name = testName; try { - delegate.miniClusterResource.before(); + delegate.setupTestCluster(); try { delegate.testTumblingTimeWindow(); - delegate.miniClusterResource.after(); + delegate.stopTestCluster(); } catch (Exception e) { - delegate.miniClusterResource.after(); + delegate.stopTestCluster(); } - delegate.miniClusterResource.before(); + delegate.setupTestCluster(); try { delegate.testSlidingTimeWindow(); - delegate.miniClusterResource.after(); + delegate.stopTestCluster(); } catch (Exception e) { - delegate.miniClusterResource.after(); + delegate.stopTestCluster(); } } finally { delegate.tempFolder.delete(); diff --git a/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java b/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java index 221f3fa1453859..d34b6c337a06ee 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java @@ -79,7 +79,7 @@ public void testJobRetrieval() throws Exception { final JobGraph jobGraph = new JobGraph(jobID, "testjob", imalock); - final ClusterClient client = new StandaloneClusterClient(cluster.configuration(), cluster.highAvailabilityServices()); + final ClusterClient client = new StandaloneClusterClient(cluster.configuration(), cluster.highAvailabilityServices(), true); // acquire the lock to make sure that the job cannot complete until the job client // has been attached in resumingThread From a108a41ed2040d2466d073b4bc44bd3b484dd20c Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 17:06:27 +0100 Subject: [PATCH 0085/2294] [hotfix] Unregister job from JobManagerRunner before completing the result future --- .../org/apache/flink/runtime/jobmaster/JobManagerRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java index 80aa673c86a70e..8b64f0ddf57e97 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java @@ -265,8 +265,8 @@ public CompletableFuture closeAsync() { @Override public void jobReachedGloballyTerminalState(ArchivedExecutionGraph executionGraph) { // complete the result future with the terminal execution graph - resultFuture.complete(executionGraph); unregisterJobFromHighAvailability(); + resultFuture.complete(executionGraph); } /** From b007d30cb1c209c0a0ce7c8197ee5311c7af2fa1 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 17:13:44 +0100 Subject: [PATCH 0086/2294] [hotfix] Close ResourceManager LeaderRetrievalService in TaskExecutor --- .../flink/runtime/taskexecutor/TaskExecutor.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index 927bd117f1601b..956a319732e728 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -56,6 +56,7 @@ import org.apache.flink.runtime.jobmaster.JobMasterGateway; import org.apache.flink.runtime.jobmaster.JobMasterId; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; +import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.StackTraceSampleResponse; import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup; @@ -176,6 +177,8 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { private final JobLeaderService jobLeaderService; + private final LeaderRetrievalService resourceManagerLeaderRetriever; + // ------------------------------------------------------------------------ private final HardwareDescription hardwareDescription; @@ -207,6 +210,7 @@ public TaskExecutor( this.taskManagerLocation = taskExecutorServices.getTaskManagerLocation(); this.localStateStoresManager = taskExecutorServices.getTaskManagerStateStore(); this.networkEnvironment = taskExecutorServices.getNetworkEnvironment(); + this.resourceManagerLeaderRetriever = haServices.getResourceManagerLeaderRetriever(); this.jobManagerConnections = new HashMap<>(4); @@ -238,7 +242,7 @@ public void start() throws Exception { // start by connecting to the ResourceManager try { - haServices.getResourceManagerLeaderRetriever().start(new ResourceManagerLeaderListener()); + resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); } catch (Exception e) { onFatalError(e); } @@ -275,6 +279,12 @@ public CompletableFuture postStop() { resourceManagerHeartbeatManager.stop(); + try { + resourceManagerLeaderRetriever.stop(); + } catch (Exception e) { + throwable = ExceptionUtils.firstOrSuppressed(e, throwable); + } + try { taskExecutorServices.shutDown(); } catch (Throwable t) { From dabefa84bedb3abbc28dcf3cec6a1bf76bff6a28 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 18:21:19 +0100 Subject: [PATCH 0087/2294] [hotfix] Correct shutdown order of RestClusterClient --- .../client/program/rest/RestClusterClient.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 98332968004bcb..976f2a4db31ca1 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -192,13 +192,6 @@ private void startLeaderRetrievers() throws Exception { @Override public void shutdown() { - try { - // we only call this for legacy reasons to shutdown components that are started in the ClusterClient constructor - super.shutdown(); - } catch (Exception e) { - log.error("An error occurred during the client shutdown.", e); - } - ExecutorUtils.gracefulShutdown(restClusterClientConfiguration.getRetryDelay(), TimeUnit.MILLISECONDS, retryExecutorService); this.restClient.shutdown(Time.seconds(5)); @@ -215,6 +208,13 @@ public void shutdown() { } catch (Exception e) { log.error("An error occurred during stopping the dispatcherLeaderRetriever", e); } + + try { + // we only call this for legacy reasons to shutdown components that are started in the ClusterClient constructor + super.shutdown(); + } catch (Exception e) { + log.error("An error occurred during the client shutdown.", e); + } } @Override From 6e33ebc9ec650be73673858da9eae2eeffbd59e8 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 19:03:35 +0100 Subject: [PATCH 0088/2294] [hotfix] Let AbstractEventTimeWindowCheckpointingITCase shutdown ZooKeeper after MiniCluster --- ...tractEventTimeWindowCheckpointingITCase.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java index f37ba0d38053eb..61baefa05d8adb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java @@ -51,6 +51,7 @@ import org.apache.curator.test.TestingServer; import org.junit.After; +import org.junit.Before; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -87,6 +88,8 @@ public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLog private TestingServer zkServer; + public MiniClusterResource miniClusterResource; + @ClassRule public static TemporaryFolder tempFolder = new TemporaryFolder(); @@ -95,9 +98,6 @@ public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLog private AbstractStateBackend stateBackend; - @Rule - public final MiniClusterResource miniClusterResource = getMiniClusterResource(); - enum StateBackendEnum { MEM, FILE, ROCKSDB_FULLY_ASYNC, ROCKSDB_INCREMENTAL, ROCKSDB_INCREMENTAL_ZK, MEM_ASYNC, FILE_ASYNC } @@ -201,8 +201,19 @@ protected Configuration createClusterConfig() throws IOException { return config; } + @Before + public void setupTestCluster() throws Exception { + miniClusterResource = getMiniClusterResource(); + miniClusterResource.before(); + } + @After public void stopTestCluster() throws IOException { + if (miniClusterResource != null) { + miniClusterResource.after(); + miniClusterResource = null; + } + if (zkServer != null) { zkServer.stop(); zkServer = null; From 07fc06dd163a9190038a2178e35ea97cf428a375 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 19:04:15 +0100 Subject: [PATCH 0089/2294] [hotfix] Let JobLeaderService terminate leader retrieval services --- .../flink/runtime/taskexecutor/JobLeaderService.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java index 53763629079ae7..500d7e4a297df8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/JobLeaderService.java @@ -190,7 +190,12 @@ public void addJob(final JobID jobId, final String defaultTargetAddress) throws JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId); - jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener)); + final Tuple2 oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener)); + + if (oldEntry != null) { + oldEntry.f0.stop(); + oldEntry.f1.stop(); + } leaderRetrievalService.start(jobManagerLeaderListener); } From f3cd9c059b42d407937a3f13f45d6e816cd556b4 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 19:04:53 +0100 Subject: [PATCH 0090/2294] [hotfix] Improve logging in ZooKeeper services --- .../flink/runtime/dispatcher/Dispatcher.java | 2 ++ .../zookeeper/ZooKeeperHaServices.java | 6 ++++++ .../ZooKeeperLeaderRetrievalService.java | 7 +++++-- .../flink/runtime/minicluster/MiniCluster.java | 14 +++++++------- .../flink/runtime/taskexecutor/TaskExecutor.java | 5 +++-- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java index bb60277f325031..7a11cf08784188 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java @@ -194,6 +194,8 @@ public CompletableFuture postStop() { if (exception != null) { throw exception; + } else { + log.info("Stopped dispatcher {}.", getAddress()); } }); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java index 6d5c721daadb63..3882479ce95f07 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/zookeeper/ZooKeeperHaServices.java @@ -34,6 +34,8 @@ import org.apache.flink.util.ExceptionUtils; import org.apache.curator.framework.CuratorFramework; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.concurrent.Executor; @@ -80,6 +82,8 @@ */ public class ZooKeeperHaServices implements HighAvailabilityServices { + private static final Logger LOG = LoggerFactory.getLogger(ZooKeeperHaServices.class); + private static final String RESOURCE_MANAGER_LEADER_PATH = "/resource_manager_lock"; private static final String DISPATCHER_LEADER_PATH = "/dispatcher_lock"; @@ -211,6 +215,8 @@ public void close() throws Exception { @Override public void closeAndCleanupAllData() throws Exception { + LOG.info("Close and clean up all data for ZooKeeperHaServices."); + Throwable exception = null; try { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java index 00ba66c4ebe165..fd0ea82b6e0454 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderretrieval/ZooKeeperLeaderRetrievalService.java @@ -55,6 +55,8 @@ public class ZooKeeperLeaderRetrievalService implements LeaderRetrievalService, /** Curator recipe to watch changes of a specific ZooKeeper node. */ private final NodeCache cache; + private final String retrievalPath; + /** Listener which will be notified about leader changes. */ private volatile LeaderRetrievalListener leaderListener; @@ -80,6 +82,7 @@ public void stateChanged(CuratorFramework client, ConnectionState newState) { public ZooKeeperLeaderRetrievalService(CuratorFramework client, String retrievalPath) { this.client = Preconditions.checkNotNull(client, "CuratorFramework client"); this.cache = new NodeCache(client, retrievalPath); + this.retrievalPath = Preconditions.checkNotNull(retrievalPath); this.leaderListener = null; this.lastLeaderAddress = null; @@ -94,7 +97,7 @@ public void start(LeaderRetrievalListener listener) throws Exception { Preconditions.checkState(leaderListener == null, "ZooKeeperLeaderRetrievalService can " + "only be started once."); - LOG.info("Starting ZooKeeperLeaderRetrievalService."); + LOG.info("Starting ZooKeeperLeaderRetrievalService {}.", retrievalPath); synchronized (lock) { leaderListener = listener; @@ -111,7 +114,7 @@ public void start(LeaderRetrievalListener listener) throws Exception { @Override public void stop() throws Exception { - LOG.info("Stopping ZooKeeperLeaderRetrievalService."); + LOG.info("Stopping ZooKeeperLeaderRetrievalService {}.", retrievalPath); synchronized (lock) { if (!running) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 2efdb03505b975..6b5f9b50aecca7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -401,13 +401,6 @@ public CompletableFuture closeAsync() { final int numComponents = 2 + miniClusterConfiguration.getNumTaskManagers(); final Collection> componentTerminationFutures = new ArrayList<>(numComponents); - componentTerminationFutures.add(shutDownDispatcher()); - - if (resourceManagerRunner != null) { - componentTerminationFutures.add(resourceManagerRunner.closeAsync()); - resourceManagerRunner = null; - } - if (taskManagers != null) { for (TaskExecutor tm : taskManagers) { if (tm != null) { @@ -418,6 +411,13 @@ public CompletableFuture closeAsync() { taskManagers = null; } + componentTerminationFutures.add(shutDownDispatcher()); + + if (resourceManagerRunner != null) { + componentTerminationFutures.add(resourceManagerRunner.closeAsync()); + resourceManagerRunner = null; + } + final FutureUtils.ConjunctFuture componentsTerminationFuture = FutureUtils.completeAll(componentTerminationFutures); final CompletableFuture metricRegistryTerminationFuture = FutureUtils.runAfterwards( diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index 956a319732e728..cab686af59aa91 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -259,7 +259,7 @@ public void start() throws Exception { */ @Override public CompletableFuture postStop() { - log.info("Stopping TaskManager {}.", getAddress()); + log.info("Stopping TaskExecutor {}.", getAddress()); Throwable throwable = null; @@ -294,6 +294,7 @@ public CompletableFuture postStop() { if (throwable != null) { return FutureUtils.completedExceptionally(new FlinkException("Error while shutting the TaskExecutor down.", throwable)); } else { + log.info("Stopped TaskExecutor {}.", getAddress()); return CompletableFuture.completedFuture(null); } } @@ -1333,7 +1334,7 @@ public ResourceID getResourceID() { */ void onFatalError(final Throwable t) { try { - log.error("Fatal error occurred in TaskExecutor.", t); + log.error("Fatal error occurred in TaskExecutor {}.", getAddress(), t); } catch (Throwable ignored) {} // The fatal error handler implementation should make sure that this call is non-blocking From b31b707cb20f34633815718ff356e187f3397620 Mon Sep 17 00:00:00 2001 From: Xpray Date: Fri, 2 Mar 2018 12:11:45 +0800 Subject: [PATCH 0091/2294] [FLINK-8821] [table] Fix non-terminating decimal error This closes #5608. --- .../apache/flink/table/api/TableConfig.scala | 21 +++++++ .../flink/table/codegen/CodeGenerator.scala | 18 +++--- .../table/codegen/calls/ScalarOperators.scala | 29 ++++++++-- .../aggfunctions/AvgAggFunction.scala | 7 ++- .../plan/nodes/dataset/DataSetAggregate.scala | 3 +- .../dataset/DataSetWindowAggregate.scala | 55 ++++++++++++------ .../datastream/DataStreamGroupAggregate.scala | 1 + .../DataStreamGroupWindowAggregate.scala | 6 +- .../datastream/DataStreamOverAggregate.scala | 12 +++- .../runtime/aggregate/AggregateUtil.scala | 57 +++++++++++++------ .../table/expressions/DecimalTypeTest.scala | 11 +++- .../aggfunctions/AvgFunctionTest.scala | 12 +++- .../runtime/batch/table/CalcITCase.scala | 8 ++- 13 files changed, 176 insertions(+), 64 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala index 6448657c5111f4..c78a022bec44a1 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.api import _root_.java.util.TimeZone +import _root_.java.math.MathContext import org.apache.flink.table.calcite.CalciteConfig @@ -41,6 +42,12 @@ class TableConfig { */ private var calciteConfig = CalciteConfig.DEFAULT + /** + * Defines the default context for decimal division calculation. + * We use Scala's default MathContext.DECIMAL128. + */ + private var decimalContext = MathContext.DECIMAL128 + /** * Sets the timezone for date/time/timestamp conversions. */ @@ -78,6 +85,20 @@ class TableConfig { def setCalciteConfig(calciteConfig: CalciteConfig): Unit = { this.calciteConfig = calciteConfig } + + /** + * Returns the default context for decimal division calculation. + * [[_root_.java.math.MathContext#DECIMAL128]] by default. + */ + def getDecimalContext: MathContext = decimalContext + + /** + * Sets the default context for decimal division calculation. + * [[_root_.java.math.MathContext#DECIMAL128]] by default. + */ + def setDecimalContext(mathContext: MathContext): Unit = { + this.decimalContext = mathContext + } } object TableConfig { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala index 756a8288dcaff5..e4064d6f1acd80 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala @@ -742,56 +742,56 @@ abstract class CodeGenerator( val right = operands(1) requireNumeric(left) requireNumeric(right) - generateArithmeticOperator("+", nullCheck, resultType, left, right) + generateArithmeticOperator("+", nullCheck, resultType, left, right, config) case PLUS | DATETIME_PLUS if isTemporal(resultType) => val left = operands.head val right = operands(1) requireTemporal(left) requireTemporal(right) - generateTemporalPlusMinus(plus = true, nullCheck, left, right) + generateTemporalPlusMinus(plus = true, nullCheck, left, right, config) case MINUS if isNumeric(resultType) => val left = operands.head val right = operands(1) requireNumeric(left) requireNumeric(right) - generateArithmeticOperator("-", nullCheck, resultType, left, right) + generateArithmeticOperator("-", nullCheck, resultType, left, right, config) case MINUS | MINUS_DATE if isTemporal(resultType) => val left = operands.head val right = operands(1) requireTemporal(left) requireTemporal(right) - generateTemporalPlusMinus(plus = false, nullCheck, left, right) + generateTemporalPlusMinus(plus = false, nullCheck, left, right, config) case MULTIPLY if isNumeric(resultType) => val left = operands.head val right = operands(1) requireNumeric(left) requireNumeric(right) - generateArithmeticOperator("*", nullCheck, resultType, left, right) + generateArithmeticOperator("*", nullCheck, resultType, left, right, config) case MULTIPLY if isTimeInterval(resultType) => val left = operands.head val right = operands(1) requireTimeInterval(left) requireNumeric(right) - generateArithmeticOperator("*", nullCheck, resultType, left, right) + generateArithmeticOperator("*", nullCheck, resultType, left, right, config) case DIVIDE | DIVIDE_INTEGER if isNumeric(resultType) => val left = operands.head val right = operands(1) requireNumeric(left) requireNumeric(right) - generateArithmeticOperator("/", nullCheck, resultType, left, right) + generateArithmeticOperator("/", nullCheck, resultType, left, right, config) case MOD if isNumeric(resultType) => val left = operands.head val right = operands(1) requireNumeric(left) requireNumeric(right) - generateArithmeticOperator("%", nullCheck, resultType, left, right) + generateArithmeticOperator("%", nullCheck, resultType, left, right, config) case UNARY_MINUS if isNumeric(resultType) => val operand = operands.head @@ -922,7 +922,7 @@ abstract class CodeGenerator( val left = operands.head val right = operands(1) requireString(left) - generateArithmeticOperator("+", nullCheck, resultType, left, right) + generateArithmeticOperator("+", nullCheck, resultType, left, right, config) // rows case ROW => diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala index a261b3d8ebbd45..57f1618a2e3633 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/calls/ScalarOperators.scala @@ -17,12 +17,15 @@ */ package org.apache.flink.table.codegen.calls +import java.math.MathContext + import org.apache.calcite.avatica.util.DateTimeUtils.MILLIS_PER_DAY import org.apache.calcite.avatica.util.{DateTimeUtils, TimeUnitRange} import org.apache.calcite.util.BuiltInMethod import org.apache.flink.api.common.typeinfo.BasicTypeInfo._ import org.apache.flink.api.common.typeinfo._ import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo, RowTypeInfo} +import org.apache.flink.table.api.TableConfig import org.apache.flink.table.codegen.CodeGenUtils._ import org.apache.flink.table.codegen.calls.CallGenerator.generateCallIfArgsNotNull import org.apache.flink.table.codegen.{CodeGenException, CodeGenerator, GeneratedExpression} @@ -46,7 +49,8 @@ object ScalarOperators { nullCheck: Boolean, resultType: TypeInformation[_], left: GeneratedExpression, - right: GeneratedExpression): GeneratedExpression = { + right: GeneratedExpression, + config: TableConfig): GeneratedExpression = { val leftCasting = operator match { case "%" => @@ -68,7 +72,15 @@ object ScalarOperators { generateOperatorIfNotNull(nullCheck, resultType, left, right) { (leftTerm, rightTerm) => if (isDecimal(resultType)) { - s"${leftCasting(leftTerm)}.${arithOpToDecMethod(operator)}(${rightCasting(rightTerm)})" + val decMethod = arithOpToDecMethod(operator) + operator match { + // include math context for decimal division + case "/" => + val mathContext = mathContextToString(config.getDecimalContext) + s"${leftCasting(leftTerm)}.$decMethod(${rightCasting(rightTerm)}, $mathContext)" + case _ => + s"${leftCasting(leftTerm)}.$decMethod(${rightCasting(rightTerm)})" + } } else { s"($resultTypeTerm) (${leftCasting(leftTerm)} $operator ${rightCasting(rightTerm)})" } @@ -814,14 +826,15 @@ object ScalarOperators { plus: Boolean, nullCheck: Boolean, left: GeneratedExpression, - right: GeneratedExpression) + right: GeneratedExpression, + config: TableConfig) : GeneratedExpression = { val op = if (plus) "+" else "-" (left.resultType, right.resultType) match { case (l: TimeIntervalTypeInfo[_], r: TimeIntervalTypeInfo[_]) if l == r => - generateArithmeticOperator(op, nullCheck, l, left, right) + generateArithmeticOperator(op, nullCheck, l, left, right, config) case (SqlTimeTypeInfo.DATE, TimeIntervalTypeInfo.INTERVAL_MILLIS) => generateOperatorIfNotNull(nullCheck, SqlTimeTypeInfo.DATE, left, right) { @@ -1290,6 +1303,14 @@ object ScalarOperators { case _ => throw new CodeGenException(s"Unsupported decimal arithmetic operator: '$operator'") } + private def mathContextToString(mathContext: MathContext): String = mathContext match { + case MathContext.DECIMAL32 => "java.math.MathContext.DECIMAL32" + case MathContext.DECIMAL64 => "java.math.MathContext.DECIMAL64" + case MathContext.DECIMAL128 => "java.math.MathContext.DECIMAL128" + case MathContext.UNLIMITED => "java.math.MathContext.UNLIMITED" + case _ => s"""new java.math.MathContext("$mathContext")""" + } + private def numericCasting( operandType: TypeInformation[_], resultType: TypeInformation[_]) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala index b651c424dc766e..26621b71961a3a 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/AvgAggFunction.scala @@ -17,7 +17,7 @@ */ package org.apache.flink.table.functions.aggfunctions -import java.math.{BigDecimal, BigInteger} +import java.math.{BigDecimal, BigInteger, MathContext} import java.lang.{Iterable => JIterable} import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} @@ -295,7 +295,8 @@ class DecimalAvgAccumulator extends JTuple2[BigDecimal, Long] { /** * Base class for built-in Big Decimal Avg aggregate function */ -class DecimalAvgAggFunction extends AggregateFunction[BigDecimal, DecimalAvgAccumulator] { +class DecimalAvgAggFunction(context: MathContext) + extends AggregateFunction[BigDecimal, DecimalAvgAccumulator] { override def createAccumulator(): DecimalAvgAccumulator = { new DecimalAvgAccumulator @@ -321,7 +322,7 @@ class DecimalAvgAggFunction extends AggregateFunction[BigDecimal, DecimalAvgAccu if (acc.f1 == 0) { null.asInstanceOf[BigDecimal] } else { - acc.f0.divide(BigDecimal.valueOf(acc.f1)) + acc.f0.divide(BigDecimal.valueOf(acc.f1), context) } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala index 7dd307b064281c..07dcf798621d3b 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetAggregate.scala @@ -110,7 +110,8 @@ class DataSetAggregate( input.getRowType, inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, rowRelDataType, - grouping) + grouping, + tableEnv.getConfig) val aggString = aggregationToString(inputType, grouping, getRowType, namedAggregates, Nil) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala index 745c4ed708a41a..53748f5c71cd0e 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetWindowAggregate.scala @@ -25,7 +25,7 @@ import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel} import org.apache.flink.api.common.operators.Order import org.apache.flink.api.java.DataSet import org.apache.flink.api.java.typeutils.{ResultTypeQueryable, RowTypeInfo} -import org.apache.flink.table.api.{BatchQueryConfig, BatchTableEnvironment} +import org.apache.flink.table.api.{BatchQueryConfig, BatchTableEnvironment, TableConfig} import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty import org.apache.flink.table.calcite.FlinkTypeFactory import org.apache.flink.table.codegen.AggregationCodeGenerator @@ -127,11 +127,12 @@ class DataSetWindowAggregate( generator, inputDS, isTimeIntervalLiteral(size), - caseSensitive) + caseSensitive, + tableEnv.getConfig) case SessionGroupWindow(_, timeField, gap) if isTimePoint(timeField.resultType) || isLong(timeField.resultType) => - createEventTimeSessionWindowDataSet(generator, inputDS, caseSensitive) + createEventTimeSessionWindowDataSet(generator, inputDS, caseSensitive, tableEnv.getConfig) case SlidingGroupWindow(_, timeField, size, slide) if isTimePoint(timeField.resultType) || isLong(timeField.resultType) => @@ -141,7 +142,8 @@ class DataSetWindowAggregate( isTimeIntervalLiteral(size), asLong(size), asLong(slide), - caseSensitive) + caseSensitive, + tableEnv.getConfig) case _ => throw new UnsupportedOperationException( @@ -153,7 +155,8 @@ class DataSetWindowAggregate( generator: AggregationCodeGenerator, inputDS: DataSet[Row], isTimeWindow: Boolean, - isParserCaseSensitive: Boolean): DataSet[Row] = { + isParserCaseSensitive: Boolean, + tableConfig: TableConfig): DataSet[Row] = { val input = inputNode.asInstanceOf[DataSetRel] @@ -164,7 +167,8 @@ class DataSetWindowAggregate( grouping, input.getRowType, inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, - isParserCaseSensitive) + isParserCaseSensitive, + tableConfig) val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction( generator, window, @@ -173,7 +177,8 @@ class DataSetWindowAggregate( inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, getRowType, grouping, - namedProperties) + namedProperties, + tableConfig) val mappedInput = inputDS .map(mapFunction) @@ -215,7 +220,8 @@ class DataSetWindowAggregate( private[this] def createEventTimeSessionWindowDataSet( generator: AggregationCodeGenerator, inputDS: DataSet[Row], - isParserCaseSensitive: Boolean): DataSet[Row] = { + isParserCaseSensitive: Boolean, + tableConfig: TableConfig): DataSet[Row] = { val input = inputNode.asInstanceOf[DataSetRel] @@ -230,7 +236,8 @@ class DataSetWindowAggregate( grouping, input.getRowType, inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, - isParserCaseSensitive) + isParserCaseSensitive, + tableConfig) val mappedInput = inputDS.map(mapFunction).name(prepareOperatorName) @@ -243,7 +250,8 @@ class DataSetWindowAggregate( if (doAllSupportPartialMerge( namedAggregates.map(_.getKey), inputType, - grouping.length)) { + grouping.length, + tableConfig)) { // gets the window-start and window-end position in the intermediate result. val windowStartPos = rowTimeFieldPos @@ -257,7 +265,8 @@ class DataSetWindowAggregate( namedAggregates, input.getRowType, inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, - grouping) + grouping, + tableConfig) // create groupReduceFunction for calculating the aggregations val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction( @@ -269,6 +278,7 @@ class DataSetWindowAggregate( rowRelDataType, grouping, namedProperties, + tableConfig, isInputCombined = true) mappedInput @@ -289,7 +299,8 @@ class DataSetWindowAggregate( namedAggregates, input.getRowType, inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, - grouping) + grouping, + tableConfig) // create groupReduceFunction for calculating the aggregations val groupReduceFunction = createDataSetWindowAggregationGroupReduceFunction( @@ -301,6 +312,7 @@ class DataSetWindowAggregate( rowRelDataType, grouping, namedProperties, + tableConfig, isInputCombined = true) mappedInput.sortPartition(rowTimeFieldPos, Order.ASCENDING) @@ -326,7 +338,8 @@ class DataSetWindowAggregate( inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, rowRelDataType, grouping, - namedProperties) + namedProperties, + tableConfig) mappedInput.groupBy(groupingKeys: _*) .sortGroup(rowTimeFieldPos, Order.ASCENDING) @@ -343,7 +356,8 @@ class DataSetWindowAggregate( inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, rowRelDataType, grouping, - namedProperties) + namedProperties, + tableConfig) mappedInput.sortPartition(rowTimeFieldPos, Order.ASCENDING).setParallelism(1) .reduceGroup(groupReduceFunction) @@ -360,7 +374,8 @@ class DataSetWindowAggregate( isTimeWindow: Boolean, size: Long, slide: Long, - isParserCaseSensitive: Boolean) + isParserCaseSensitive: Boolean, + tableConfig: TableConfig) : DataSet[Row] = { val input = inputNode.asInstanceOf[DataSetRel] @@ -374,7 +389,8 @@ class DataSetWindowAggregate( grouping, input.getRowType, inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, - isParserCaseSensitive) + isParserCaseSensitive, + tableConfig) val mappedDataSet = inputDS .map(mapFunction) @@ -389,7 +405,8 @@ class DataSetWindowAggregate( val isPartial = doAllSupportPartialMerge( namedAggregates.map(_.getKey), inputType, - grouping.length) + grouping.length, + tableConfig) // only pre-tumble if it is worth it val isLittleTumblingSize = determineLargestTumblingSize(size, slide) <= 1 @@ -411,7 +428,8 @@ class DataSetWindowAggregate( grouping, input.getRowType, inputDS.getType.asInstanceOf[RowTypeInfo].getFieldTypes, - isParserCaseSensitive) + isParserCaseSensitive, + tableConfig) mappedDataSet.asInstanceOf[DataSet[Row]] .groupBy(groupingKeysAndAlignedRowtime: _*) @@ -451,6 +469,7 @@ class DataSetWindowAggregate( rowRelDataType, grouping, namedProperties, + tableConfig, isInputCombined = false) // gets the window-start position in the intermediate result. diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala index 71de57c942c5c8..5f4b186ca410e3 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala @@ -138,6 +138,7 @@ class DataStreamGroupAggregate( inputSchema.fieldTypeInfos, groupings, queryConfig, + tableEnv.getConfig, DataStreamRetractionRules.isAccRetract(this), DataStreamRetractionRules.isAccRetract(getInput)) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala index d527dc8b94f6cd..0a014b6bd2db75 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala @@ -207,7 +207,8 @@ class DataStreamGroupWindowAggregate( inputSchema.fieldTypeInfos, schema.relDataType, grouping, - needMerge) + needMerge, + tableEnv.getConfig) windowedStream .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, outRowType) @@ -232,7 +233,8 @@ class DataStreamGroupWindowAggregate( inputSchema.fieldTypeInfos, schema.relDataType, Array[Int](), - needMerge) + needMerge, + tableEnv.getConfig) windowedStream .aggregate(aggFunction, windowFunction, accumulatorRowType, aggResultRowType, outRowType) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala index 635c7bc2d03226..c1693d98515de0 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala @@ -28,7 +28,7 @@ import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel} import org.apache.calcite.rex.RexLiteral import org.apache.flink.api.java.functions.NullByteKeySelector import org.apache.flink.streaming.api.datastream.DataStream -import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment, TableException} +import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment, TableConfig, TableException} import org.apache.flink.table.calcite.FlinkTypeFactory import org.apache.flink.table.codegen.AggregationCodeGenerator import org.apache.flink.table.plan.nodes.OverAggregate @@ -172,6 +172,7 @@ class DataStreamOverAggregate( // unbounded OVER window createUnboundedAndCurrentRowOverWindow( queryConfig, + tableEnv.getConfig, generator, inputDS, rowTimeIdx, @@ -188,7 +189,8 @@ class DataStreamOverAggregate( inputDS, rowTimeIdx, aggregateInputType, - isRowsClause = overWindow.isRows) + isRowsClause = overWindow.isRows, + tableEnv.getConfig) } else { throw new TableException("OVER RANGE FOLLOWING windows are not supported yet.") } @@ -196,6 +198,7 @@ class DataStreamOverAggregate( def createUnboundedAndCurrentRowOverWindow( queryConfig: StreamQueryConfig, + tableConfig: TableConfig, generator: AggregationCodeGenerator, inputDS: DataStream[CRow], rowTimeIdx: Option[Int], @@ -219,6 +222,7 @@ class DataStreamOverAggregate( inputSchema.typeInfo, inputSchema.fieldTypeInfos, queryConfig, + tableConfig, rowTimeIdx, partitionKeys.nonEmpty, isRowsClause) @@ -249,7 +253,8 @@ class DataStreamOverAggregate( inputDS: DataStream[CRow], rowTimeIdx: Option[Int], aggregateInputType: RelDataType, - isRowsClause: Boolean): DataStream[CRow] = { + isRowsClause: Boolean, + tableConfig: TableConfig): DataStream[CRow] = { val overWindow: Group = logicWindow.groups.get(0) @@ -272,6 +277,7 @@ class DataStreamOverAggregate( inputSchema.fieldTypeInfos, precedingOffset, queryConfig, + tableConfig, isRowsClause, rowTimeIdx ) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala index 361a87e2147ad8..df9b1c5520467c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala @@ -32,7 +32,7 @@ import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction, WindowFunction} import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow} import org.apache.flink.table.api.dataview.DataViewSpec -import org.apache.flink.table.api.{StreamQueryConfig, TableException} +import org.apache.flink.table.api.{StreamQueryConfig, TableConfig, TableException} import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty import org.apache.flink.table.calcite.FlinkTypeFactory import org.apache.flink.table.codegen.AggregationCodeGenerator @@ -78,6 +78,7 @@ object AggregateUtil { inputTypeInfo: TypeInformation[Row], inputFieldTypeInfo: Seq[TypeInformation[_]], queryConfig: StreamQueryConfig, + tableConfig: TableConfig, rowTimeIdx: Option[Int], isPartitioned: Boolean, isRowsClause: Boolean) @@ -88,6 +89,7 @@ object AggregateUtil { namedAggregates.map(_.getKey), aggregateInputType, needRetraction = false, + tableConfig, isStateBackedDataViews = true) val aggregationStateType: RowTypeInfo = new RowTypeInfo(accTypes: _*) @@ -159,6 +161,7 @@ object AggregateUtil { inputFieldTypes: Seq[TypeInformation[_]], groupings: Array[Int], queryConfig: StreamQueryConfig, + tableConfig: TableConfig, generateRetraction: Boolean, consumeRetraction: Boolean): ProcessFunction[CRow, CRow] = { @@ -167,6 +170,7 @@ object AggregateUtil { namedAggregates.map(_.getKey), inputRowType, consumeRetraction, + tableConfig, isStateBackedDataViews = true) val aggMapping = aggregates.indices.map(_ + groupings.length).toArray @@ -223,6 +227,7 @@ object AggregateUtil { inputFieldTypeInfo: Seq[TypeInformation[_]], precedingOffset: Long, queryConfig: StreamQueryConfig, + tableConfig: TableConfig, isRowsClause: Boolean, rowTimeIdx: Option[Int]) : ProcessFunction[CRow, CRow] = { @@ -233,6 +238,7 @@ object AggregateUtil { namedAggregates.map(_.getKey), aggregateInputType, needRetract, + tableConfig, isStateBackedDataViews = true) val aggregationStateType: RowTypeInfo = new RowTypeInfo(accTypes: _*) @@ -325,14 +331,16 @@ object AggregateUtil { groupings: Array[Int], inputType: RelDataType, inputFieldTypeInfo: Seq[TypeInformation[_]], - isParserCaseSensitive: Boolean) + isParserCaseSensitive: Boolean, + tableConfig: TableConfig) : MapFunction[Row, Row] = { val needRetract = false val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions( namedAggregates.map(_.getKey), inputType, - needRetract) + needRetract, + tableConfig) val mapReturnType: RowTypeInfo = createRowTypeForKeysAndAggregates( @@ -430,14 +438,16 @@ object AggregateUtil { groupings: Array[Int], physicalInputRowType: RelDataType, physicalInputTypes: Seq[TypeInformation[_]], - isParserCaseSensitive: Boolean) + isParserCaseSensitive: Boolean, + tableConfig: TableConfig) : RichGroupReduceFunction[Row, Row] = { val needRetract = false val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions( namedAggregates.map(_.getKey), physicalInputRowType, - needRetract) + needRetract, + tableConfig) val returnType: RowTypeInfo = createRowTypeForKeysAndAggregates( groupings, @@ -543,6 +553,7 @@ object AggregateUtil { outputType: RelDataType, groupings: Array[Int], properties: Seq[NamedWindowProperty], + tableConfig: TableConfig, isInputCombined: Boolean = false) : RichGroupReduceFunction[Row, Row] = { @@ -550,7 +561,8 @@ object AggregateUtil { val (aggFieldIndexes, aggregates, _, _) = transformToAggregateFunctions( namedAggregates.map(_.getKey), physicalInputRowType, - needRetract) + needRetract, + tableConfig) val aggMapping = aggregates.indices.toArray.map(_ + groupings.length) @@ -695,13 +707,15 @@ object AggregateUtil { namedAggregates: Seq[CalcitePair[AggregateCall, String]], physicalInputRowType: RelDataType, physicalInputTypes: Seq[TypeInformation[_]], - groupings: Array[Int]): MapPartitionFunction[Row, Row] = { + groupings: Array[Int], + tableConfig: TableConfig): MapPartitionFunction[Row, Row] = { val needRetract = false val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions( namedAggregates.map(_.getKey), physicalInputRowType, - needRetract) + needRetract, + tableConfig) val aggMapping = aggregates.indices.map(_ + groupings.length).toArray @@ -767,14 +781,16 @@ object AggregateUtil { namedAggregates: Seq[CalcitePair[AggregateCall, String]], physicalInputRowType: RelDataType, physicalInputTypes: Seq[TypeInformation[_]], - groupings: Array[Int]) + groupings: Array[Int], + tableConfig: TableConfig) : GroupCombineFunction[Row, Row] = { val needRetract = false val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions( namedAggregates.map(_.getKey), physicalInputRowType, - needRetract) + needRetract, + tableConfig) val aggMapping = aggregates.indices.map(_ + groupings.length).toArray @@ -831,7 +847,8 @@ object AggregateUtil { inputType: RelDataType, inputFieldTypeInfo: Seq[TypeInformation[_]], outputType: RelDataType, - groupings: Array[Int]): ( + groupings: Array[Int], + tableConfig: TableConfig): ( Option[DataSetPreAggFunction], Option[TypeInformation[Row]], Either[DataSetAggFunction, DataSetFinalAggFunction]) = { @@ -840,7 +857,8 @@ object AggregateUtil { val (aggInFields, aggregates, accTypes, _) = transformToAggregateFunctions( namedAggregates.map(_.getKey), inputType, - needRetract) + needRetract, + tableConfig) val (gkeyOutMapping, aggOutMapping) = getOutputMappings( namedAggregates, @@ -992,7 +1010,8 @@ object AggregateUtil { inputFieldTypeInfo: Seq[TypeInformation[_]], outputType: RelDataType, groupingKeys: Array[Int], - needMerge: Boolean) + needMerge: Boolean, + tableConfig: TableConfig) : (DataStreamAggFunction[CRow, Row, Row], RowTypeInfo, RowTypeInfo) = { val needRetract = false @@ -1000,7 +1019,8 @@ object AggregateUtil { transformToAggregateFunctions( namedAggregates.map(_.getKey), inputType, - needRetract) + needRetract, + tableConfig) val aggMapping = aggregates.indices.toArray val outputArity = aggregates.length @@ -1036,12 +1056,14 @@ object AggregateUtil { private[flink] def doAllSupportPartialMerge( aggregateCalls: Seq[AggregateCall], inputType: RelDataType, - groupKeysCount: Int): Boolean = { + groupKeysCount: Int, + tableConfig: TableConfig): Boolean = { val aggregateList = transformToAggregateFunctions( aggregateCalls, inputType, - needRetraction = false)._2 + needRetraction = false, + tableConfig)._2 doAllSupportPartialMerge(aggregateList) } @@ -1121,6 +1143,7 @@ object AggregateUtil { aggregateCalls: Seq[AggregateCall], aggregateInputType: RelDataType, needRetraction: Boolean, + tableConfig: TableConfig, isStateBackedDataViews: Boolean = false) : (Array[Array[Int]], Array[TableAggregateFunction[_, _]], @@ -1251,7 +1274,7 @@ object AggregateUtil { case DOUBLE => new DoubleAvgAggFunction case DECIMAL => - new DecimalAvgAggFunction + new DecimalAvgAggFunction(tableConfig.getDecimalContext) case sqlType: SqlTypeName => throw new TableException(s"Avg aggregate does no support type: '$sqlType'") } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala index 42f800817701d6..5de6f2c5aaa24d 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/DecimalTypeTest.scala @@ -233,6 +233,13 @@ class DecimalTypeTest extends ExpressionTestBase { "-f0", "-f0", "-123456789.123456789123456789") + + testAllApis( + BigDecimal("1").toExpr / BigDecimal("3"), + "1p / 3p", + "CAST('1' AS DECIMAL) / CAST('3' AS DECIMAL)", + "0.3333333333333333333333333333333333" + ) } @Test @@ -287,7 +294,7 @@ class DecimalTypeTest extends ExpressionTestBase { // ---------------------------------------------------------------------------------------------- - def testData = { + def testData: Row = { val testData = new Row(6) testData.setField(0, BigDecimal("123456789.123456789123456789").bigDecimal) testData.setField(1, BigDecimal("123456789123456789123456789").bigDecimal) @@ -298,7 +305,7 @@ class DecimalTypeTest extends ExpressionTestBase { testData } - def typeInfo = { + def typeInfo: TypeInformation[Any] = { new RowTypeInfo( Types.DECIMAL, Types.DECIMAL, diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggfunctions/AvgFunctionTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggfunctions/AvgFunctionTest.scala index 0671b404306844..d413c6c0c46f12 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggfunctions/AvgFunctionTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/aggfunctions/AvgFunctionTest.scala @@ -18,7 +18,7 @@ package org.apache.flink.table.runtime.aggfunctions -import java.math.BigDecimal +import java.math.{BigDecimal, MathContext} import org.apache.flink.table.functions.AggregateFunction import org.apache.flink.table.functions.aggfunctions._ @@ -178,17 +178,23 @@ class DecimalAvgAggFunctionTest extends AggFunctionTestBase[BigDecimal, DecimalA null, null, null + ), + Seq( + new BigDecimal("0.3"), + new BigDecimal("0.3"), + new BigDecimal("0.4") ) ) override def expectedResults: Seq[BigDecimal] = Seq( BigDecimal.ZERO, BigDecimal.ONE, - null + null, + BigDecimal.ONE.divide(new BigDecimal("3"), MathContext.DECIMAL128) ) override def aggregator: AggregateFunction[BigDecimal, DecimalAvgAccumulator] = - new DecimalAvgAggFunction() + new DecimalAvgAggFunction(MathContext.DECIMAL128) override def retractFunc = aggregator.getClass.getMethod("retract", accType, classOf[Any]) } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala index 1b89229fb9e465..aa37d1bd43dcdb 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CalcITCase.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime.batch.table +import java.math.MathContext import java.sql.{Date, Time, Timestamp} import java.util @@ -41,6 +42,7 @@ import org.junit.runners.Parameterized import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.math.BigDecimal.RoundingMode @RunWith(classOf[Parameterized]) class CalcITCase( @@ -330,6 +332,7 @@ class CalcITCase( def testAdvancedDataTypes(): Unit = { val env = ExecutionEnvironment.getExecutionEnvironment val tEnv = TableEnvironment.getTableEnvironment(env, config) + tEnv.getConfig.setDecimalContext(new MathContext(30)) val t = env .fromElements(( @@ -341,10 +344,11 @@ class CalcITCase( .toTable(tEnv, 'a, 'b, 'c, 'd, 'e) .select('a, 'b, 'c, 'd, 'e, BigDecimal("11.2"), BigDecimal("11.2").bigDecimal, Date.valueOf("1984-07-12"), Time.valueOf("14:34:24"), - Timestamp.valueOf("1984-07-12 14:34:24")) + Timestamp.valueOf("1984-07-12 14:34:24"), + BigDecimal("1").toExpr / BigDecimal("3")) val expected = "78.454654654654654,4E+9999,1984-07-12,14:34:24,1984-07-12 14:34:24.0," + - "11.2,11.2,1984-07-12,14:34:24,1984-07-12 14:34:24.0" + "11.2,11.2,1984-07-12,14:34:24,1984-07-12 14:34:24.0,0.333333333333333333333333333333" val results = t.toDataSet[Row].collect() TestBaseUtils.compareResultAsText(results.asJava, expected) } From 2d3c5d597359f51f310f9eeb994c55f516eb1a30 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 2 Mar 2018 20:19:38 +0100 Subject: [PATCH 0092/2294] [FLINK-8842] Change the Rest default port to 8081 This closes #5626. --- .../flink/configuration/RestOptions.java | 2 +- .../entrypoint/ClusterConfiguration.java | 9 +++++++- .../runtime/entrypoint/ClusterEntrypoint.java | 22 +++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java index a66b27c7bb515f..888be082398888 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java @@ -41,7 +41,7 @@ public class RestOptions { */ public static final ConfigOption REST_PORT = key("rest.port") - .defaultValue(9065) + .defaultValue(8081) .withDescription("The port that the server listens on / the client connects to."); /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterConfiguration.java index dbec0b6a9eff9d..7f8b5096d136c0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterConfiguration.java @@ -27,11 +27,18 @@ public class ClusterConfiguration { private final String configDir; - public ClusterConfiguration(String configDir) { + private final int restPort; + + public ClusterConfiguration(String configDir, int restPort) { this.configDir = Preconditions.checkNotNull(configDir); + this.restPort = restPort; } public String getConfigDir() { return configDir; } + + public int getRestPort() { + return restPort; + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index ba286703412c7b..19781f8ee6471e 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -25,6 +25,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.WebOptions; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.runtime.akka.AkkaUtils; @@ -594,11 +595,28 @@ protected static ClusterConfiguration parseArguments(String[] args) { final String configDir = parameterTool.get("configDir", ""); - return new ClusterConfiguration(configDir); + final int restPort; + + final String portKey = "webui-port"; + if (parameterTool.has(portKey)) { + restPort = Integer.valueOf(parameterTool.get(portKey)); + } else { + restPort = -1; + } + + return new ClusterConfiguration(configDir, restPort); } protected static Configuration loadConfiguration(ClusterConfiguration clusterConfiguration) { - return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir()); + final Configuration configuration = GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir()); + + final int restPort = clusterConfiguration.getRestPort(); + + if (restPort >= 0) { + configuration.setInteger(RestOptions.REST_PORT, restPort); + } + + return configuration; } /** From 344a47756fb2723a1f5fda2b67315c27f2125423 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 2 Mar 2018 20:31:36 +0100 Subject: [PATCH 0093/2294] [hotfix] Enable standalone HA mode by choosing HA port range --- .../flink/runtime/entrypoint/ClusterEntrypoint.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 19781f8ee6471e..07b3b683a3f289 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -24,6 +24,7 @@ import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.configuration.HighAvailabilityOptions; import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.WebOptions; @@ -59,6 +60,7 @@ import org.apache.flink.runtime.security.SecurityConfiguration; import org.apache.flink.runtime.security.SecurityContext; import org.apache.flink.runtime.security.SecurityUtils; +import org.apache.flink.runtime.util.ZooKeeperUtils; import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; import org.apache.flink.runtime.webmonitor.retriever.LeaderGatewayRetriever; import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever; @@ -354,7 +356,11 @@ protected void startClusterComponents( * @return Port range for the common {@link RpcService} */ protected String getRPCPortRange(Configuration configuration) { - return String.valueOf(configuration.getInteger(JobManagerOptions.PORT)); + if (ZooKeeperUtils.isZooKeeperRecoveryMode(configuration)) { + return configuration.getString(HighAvailabilityOptions.HA_JOB_MANAGER_PORT_RANGE); + } else { + return String.valueOf(configuration.getInteger(JobManagerOptions.PORT)); + } } protected RpcService createRpcService( From 193386bcd5d0e2003a831138d6282af1880e1ab8 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 2 Mar 2018 15:27:13 +0100 Subject: [PATCH 0094/2294] [FLINK-8840] [yarn] Pull YarnClient and YarnConfiguration instantiation out of AbstractYarnClusterClient For better testability, this commit moves the YarnClient and YarnConfiguration out of the AbstractYarnClusterDescriptor. --- .../yarn/TestingYarnClusterDescriptor.java | 12 ++- .../yarn/YARNHighAvailabilityITCase.java | 7 +- .../org/apache/flink/yarn/YARNITCase.java | 6 +- .../flink/yarn/YARNSessionFIFOITCase.java | 5 +- .../org/apache/flink/yarn/YarnTestBase.java | 10 ++- .../yarn/AbstractYarnClusterDescriptor.java | 16 ++-- .../yarn/Flip6YarnClusterDescriptor.java | 12 ++- .../flink/yarn/YarnClusterDescriptor.java | 12 ++- .../flink/yarn/cli/FlinkYarnSessionCli.java | 30 ++++++- .../flink/yarn/AbstractYarnClusterTest.java | 20 ++++- .../flink/yarn/YarnClusterDescriptorTest.java | 79 ++++++++++++++++--- 11 files changed, 175 insertions(+), 34 deletions(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java index ec41d8e12b9bb5..4d2aaa02efe342 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnClusterDescriptor.java @@ -24,6 +24,7 @@ import org.apache.flink.util.Preconditions; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import java.io.File; import java.io.FilenameFilter; @@ -37,11 +38,18 @@ */ public class TestingYarnClusterDescriptor extends YarnClusterDescriptor { - public TestingYarnClusterDescriptor(Configuration configuration, String configurationDirectory) { + public TestingYarnClusterDescriptor( + Configuration configuration, + YarnConfiguration yarnConfiguration, + String configurationDirectory, + YarnClient yarnClient, + boolean sharedYarnClient) { super( configuration, + yarnConfiguration, configurationDirectory, - YarnClient.createYarnClient()); + yarnClient, + sharedYarnClient); List filesToShip = new ArrayList<>(); File testingJar = YarnTestBase.findFile("..", new TestJarFinder("flink-yarn-tests")); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java index 05be03a1755616..f9c03f937849fa 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java @@ -110,7 +110,12 @@ public void testMultipleAMKill() throws Exception { final int numberKillingAttempts = numberApplicationAttempts - 1; String confDirPath = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR); final Configuration configuration = GlobalConfiguration.loadConfiguration(); - TestingYarnClusterDescriptor flinkYarnClient = new TestingYarnClusterDescriptor(configuration, confDirPath); + TestingYarnClusterDescriptor flinkYarnClient = new TestingYarnClusterDescriptor( + configuration, + getYarnConfiguration(), + confDirPath, + getYarnClient(), + true); Assert.assertNotNull("unable to get yarn client", flinkYarnClient); flinkYarnClient.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath())); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java index 037e086ae85832..ef6706ad5fe36c 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java @@ -55,12 +55,14 @@ public static void setup() { public void testPerJobMode() throws Exception { Configuration configuration = new Configuration(); configuration.setString(AkkaOptions.ASK_TIMEOUT, "30 s"); - final YarnClient yarnClient = YarnClient.createYarnClient(); + final YarnClient yarnClient = getYarnClient(); try (final Flip6YarnClusterDescriptor flip6YarnClusterDescriptor = new Flip6YarnClusterDescriptor( configuration, + getYarnConfiguration(), System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR), - yarnClient)) { + yarnClient, + true)) { flip6YarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath())); flip6YarnClusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles())); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java index e54518793526da..b3dcaca1459656 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java @@ -231,12 +231,13 @@ public void testJavaAPI() throws Exception { String confDirPath = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR); Configuration configuration = GlobalConfiguration.loadConfiguration(); - final YarnClient yarnClient = YarnClient.createYarnClient(); try (final AbstractYarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor( configuration, + getYarnConfiguration(), confDirPath, - yarnClient)) { + getYarnClient(), + true)) { Assert.assertNotNull("unable to get yarn client", clusterDescriptor); clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath())); clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles())); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index 7bca32192489b1..b74a1557bb22b6 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -128,7 +128,7 @@ public abstract class YarnTestBase extends TestLogger { */ protected static File flinkUberjar; - protected static final Configuration YARN_CONFIGURATION; + protected static final YarnConfiguration YARN_CONFIGURATION; /** * lib/ folder of the flink distribution. @@ -213,6 +213,14 @@ public void checkClusterEmpty() throws IOException, YarnException { flip6 = CoreOptions.FLIP6_MODE.equalsIgnoreCase(flinkConfiguration.getString(CoreOptions.MODE)); } + protected YarnClient getYarnClient() { + return yarnClient; + } + + protected static YarnConfiguration getYarnConfiguration() { + return YARN_CONFIGURATION; + } + /** * Locate a file or directory. */ diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java index 6b930163896c00..bdb59b11b5fd7c 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java @@ -118,6 +118,9 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor private final YarnClient yarnClient; + /** True if the descriptor must not shut down the YarnClient. */ + private final boolean sharedYarnClient; + private String yarnQueue; private String configurationDirectory; @@ -145,10 +148,12 @@ public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor public AbstractYarnClusterDescriptor( Configuration flinkConfiguration, + YarnConfiguration yarnConfiguration, String configurationDirectory, - YarnClient yarnClient) { + YarnClient yarnClient, + boolean sharedYarnClient) { - yarnConfiguration = new YarnConfiguration(); + this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration); // for unit tests only if (System.getenv("IN_TESTS") != null) { @@ -160,8 +165,7 @@ public AbstractYarnClusterDescriptor( } this.yarnClient = Preconditions.checkNotNull(yarnClient); - yarnClient.init(yarnConfiguration); - yarnClient.start(); + this.sharedYarnClient = sharedYarnClient; this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration); userJarInclusion = getUserJarInclusionMode(flinkConfiguration); @@ -328,7 +332,9 @@ public void setZookeeperNamespace(String zookeeperNamespace) { @Override public void close() { - yarnClient.stop(); + if (!sharedYarnClient) { + yarnClient.stop(); + } } // ------------------------------------------------------------- diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java index 461dd555a45e7a..9860363c00e32e 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; /** * Implementation of {@link org.apache.flink.yarn.AbstractYarnClusterDescriptor} which is used to start the @@ -39,9 +40,16 @@ public class Flip6YarnClusterDescriptor extends AbstractYarnClusterDescriptor { public Flip6YarnClusterDescriptor( Configuration flinkConfiguration, + YarnConfiguration yarnConfiguration, String configurationDirectory, - YarnClient yarnCLient) { - super(flinkConfiguration, configurationDirectory, yarnCLient); + YarnClient yarnClient, + boolean sharedYarnClient) { + super( + flinkConfiguration, + yarnConfiguration, + configurationDirectory, + yarnClient, + sharedYarnClient); } @Override diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index a5254a0e3a376a..8625cee8240c64 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; /** * Default implementation of {@link AbstractYarnClusterDescriptor} which starts an {@link YarnApplicationMasterRunner}. @@ -34,9 +35,16 @@ public class YarnClusterDescriptor extends AbstractYarnClusterDescriptor { public YarnClusterDescriptor( Configuration flinkConfiguration, + YarnConfiguration yarnConfiguration, String configurationDirectory, - YarnClient yarnClient) { - super(flinkConfiguration, configurationDirectory, yarnClient); + YarnClient yarnClient, + boolean sharedYarnClient) { + super( + flinkConfiguration, + yarnConfiguration, + configurationDirectory, + yarnClient, + sharedYarnClient); } @Override diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index e4e3dbd5566cb1..7773600dabd716 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.slf4j.Logger; @@ -159,6 +160,8 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine Date: Thu, 1 Mar 2018 20:09:55 +0100 Subject: [PATCH 0095/2294] [hotfix] [flip6] Harden JobMaster#triggerSavepoint Check first whether the CheckpointCoordinator has been set before triggering a savepoint. If it has not been set, then return a failure message. --- .../flink/runtime/jobmaster/JobMaster.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index bfe8aaaa2424a8..cc4cdbae15aaac 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -971,14 +971,20 @@ public CompletableFuture requestJob(Time timeout) { @Override public CompletableFuture triggerSavepoint( - @Nullable final String targetDirectory, - final Time timeout) { - try { - return executionGraph.getCheckpointCoordinator() + @Nullable final String targetDirectory, + final Time timeout) { + final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); + + if (checkpointCoordinator != null) { + return checkpointCoordinator .triggerSavepoint(System.currentTimeMillis(), targetDirectory) .thenApply(CompletedCheckpoint::getExternalPointer); - } catch (Exception e) { - return FutureUtils.completedExceptionally(e); + } else { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format( + "Cannot trigger a savepoint because the job %s is not a streaming job.", + jobGraph.getJobID()))); } } From 0424cbfcdd8005d18eeaf225001d5a5e7be25b67 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 1 Mar 2018 23:35:25 +0100 Subject: [PATCH 0096/2294] [FLINK-8826] [flip6] Start Yarn TaskExecutor with proper slots and memory Read the default TaskManager memory and number of slots from the configuration when the YarnResourceManager is started. This closes #5625. --- flink-end-to-end-tests/test-scripts/common.sh | 3 + flink-yarn-tests/pom.xml | 8 + .../flink/yarn/YarnConfigurationITCase.java | 194 ++++++++++++++++++ .../org/apache/flink/yarn/YarnTestBase.java | 31 ++- .../yarn/AbstractYarnClusterDescriptor.java | 8 +- .../flink/yarn/YarnResourceManager.java | 27 +-- .../flink/yarn/FlinkYarnSessionCliTest.java | 64 ++++++ 7 files changed, 312 insertions(+), 23 deletions(-) create mode 100644 flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index 7492f365afde16..e6d21a26b0907d 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -75,6 +75,7 @@ function stop_cluster { | grep -v "RejectedExecutionException" \ | grep -v "An exception was thrown by an exception handler" \ | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/exceptions/YarnException" \ + | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration" \ | grep -iq "error"; then echo "Found error in log files:" cat $FLINK_DIR/log/* @@ -92,7 +93,9 @@ function stop_cluster { | grep -v "RejectedExecutionException" \ | grep -v "An exception was thrown by an exception handler" \ | grep -v "Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.exceptions.YarnException" \ + | grep -v "Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configuration" \ | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/exceptions/YarnException" \ + | grep -v "java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration" \ | grep -iq "exception"; then echo "Found exception in log files:" cat $FLINK_DIR/log/* diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml index b5a86b4b8f2f72..0b05b287b2434b 100644 --- a/flink-yarn-tests/pom.xml +++ b/flink-yarn-tests/pom.xml @@ -378,6 +378,14 @@ under the License. true StreamingWordCount.jar + + org.apache.flink + flink-examples-streaming_${scala.binary.version} + jar + WindowJoin + true + WindowJoin.jar + ${project.build.directory}/programs false diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java new file mode 100644 index 00000000000000..2a1b099399ac0b --- /dev/null +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.client.cli.CliFrontend; +import org.apache.flink.client.deployment.ClusterSpecification; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.client.program.PackagedProgram; +import org.apache.flink.client.program.PackagedProgramUtils; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.ResourceManagerOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.rest.RestClient; +import org.apache.flink.runtime.rest.RestClientConfiguration; +import org.apache.flink.runtime.rest.messages.EmptyMessageParameters; +import org.apache.flink.runtime.rest.messages.EmptyRequestBody; +import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo; +import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagersHeaders; +import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagersInfo; +import org.apache.flink.runtime.testingUtils.TestingUtils; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * Test cases which ensure that the Yarn containers are started with the correct + * settings. + */ +public class YarnConfigurationITCase extends YarnTestBase { + + private static final Time TIMEOUT = Time.seconds(10L); + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + /** + * Tests that the Flink components are started with the correct + * memory settings. + */ + @Test(timeout = 60000) + public void testFlinkContainerMemory() throws Exception { + final YarnClient yarnClient = getYarnClient(); + final Configuration configuration = new Configuration(flinkConfiguration); + + final int masterMemory = 64; + final int taskManagerMemory = 128; + final int slotsPerTaskManager = 3; + + // disable heap cutoff min + configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0); + configuration.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, (1L << 20)); + configuration.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, (4L << 20)); + + final YarnConfiguration yarnConfiguration = getYarnConfiguration(); + final Flip6YarnClusterDescriptor clusterDescriptor = new Flip6YarnClusterDescriptor( + configuration, + yarnConfiguration, + CliFrontend.getConfigurationDirectoryFromEnv(), + yarnClient, + true); + + clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath())); + clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles())); + + final File streamingWordCountFile = new File("target/programs/WindowJoin.jar"); + + assertThat(streamingWordCountFile.exists(), is(true)); + + final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile); + final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1); + + try { + final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder() + .setMasterMemoryMB(masterMemory) + .setTaskManagerMemoryMB(taskManagerMemory) + .setSlotsPerTaskManager(slotsPerTaskManager) + .createClusterSpecification(); + + final ClusterClient clusterClient = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true); + + final ApplicationId clusterId = clusterClient.getClusterId(); + + final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration), TestingUtils.defaultExecutor()); + + try { + final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId); + + final ApplicationAttemptId currentApplicationAttemptId = applicationReport.getCurrentApplicationAttemptId(); + + // wait until we have second container allocated + List containers = yarnClient.getContainers(currentApplicationAttemptId); + + while (containers.size() < 2) { + // this is nasty but Yarn does not offer a better way to wait + Thread.sleep(50L); + containers = yarnClient.getContainers(currentApplicationAttemptId); + } + + for (ContainerReport container : containers) { + if (container.getContainerId().getId() == 1) { + // this should be the application master + assertThat(container.getAllocatedResource().getMemory(), is(masterMemory)); + } else { + assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory)); + } + } + + final URI webURI = new URI(clusterClient.getWebInterfaceURL()); + + CompletableFuture taskManagersInfoCompletableFuture; + Collection taskManagerInfos; + + while (true) { + taskManagersInfoCompletableFuture = restClient.sendRequest( + webURI.getHost(), + webURI.getPort(), + TaskManagersHeaders.getInstance(), + EmptyMessageParameters.getInstance(), + EmptyRequestBody.getInstance()); + + final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get(); + + taskManagerInfos = taskManagersInfo.getTaskManagerInfos(); + + if (taskManagerInfos.isEmpty()) { + Thread.sleep(100L); + } else { + break; + } + } + + // there should be at least one TaskManagerInfo + final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next(); + + assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager)); + + final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters.create( + configuration, + taskManagerMemory, + slotsPerTaskManager); + + final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L; + + assertThat((double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize, is(closeTo(1.0, 0.1))); + } finally { + restClient.shutdown(TIMEOUT); + clusterClient.shutdown(); + } + + clusterDescriptor.terminateCluster(clusterId); + + } finally { + clusterDescriptor.close(); + } + } +} diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index b74a1557bb22b6..3ec805e5058c5c 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -55,6 +55,8 @@ import org.slf4j.Marker; import org.slf4j.MarkerFactory; +import javax.annotation.Nullable; + import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; @@ -73,6 +75,7 @@ import java.util.List; import java.util.Map; import java.util.Scanner; +import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; @@ -140,9 +143,15 @@ public abstract class YarnTestBase extends TestLogger { */ protected static File tempConfPathForSecureRun = null; + private YarnClient yarnClient = null; + + protected org.apache.flink.configuration.Configuration flinkConfiguration; + + protected boolean flip6; + static { YARN_CONFIGURATION = new YarnConfiguration(); - YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 32); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 4096); // 4096 is the available memory anyways YARN_CONFIGURATION.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); YARN_CONFIGURATION.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); @@ -186,15 +195,11 @@ public void sleep() { } } - private YarnClient yarnClient = null; - protected org.apache.flink.configuration.Configuration flinkConfiguration; - protected boolean flip6; - @Before public void checkClusterEmpty() throws IOException, YarnException { if (yarnClient == null) { yarnClient = YarnClient.createYarnClient(); - yarnClient.init(YARN_CONFIGURATION); + yarnClient.init(getYarnConfiguration()); yarnClient.start(); } @@ -213,6 +218,7 @@ public void checkClusterEmpty() throws IOException, YarnException { flip6 = CoreOptions.FLIP6_MODE.equalsIgnoreCase(flinkConfiguration.getString(CoreOptions.MODE)); } + @Nullable protected YarnClient getYarnClient() { return yarnClient; } @@ -409,15 +415,15 @@ public static int getRunningContainers() { return count; } - public static void startYARNSecureMode(Configuration conf, String principal, String keytab) { + public static void startYARNSecureMode(YarnConfiguration conf, String principal, String keytab) { start(conf, principal, keytab); } - public static void startYARNWithConfig(Configuration conf) { + public static void startYARNWithConfig(YarnConfiguration conf) { start(conf, null, null); } - private static void start(Configuration conf, String principal, String keytab) { + private static void start(YarnConfiguration conf, String principal, String keytab) { // set the home directory to a temp directory. Flink on YARN is using the home dir to distribute the file File homeDir = null; try { @@ -444,7 +450,12 @@ private static void start(Configuration conf, String principal, String keytab) { try { LOG.info("Starting up MiniYARNCluster"); if (yarnCluster == null) { - yarnCluster = new MiniYARNCluster(conf.get(YarnTestBase.TEST_CLUSTER_NAME_KEY), NUM_NODEMANAGERS, 1, 1); + final String testName = conf.get(YarnTestBase.TEST_CLUSTER_NAME_KEY); + yarnCluster = new MiniYARNCluster( + testName == null ? "YarnTest_" + UUID.randomUUID() : testName, + NUM_NODEMANAGERS, + 1, + 1); yarnCluster.init(conf); yarnCluster.start(); diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java index bdb59b11b5fd7c..bdb471a142f96e 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java @@ -33,6 +33,7 @@ import org.apache.flink.configuration.ResourceManagerOptions; import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.SecurityOptions; +import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.clusterframework.BootstrapTools; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; @@ -800,10 +801,15 @@ public ApplicationReport startAppMaster( homeDir, ""); + // set the right configuration values for the TaskManager configuration.setInteger( - ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, + TaskManagerOptions.NUM_TASK_SLOTS, clusterSpecification.getSlotsPerTaskManager()); + configuration.setInteger( + TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, + clusterSpecification.getTaskManagerMemoryMB()); + // Upload the flink configuration // write out configuration file File tmpConfigurationFile = File.createTempFile(appId + "-flink-conf.yaml", null); diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index f3ec04bf1e6e0f..46ef81bed17071 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -20,6 +20,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.clusterframework.ApplicationStatus; import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; import org.apache.flink.runtime.clusterframework.types.ResourceID; @@ -75,18 +76,9 @@ public class YarnResourceManager extends ResourceManager impleme /** YARN container map. Package private for unit test purposes. */ final ConcurrentMap workerNodeMap; - /** The default registration timeout for task executor in seconds. */ - private static final int DEFAULT_TASK_MANAGER_REGISTRATION_DURATION = 300; - /** The heartbeat interval while the resource master is waiting for containers. */ private static final int FAST_YARN_HEARTBEAT_INTERVAL_MS = 500; - /** The default heartbeat interval during regular operation. */ - private static final int DEFAULT_YARN_HEARTBEAT_INTERVAL_MS = 5000; - - /** The default memory of task executor to allocate (in MB). */ - private static final int DEFAULT_TSK_EXECUTOR_MEMORY_SIZE = 1024; - /** Environment variable name of the final container id used by the YarnResourceManager. * Container ID generation may vary across Hadoop versions. */ static final String ENV_FLINK_CONTAINER_ID = "_FLINK_CONTAINER_ID"; @@ -105,6 +97,12 @@ public class YarnResourceManager extends ResourceManager impleme @Nullable private final String webInterfaceUrl; + private final int defaultTaskManagerMemoryMB; + + private final int defaultNumSlots; + + private final int defaultCpus; + /** Client to communicate with the Resource Manager (YARN's master). */ private AMRMClientAsync resourceManagerClient; @@ -163,6 +161,9 @@ public YarnResourceManager( numPendingContainerRequests = 0; this.webInterfaceUrl = webInterfaceUrl; + this.defaultTaskManagerMemoryMB = flinkConfig.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY); + this.defaultNumSlots = flinkConfig.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); + this.defaultCpus = flinkConfig.getInteger(YarnConfigOptions.VCORES, defaultNumSlots); } protected AMRMClientAsync createAndStartResourceManagerClient( @@ -285,8 +286,8 @@ public void startNewWorker(ResourceProfile resourceProfile) { // Priority for worker containers - priorities are intra-application //TODO: set priority according to the resource allocated Priority priority = Priority.newInstance(generatePriority(resourceProfile)); - int mem = resourceProfile.getMemoryInMB() < 0 ? DEFAULT_TSK_EXECUTOR_MEMORY_SIZE : (int) resourceProfile.getMemoryInMB(); - int vcore = resourceProfile.getCpuCores() < 1 ? 1 : (int) resourceProfile.getCpuCores(); + int mem = resourceProfile.getMemoryInMB() < 0 ? defaultTaskManagerMemoryMB : (int) resourceProfile.getMemoryInMB(); + int vcore = resourceProfile.getCpuCores() < 1 ? defaultCpus : (int) resourceProfile.getCpuCores(); Resource capability = Resource.newInstance(mem, vcore); requestYarnContainer(capability, priority); } @@ -445,8 +446,10 @@ private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource // init the ContainerLaunchContext final String currDir = env.get(ApplicationConstants.Environment.PWD.key()); + final int numSlots = flinkConfig.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); + final ContaineredTaskManagerParameters taskManagerParameters = - ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), 1); + ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numSlots); log.info("TaskExecutor {} will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB", diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java index 0a02474243a16b..20ce314399f5bc 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java @@ -21,6 +21,8 @@ import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.HighAvailabilityOptions; +import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.util.FlinkException; import org.apache.flink.util.TestLogger; import org.apache.flink.yarn.cli.FlinkYarnSessionCli; @@ -42,7 +44,9 @@ import java.nio.file.StandardOpenOption; import java.util.Map; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -249,6 +253,66 @@ public void testYarnIDOverridesPropertiesFile() throws Exception { assertEquals(TEST_YARN_APPLICATION_ID_2, clusterId); } + /** + * Tests that the command line arguments override the configuration settings + * when the {@link ClusterSpecification} is created. + */ + @Test + public void testCommandLineClusterSpecification() throws Exception { + final Configuration configuration = new Configuration(); + configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, 1337); + configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, 7331); + configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 2); + + final int jobManagerMemory = 42; + final int taskManagerMemory = 41; + final int slotsPerTaskManager = 30; + final String[] args = {"-yjm", String.valueOf(jobManagerMemory), "-ytm", String.valueOf(taskManagerMemory), "-ys", String.valueOf(slotsPerTaskManager)}; + final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( + configuration, + tmp.getRoot().getAbsolutePath(), + "y", + "yarn"); + + CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); + + final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); + + assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory)); + assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory)); + assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager)); + } + + /** + * Tests that the configuration settings are used to create the + * {@link ClusterSpecification}. + */ + @Test + public void testConfigurationClusterSpecification() throws Exception { + final Configuration configuration = new Configuration(); + final int jobManagerMemory = 1337; + configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory); + final int taskManagerMemory = 7331; + configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory); + final int slotsPerTaskManager = 42; + configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager); + + final String[] args = {}; + final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( + configuration, + tmp.getRoot().getAbsolutePath(), + "y", + "yarn"); + + CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); + + final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); + + assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory)); + assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory)); + assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager)); + } + /////////// // Utils // /////////// From ebcc37bd06386ff652c6794630a9494a11b5a2a0 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 2 Mar 2018 12:18:05 +0100 Subject: [PATCH 0097/2294] [hotfix] Set default number of TaskManagers in FlinkYarnSessionCli for Flip6 --- .../apache/flink/yarn/cli/FlinkYarnSessionCli.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index 7773600dabd716..2cdc19d3c93600 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -378,7 +378,14 @@ private ClusterSpecification createClusterSpecification(Configuration configurat throw new IllegalArgumentException("Missing required argument " + container.getOpt()); } - int numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); + // TODO: The number of task manager should be deprecated soon + final int numberTaskManagers; + + if (cmd.hasOption(container.getOpt())) { + numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); + } else { + numberTaskManagers = 1; + } // JobManager Memory final int jobManagerMemoryMB = configuration.getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY); @@ -386,7 +393,7 @@ private ClusterSpecification createClusterSpecification(Configuration configurat // Task Managers memory final int taskManagerMemoryMB = configuration.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY); - int slotsPerTaskManager = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); + int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) From 51fab817af2c52478f532324411917ed74dd1b8b Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 2 Mar 2018 12:42:43 +0100 Subject: [PATCH 0098/2294] [hotfix] Print correct web monitor URL in FlinkYarnSessionCli --- .../flink/client/program/rest/RestClusterClient.java | 10 +++++++++- .../java/org/apache/flink/util/ExceptionUtils.java | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 976f2a4db31ca1..5a0936d16ca3db 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -101,6 +101,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -525,7 +526,14 @@ public void waitForClusterToBeReady() { @Override public String getWebInterfaceURL() { - return getWebMonitorBaseUrl().toString(); + try { + return getWebMonitorBaseUrl().get().toString(); + } catch (InterruptedException | ExecutionException e) { + ExceptionUtils.checkInterrupted(e); + + log.warn("Could not retrieve the web interface URL for the cluster.", e); + return "Unknown address."; + } } @Override diff --git a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java index 42deb69cf36329..b9a21ae3495287 100644 --- a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java +++ b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java @@ -398,6 +398,18 @@ public static void tryDeserializeAndThrow(Throwable throwable, ClassLoader class } } + /** + * Checks whether the given exception is a {@link InterruptedException} and sets + * the interrupted flag accordingly. + * + * @param e to check whether it is an {@link InterruptedException} + */ + public static void checkInterrupted(Throwable e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + // ------------------------------------------------------------------------ // Lambda exception utilities // ------------------------------------------------------------------------ From 25ac971d39e0ed58965334a6735396d5d10e7278 Mon Sep 17 00:00:00 2001 From: gyao Date: Fri, 2 Mar 2018 15:11:36 +0100 Subject: [PATCH 0099/2294] [FLINK-8459][flip6] Implement RestClusterClient.cancelWithSavepoint Introduce cancelJob flag to existing triggerSavepoint methods in Dispatcher and JobMaster. Stop checkpoint scheduler before taking savepoint to make sure that the savepoint created by this command is the last one. This closes #5622. --- .../client/program/MiniClusterClient.java | 2 +- .../program/rest/RestClusterClient.java | 15 +- .../flink/runtime/dispatcher/Dispatcher.java | 3 +- .../executiongraph/ExecutionGraph.java | 1 + .../flink/runtime/jobmaster/JobMaster.java | 45 +++- .../runtime/jobmaster/JobMasterGateway.java | 1 + .../runtime/minicluster/MiniCluster.java | 11 + .../job/savepoints/SavepointHandlers.java | 3 +- .../SavepointTriggerRequestBody.java | 12 +- .../runtime/webmonitor/RestfulGateway.java | 1 + .../utils/TestingJobMasterGateway.java | 4 +- ...ractAsynchronousOperationHandlersTest.java | 2 +- .../job/savepoints/SavepointHandlersTest.java | 2 +- .../SavepointTriggerRequestBodyTest.java | 25 +- .../webmonitor/TestingRestfulGateway.java | 2 +- .../JobMasterTriggerSavepointIT.java | 219 ++++++++++++++++++ 16 files changed, 323 insertions(+), 25 deletions(-) create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointIT.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index aca75e0c15087c..dd99f0dabeb998 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -87,7 +87,7 @@ public void cancel(JobID jobId) throws Exception { @Override public String cancelWithSavepoint(JobID jobId, @Nullable String savepointDirectory) throws Exception { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + return miniCluster.triggerSavepoint(jobId, savepointDirectory, true).get(); } @Override diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 5a0936d16ca3db..560a10e955072f 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -354,24 +354,29 @@ public void cancel(JobID jobID) throws Exception { @Override public String cancelWithSavepoint(JobID jobId, @Nullable String savepointDirectory) throws Exception { - throw new UnsupportedOperationException("Not implemented yet."); + return triggerSavepoint(jobId, savepointDirectory, true).get(); } @Override public CompletableFuture triggerSavepoint( final JobID jobId, final @Nullable String savepointDirectory) throws FlinkException { + return triggerSavepoint(jobId, savepointDirectory, false); + } + + private CompletableFuture triggerSavepoint( + final JobID jobId, + final @Nullable String savepointDirectory, + final boolean cancelJob) { final SavepointTriggerHeaders savepointTriggerHeaders = SavepointTriggerHeaders.getInstance(); final SavepointTriggerMessageParameters savepointTriggerMessageParameters = savepointTriggerHeaders.getUnresolvedMessageParameters(); savepointTriggerMessageParameters.jobID.resolve(jobId); - final CompletableFuture responseFuture; - - responseFuture = sendRequest( + final CompletableFuture responseFuture = sendRequest( savepointTriggerHeaders, savepointTriggerMessageParameters, - new SavepointTriggerRequestBody(savepointDirectory)); + new SavepointTriggerRequestBody(savepointDirectory, cancelJob)); return responseFuture.thenCompose(savepointTriggerResponseBody -> { final TriggerId savepointTriggerId = savepointTriggerResponseBody.getTriggerId(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java index 7a11cf08784188..9b2411c66b789c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java @@ -487,11 +487,12 @@ public CompletableFuture getBlobServerPort(Time timeout) { public CompletableFuture triggerSavepoint( final JobID jobId, final String targetDirectory, + final boolean cancelJob, final Time timeout) { if (jobManagerRunners.containsKey(jobId)) { return jobManagerRunners.get(jobId) .getJobManagerGateway() - .triggerSavepoint(targetDirectory, timeout); + .triggerSavepoint(targetDirectory, cancelJob, timeout); } else { return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId)); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java index 4e8b972bc07860..ee23884d3a6f77 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java @@ -524,6 +524,7 @@ public void enableCheckpointing( } } + @Nullable public CheckpointCoordinator getCheckpointCoordinator() { return checkpointCoordinator; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index cc4cdbae15aaac..74f9b656c6f0ed 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -972,19 +972,43 @@ public CompletableFuture requestJob(Time timeout) { @Override public CompletableFuture triggerSavepoint( @Nullable final String targetDirectory, + final boolean cancelJob, final Time timeout) { + final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); + if (checkpointCoordinator == null) { + return FutureUtils.completedExceptionally(new IllegalStateException( + String.format("Job %s is not a streaming job.", jobGraph.getJobID()))); + } - if (checkpointCoordinator != null) { - return checkpointCoordinator - .triggerSavepoint(System.currentTimeMillis(), targetDirectory) - .thenApply(CompletedCheckpoint::getExternalPointer); - } else { - return FutureUtils.completedExceptionally( - new FlinkException( - String.format( - "Cannot trigger a savepoint because the job %s is not a streaming job.", - jobGraph.getJobID()))); + if (cancelJob) { + checkpointCoordinator.stopCheckpointScheduler(); + } + return checkpointCoordinator + .triggerSavepoint(System.currentTimeMillis(), targetDirectory) + .thenApply(CompletedCheckpoint::getExternalPointer) + .thenApplyAsync(path -> { + if (cancelJob) { + log.info("Savepoint stored in {}. Now cancelling {}.", path, jobGraph.getJobID()); + cancel(timeout); + } + return path; + }, getMainThreadExecutor()) + .exceptionally(throwable -> { + if (cancelJob) { + startCheckpointScheduler(checkpointCoordinator); + } + throw new CompletionException(throwable); + }); + } + + private void startCheckpointScheduler(final CheckpointCoordinator checkpointCoordinator) { + if (checkpointCoordinator.isPeriodicCheckpointingConfigured()) { + try { + checkpointCoordinator.startCheckpointScheduler(); + } catch (IllegalStateException ignored) { + // Concurrent shut down of the coordinator + } } } @@ -1321,6 +1345,7 @@ private CompletableFuture restoreExecutionGraphFromRescalingSave private CompletableFuture getJobModificationSavepoint(Time timeout) { return triggerSavepoint( null, + false, timeout) .handleAsync( (String savepointPath, Throwable throwable) -> { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java index 6173a26cd82929..1e1bdda45117a5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java @@ -263,6 +263,7 @@ CompletableFuture registerTaskManager( */ CompletableFuture triggerSavepoint( @Nullable final String targetDirectory, + final boolean cancelJob, final Time timeout); /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 6b5f9b50aecca7..98c8ca22e9ba43 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -483,6 +483,17 @@ public CompletableFuture cancelJob(JobID jobId) { } } + public CompletableFuture triggerSavepoint(JobID jobId, String targetDirectory, boolean cancelJob) { + try { + return getDispatcherGateway().triggerSavepoint(jobId, targetDirectory, cancelJob, rpcTimeout); + } catch (LeaderRetrievalException | InterruptedException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Could not trigger savepoint for job %s.", jobId), + e)); + } + } + // ------------------------------------------------------------------------ // running jobs // ------------------------------------------------------------------------ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java index cb3ff5bb06c46a..17e263bec6e636 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java @@ -127,8 +127,9 @@ protected CompletableFuture triggerOperation(HandlerRequest requestMultipleJobDetails( default CompletableFuture triggerSavepoint( JobID jobId, String targetDirectory, + boolean cancelJob, @RpcTimeout Time timeout) { throw new UnsupportedOperationException(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java index cac7e90bd09b36..0d57a56b2ceb38 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java @@ -48,6 +48,8 @@ import org.apache.flink.runtime.taskmanager.TaskExecutionState; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; +import javax.annotation.Nullable; + import java.net.InetSocketAddress; import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -153,7 +155,7 @@ public CompletableFuture requestJob(Time timeout) { } @Override - public CompletableFuture triggerSavepoint(String targetDirectory, Time timeout) { + public CompletableFuture triggerSavepoint(@Nullable final String targetDirectory, final boolean cancelJob, final Time timeout) { throw new UnsupportedOperationException(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/async/AbstractAsynchronousOperationHandlersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/async/AbstractAsynchronousOperationHandlersTest.java index 848e2539d7b079..7ad140e6d20bd6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/async/AbstractAsynchronousOperationHandlersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/async/AbstractAsynchronousOperationHandlersTest.java @@ -305,7 +305,7 @@ protected TestingTriggerHandler(CompletableFuture localRestAddress, Gate @Override protected CompletableFuture triggerOperation(HandlerRequest request, RestfulGateway gateway) throws RestHandlerException { - return gateway.triggerSavepoint(new JobID(), null, timeout); + return gateway.triggerSavepoint(new JobID(), null, false, timeout); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java index a8f4b3f2754062..06944525a39dda 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java @@ -200,7 +200,7 @@ private static HandlerRequest( - new SavepointTriggerRequestBody(targetDirectory), + new SavepointTriggerRequestBody(targetDirectory, false), new SavepointTriggerMessageParameters(), Collections.singletonMap(JobIDPathParameter.KEY, JOB_ID.toString()), Collections.emptyMap()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java index f7c3973627c897..f79f8a2682188c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java @@ -20,22 +20,43 @@ import org.apache.flink.runtime.rest.messages.RestRequestMarshallingTestBase; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; + import static org.junit.Assert.assertEquals; /** * Tests for {@link SavepointTriggerRequestBody}. */ +@RunWith(Parameterized.class) public class SavepointTriggerRequestBodyTest extends RestRequestMarshallingTestBase { + private final SavepointTriggerRequestBody savepointTriggerRequestBody; + + public SavepointTriggerRequestBodyTest(final SavepointTriggerRequestBody savepointTriggerRequestBody) { + this.savepointTriggerRequestBody = savepointTriggerRequestBody; + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][]{ + {new SavepointTriggerRequestBody("/tmp", true)}, + {new SavepointTriggerRequestBody("/tmp", false)} + }); + } + @Override protected Class getTestRequestClass() { return SavepointTriggerRequestBody.class; } @Override - protected SavepointTriggerRequestBody getTestRequestInstance() throws Exception { - return new SavepointTriggerRequestBody("/tmp"); + protected SavepointTriggerRequestBody getTestRequestInstance() { + return savepointTriggerRequestBody; } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java index 5eff5a680e6494..b92ba5182bee4a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java @@ -190,7 +190,7 @@ public CompletableFuture requestOperatorBackP } @Override - public CompletableFuture triggerSavepoint(JobID jobId, String targetDirectory, Time timeout) { + public CompletableFuture triggerSavepoint(JobID jobId, String targetDirectory, boolean cancelJob, Time timeout) { return triggerSavepointFunction.apply(jobId, targetDirectory); } diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointIT.java b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointIT.java new file mode 100644 index 00000000000000..f9edfa63695409 --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointIT.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.client.program.MiniClusterClient; +import org.apache.flink.runtime.checkpoint.CheckpointMetaData; +import org.apache.flink.runtime.checkpoint.CheckpointMetrics; +import org.apache.flink.runtime.checkpoint.CheckpointOptions; +import org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy; +import org.apache.flink.runtime.checkpoint.CheckpointTriggerException; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.runtime.checkpoint.TaskStateSnapshot; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.jobgraph.JobVertex; +import org.apache.flink.runtime.jobgraph.OperatorID; +import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; +import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; +import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings; +import org.apache.flink.test.util.AbstractTestBase; +import org.apache.flink.testutils.category.Flip6; +import org.apache.flink.util.ExceptionUtils; + +import org.junit.Assume; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.isOneOf; + +/** + * Tests for {@link org.apache.flink.runtime.jobmaster.JobMaster#triggerSavepoint(String, boolean, Time)}. + * + * @see org.apache.flink.runtime.jobmaster.JobMaster + */ +@Category(Flip6.class) +public class JobMasterTriggerSavepointIT extends AbstractTestBase { + + private static CountDownLatch invokeLatch; + + private static volatile CountDownLatch triggerCheckpointLatch; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private Path savepointDirectory; + private MiniClusterClient clusterClient; + private JobGraph jobGraph; + + @Before + public void setUp() throws Exception { + invokeLatch = new CountDownLatch(1); + triggerCheckpointLatch = new CountDownLatch(1); + savepointDirectory = temporaryFolder.newFolder().toPath(); + + Assume.assumeTrue( + "ClusterClient is not an instance of MiniClusterClient", + miniClusterResource.getClusterClient() instanceof MiniClusterClient); + + clusterClient = (MiniClusterClient) miniClusterResource.getClusterClient(); + clusterClient.setDetached(true); + + jobGraph = new JobGraph(); + + final JobVertex vertex = new JobVertex("testVertex"); + vertex.setInvokableClass(NoOpBlockingInvokable.class); + jobGraph.addVertex(vertex); + + jobGraph.setSnapshotSettings(new JobCheckpointingSettings( + Collections.singletonList(vertex.getID()), + Collections.singletonList(vertex.getID()), + Collections.singletonList(vertex.getID()), + new CheckpointCoordinatorConfiguration( + 10, + 60_000, + 10, + 1, + CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, + true), + null + )); + + clusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader()); + invokeLatch.await(60, TimeUnit.SECONDS); + waitForJob(); + } + + @Test + public void testStopJobAfterSavepoint() throws Exception { + final String savepointLocation = cancelWithSavepoint(); + final JobStatus jobStatus = clusterClient.getJobStatus(jobGraph.getJobID()).get(60, TimeUnit.SECONDS); + + assertThat(jobStatus, isOneOf(JobStatus.CANCELED, JobStatus.CANCELLING)); + + final List savepoints = Files.list(savepointDirectory).map(Path::getFileName).collect(Collectors.toList()); + assertThat(savepoints, hasItem(Paths.get(savepointLocation).getFileName())); + } + + @Test + public void testDoNotCancelJobIfSavepointFails() throws Exception { + try { + Files.setPosixFilePermissions(savepointDirectory, Collections.emptySet()); + } catch (IOException e) { + Assume.assumeNoException(e); + } + + try { + cancelWithSavepoint(); + } catch (Exception e) { + assertThat(ExceptionUtils.findThrowable(e, CheckpointTriggerException.class).isPresent(), equalTo(true)); + } + + final JobStatus jobStatus = clusterClient.getJobStatus(jobGraph.getJobID()).get(60, TimeUnit.SECONDS); + assertThat(jobStatus, equalTo(JobStatus.RUNNING)); + + // assert that checkpoints are continued to be triggered + triggerCheckpointLatch = new CountDownLatch(1); + assertThat(triggerCheckpointLatch.await(60, TimeUnit.SECONDS), equalTo(true)); + } + + private void waitForJob() throws Exception { + for (int i = 0; i < 60; i++) { + try { + final JobStatus jobStatus = clusterClient.getJobStatus(jobGraph.getJobID()).get(60, TimeUnit.SECONDS); + assertThat(jobStatus.isGloballyTerminalState(), equalTo(false)); + if (jobStatus == JobStatus.RUNNING) { + return; + } + } catch (ExecutionException ignored) { + // JobManagerRunner is not yet registered in Dispatcher + } + Thread.sleep(1000); + } + throw new AssertionError("Job did not become running within timeout."); + } + + /** + * Invokable which calls {@link CountDownLatch#countDown()} on + * {@link JobMasterTriggerSavepointIT#invokeLatch}, and then blocks afterwards. + */ + public static class NoOpBlockingInvokable extends AbstractInvokable { + + public NoOpBlockingInvokable(final Environment environment) { + super(environment); + } + + @Override + public void invoke() { + invokeLatch.countDown(); + try { + Thread.sleep(Long.MAX_VALUE); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Override + public boolean triggerCheckpoint(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions) throws Exception { + final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot(); + checkpointStateHandles.putSubtaskStateByOperatorID( + OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()), + new OperatorSubtaskState()); + + getEnvironment().acknowledgeCheckpoint( + checkpointMetaData.getCheckpointId(), + new CheckpointMetrics(), + checkpointStateHandles); + + triggerCheckpointLatch.countDown(); + + return true; + } + + @Override + public void notifyCheckpointComplete(final long checkpointId) throws Exception { + } + } + + private String cancelWithSavepoint() throws Exception { + return clusterClient.cancelWithSavepoint( + jobGraph.getJobID(), + savepointDirectory.toAbsolutePath().toString()); + } + +} From 1e48b722d23dc160e065b04c93d41b8df1400c19 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 26 Feb 2018 11:52:50 +0100 Subject: [PATCH 0100/2294] [FLINK-8758] Make non-blocking ClusterClient.submitJob() public --- .../java/org/apache/flink/client/program/ClusterClient.java | 2 +- .../apache/flink/client/program/StandaloneClusterClient.java | 2 +- .../org/apache/flink/client/program/rest/RestClusterClient.java | 2 +- .../src/main/java/org/apache/flink/yarn/YarnClusterClient.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index 1cf2bc2847c362..7817f8f85d079b 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -987,7 +987,7 @@ public Configuration getFlinkConfiguration() { * @param jobGraph The JobGraph to be submitted * @return JobSubmissionResult */ - protected abstract JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) + public abstract JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException; /** diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java index ee8ad44ddc0e1c..1c9c690710ee1f 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java @@ -107,7 +107,7 @@ public boolean hasUserJarsInClassPath(List userJarFiles) { } @Override - protected JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) + public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { if (isDetached()) { return super.runDetached(jobGraph, classLoader); diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 560a10e955072f..18ff0992cb685a 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -219,7 +219,7 @@ public void shutdown() { } @Override - protected JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { + public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { log.info("Submitting job {}.", jobGraph.getJobID()); final CompletableFuture jobSubmissionFuture = submitJob(jobGraph); diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java index f2be264fe20d76..e0010c769ae0e7 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java @@ -148,7 +148,7 @@ public boolean hasUserJarsInClassPath(List userJarFiles) { } @Override - protected JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { + public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { if (isDetached()) { if (newlyCreatedCluster) { stopAfterJob(jobGraph.getJobID()); From 7d1c6b44ea2ee8f6b176d3caf57e76a88ea17686 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 26 Feb 2018 10:12:44 +0100 Subject: [PATCH 0101/2294] [FLINK-8700] Add getters to JobDetailsInfo --- .../runtime/rest/messages/RequestBody.java | 2 +- .../runtime/rest/messages/ResponseBody.java | 2 +- .../rest/messages/job/JobDetailsInfo.java | 70 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RequestBody.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RequestBody.java index ca55b17532ba3a..8098ccfc1503fe 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RequestBody.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/RequestBody.java @@ -26,7 +26,7 @@ * *

    All fields that should part of the JSON request must be accessible either by being public or having a getter. * - *

    When adding methods that are prefixed with {@code get} make sure to annotate them with {@code @JsonIgnore}. + *

    When adding methods that are prefixed with {@code get/is} make sure to annotate them with {@code @JsonIgnore}. */ public interface RequestBody { } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/ResponseBody.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/ResponseBody.java index d4e94d1d6abdc6..ff77966f4dcbe9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/ResponseBody.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/ResponseBody.java @@ -26,7 +26,7 @@ * *

    All fields that should part of the JSON response must be accessible either by being public or having a getter. * - *

    When adding methods that are prefixed with {@code get} make sure to annotate them with {@code @JsonIgnore}. + *

    When adding methods that are prefixed with {@code get/is} make sure to annotate them with {@code @JsonIgnore}. */ public interface ResponseBody { } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobDetailsInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobDetailsInfo.java index 2c74389d96e8d8..f839c182a5d845 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobDetailsInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobDetailsInfo.java @@ -32,6 +32,7 @@ import org.apache.flink.util.Preconditions; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonRawValue; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -165,6 +166,66 @@ public int hashCode() { return Objects.hash(jobId, name, isStoppable, jobStatus, startTime, endTime, duration, now, timestamps, jobVertexInfos, jobVerticesPerState, jsonPlan); } + @JsonIgnore + public JobID getJobId() { + return jobId; + } + + @JsonIgnore + public String getName() { + return name; + } + + @JsonIgnore + public boolean isStoppable() { + return isStoppable; + } + + @JsonIgnore + public JobStatus getJobStatus() { + return jobStatus; + } + + @JsonIgnore + public long getStartTime() { + return startTime; + } + + @JsonIgnore + public long getEndTime() { + return endTime; + } + + @JsonIgnore + public long getDuration() { + return duration; + } + + @JsonIgnore + public long getNow() { + return now; + } + + @JsonIgnore + public Map getTimestamps() { + return timestamps; + } + + @JsonIgnore + public Collection getJobVertexInfos() { + return jobVertexInfos; + } + + @JsonIgnore + public Map getJobVerticesPerState() { + return jobVerticesPerState; + } + + @JsonIgnore + public String getJsonPlan() { + return jsonPlan; + } + // --------------------------------------------------- // Static inner classes // --------------------------------------------------- @@ -242,38 +303,47 @@ public JobVertexDetailsInfo( this.jobVertexMetrics = Preconditions.checkNotNull(jobVertexMetrics); } + @JsonIgnore public JobVertexID getJobVertexID() { return jobVertexID; } + @JsonIgnore public String getName() { return name; } + @JsonIgnore public int getParallelism() { return parallelism; } + @JsonIgnore public ExecutionState getExecutionState() { return executionState; } + @JsonIgnore public long getStartTime() { return startTime; } + @JsonIgnore public long getEndTime() { return endTime; } + @JsonIgnore public long getDuration() { return duration; } + @JsonIgnore public Map getTasksPerState() { return tasksPerState; } + @JsonIgnore public IOMetricsInfo getJobVertexMetrics() { return jobVertexMetrics; } From 56fef58a3edd167b4bf2afc7b2dd63a94b4b478f Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 26 Feb 2018 11:53:47 +0100 Subject: [PATCH 0102/2294] [FLINK-8700] Add ClusterClient.getJobStatus() --- .../flink/client/program/ClusterClient.java | 29 +++++++++++++++++++ .../client/program/MiniClusterClient.java | 1 + .../program/rest/RestClusterClient.java | 14 +++++++++ 3 files changed, 44 insertions(+) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index 7817f8f85d079b..18000b7f127d1f 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -49,6 +49,7 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; @@ -583,6 +584,34 @@ public JobListeningContext connectToJob(JobID jobID) throws JobExecutionExceptio printStatusDuringExecution); } + /** + * Requests the {@link JobStatus} of the job with the given {@link JobID}. + */ + public CompletableFuture getJobStatus(JobID jobId) { + final ActorGateway jobManager; + try { + jobManager = getJobManagerGateway(); + } catch (FlinkException e) { + throw new RuntimeException("Could not retrieve JobManage gateway.", e); + } + + Future response = jobManager.ask(JobManagerMessages.getRequestJobStatus(jobId), timeout); + + CompletableFuture javaFuture = FutureUtils.toJava(response); + + return javaFuture.thenApply((responseMessage) -> { + if (responseMessage instanceof JobManagerMessages.CurrentJobStatus) { + return ((JobManagerMessages.CurrentJobStatus) responseMessage).status(); + } else if (responseMessage instanceof JobManagerMessages.JobNotFound) { + throw new CompletionException( + new IllegalStateException("Could not find job with JobId " + jobId)); + } else { + throw new CompletionException( + new IllegalStateException("Unknown JobManager response of type " + responseMessage.getClass())); + } + }); + } + /** * Cancels a job identified by the job id. * @param jobId the job id diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index dd99f0dabeb998..67e49fa28e31f3 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -120,6 +120,7 @@ public Map getAccumulators(JobID jobID, ClassLoader loader) thro throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); } + @Override public CompletableFuture getJobStatus(JobID jobId) { return miniCluster.getJobStatus(jobId); } diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 18ff0992cb685a..8cf0d2c7eb2009 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -35,6 +35,7 @@ import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; @@ -61,6 +62,8 @@ import org.apache.flink.runtime.rest.messages.ResponseBody; import org.apache.flink.runtime.rest.messages.TerminationModeQueryParameter; import org.apache.flink.runtime.rest.messages.TriggerId; +import org.apache.flink.runtime.rest.messages.job.JobDetailsHeaders; +import org.apache.flink.runtime.rest.messages.job.JobDetailsInfo; import org.apache.flink.runtime.rest.messages.job.JobExecutionResultHeaders; import org.apache.flink.runtime.rest.messages.job.JobSubmitHeaders; import org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody; @@ -254,6 +257,17 @@ public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) } } + @Override + public CompletableFuture getJobStatus(JobID jobId) { + JobDetailsHeaders detailsHeaders = JobDetailsHeaders.getInstance(); + final JobMessageParameters params = new JobMessageParameters(); + params.jobPathParameter.resolve(jobId); + + CompletableFuture responseFuture = sendRequest(detailsHeaders, params); + + return responseFuture.thenApply(JobDetailsInfo::getJobStatus); + } + /** * Requests the {@link JobResult} for the given {@link JobID}. The method retries multiple * times to poll the {@link JobResult} before giving up. From c74d8cac25e8c025ced11a9c03cd0cf07a8c2d6b Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 27 Feb 2018 17:29:00 +0100 Subject: [PATCH 0103/2294] [FLINK-8818][yarn/s3][tests] harden YarnFileStageTest upload test for eventual consistent read-after-write In case the newly written object cannot be read (yet), we do 4 more retries to retrieve the value and wait 50ms each. While this does not solve all the cases it should make the (rare) case of the written object not being available for read even more unlikely. This closes #5601. --- .../apache/flink/yarn/YarnFileStageTest.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java index 5cbe1be7eeafb0..527782c257aa8e 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java @@ -41,6 +41,7 @@ import java.io.DataOutputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.util.ArrayList; import java.util.Collections; @@ -200,13 +201,23 @@ static void testCopyFromLocalRecursive( while (targetFilesIterator.hasNext()) { LocatedFileStatus targetFile = targetFilesIterator.next(); - try (FSDataInputStream in = targetFileSystem.open(targetFile.getPath())) { - String absolutePathString = targetFile.getPath().toString(); - String relativePath = absolutePathString.substring(workDirPrefixLength); - targetFiles.put(relativePath, in.readUTF()); - - assertEquals("extraneous data in file " + relativePath, -1, in.read()); - } + int retries = 5; + do { + try (FSDataInputStream in = targetFileSystem.open(targetFile.getPath())) { + String absolutePathString = targetFile.getPath().toString(); + String relativePath = absolutePathString.substring(workDirPrefixLength); + targetFiles.put(relativePath, in.readUTF()); + + assertEquals("extraneous data in file " + relativePath, -1, in.read()); + break; + } catch (FileNotFoundException e) { + // For S3, read-after-write may be eventually consistent, i.e. when trying + // to access the object before writing it; see + // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel + // -> try again a bit later + Thread.sleep(50); + } + } while ((retries--) > 0); } assertThat(targetFiles, equalTo(srcFiles)); From fc0001c8585ff68f1d2568434727b29c8546d909 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Thu, 1 Mar 2018 13:53:21 +0100 Subject: [PATCH 0104/2294] [FLINK-8769][flip6] do not print error causing exceptions without debugging In DispatcherRestEndpoint and TaskExecutor, there were two places where without errors (running a job inside an IDE) exceptions were logged. While for debugging they may be useful, for normal operation it is enough to print the messages themselves, especially since some more details were already logged before. This closes #5611. --- .../dispatcher/DispatcherRestEndpoint.java | 7 ++++++- .../flink/runtime/taskexecutor/TaskExecutor.java | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java index b5205ab28d8705..9df6deec49fcac 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java @@ -120,7 +120,12 @@ protected List> initiali // register extension handlers handlers.addAll(webSubmissionExtension.getHandlers()); } catch (FlinkException e) { - log.info("Failed to load web based job submission extension.", e); + if (log.isDebugEnabled()) { + log.debug("Failed to load web based job submission extension.", e); + } else { + log.info("Failed to load web based job submission extension. " + + "Probable reason: flink-runtime-web is not in the classpath."); + } } } else { log.info("Web-based job submission is not enabled."); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index cab686af59aa91..fc69984c8a0c87 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -913,13 +913,25 @@ private void closeResourceManagerConnection(Exception cause) { if (resourceManagerConnection != null) { if (resourceManagerConnection.isConnected()) { - log.info("Close ResourceManager connection {}.", resourceManagerConnection.getResourceManagerId(), cause); + if (log.isDebugEnabled()) { + log.debug("Close ResourceManager connection {}.", + resourceManagerConnection.getResourceManagerId(), cause); + } else { + log.info("Close ResourceManager connection {}.", + resourceManagerConnection.getResourceManagerId()); + } resourceManagerHeartbeatManager.unmonitorTarget(resourceManagerConnection.getResourceManagerId()); ResourceManagerGateway resourceManagerGateway = resourceManagerConnection.getTargetGateway(); resourceManagerGateway.disconnectTaskManager(getResourceID(), cause); } else { - log.info("Terminating registration attempts towards ResourceManager {}.", resourceManagerConnection.getTargetAddress(), cause); + if (log.isDebugEnabled()) { + log.debug("Terminating registration attempts towards ResourceManager {}.", + resourceManagerConnection.getTargetAddress(), cause); + } else { + log.info("Terminating registration attempts towards ResourceManager {}.", + resourceManagerConnection.getTargetAddress()); + } } resourceManagerConnection.close(); From 3c8a673f99dc6ba28987b827cfc65f348a26ed32 Mon Sep 17 00:00:00 2001 From: Zhijiang Date: Fri, 23 Feb 2018 11:29:16 +0800 Subject: [PATCH 0105/2294] [FLINK-8458][config][docs] Add config of credit-based network buffers This closes #5317. --- .../generated/netty_configuration.html | 15 +++++++++++++++ .../flink/configuration/TaskManagerOptions.java | 17 +++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/_includes/generated/netty_configuration.html b/docs/_includes/generated/netty_configuration.html index 47c48c0aa38d29..e97fda45d1055b 100644 --- a/docs/_includes/generated/netty_configuration.html +++ b/docs/_includes/generated/netty_configuration.html @@ -42,5 +42,20 @@ "nio" The Netty transport type, either "nio" or "epoll" + +
    taskmanager.network.credit-based-flow-control.enabled
    + true + Boolean flag to enable/disable network credit-based flow control + + +
    taskmanager.network.memory.buffers-per-channel
    + 2 + Number of network buffers to use for each outgoing/incoming channel (subpartition/input channel). In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is for parallel serialization + + +
    taskmanager.network.memory.floating-buffers-per-gate
    + 8 + Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster + diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java index cc3284cdbde7ce..4e08fdaa9910a2 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java @@ -269,7 +269,10 @@ public class TaskManagerOptions { public static final ConfigOption NETWORK_BUFFERS_PER_CHANNEL = key("taskmanager.network.memory.buffers-per-channel") .defaultValue(2) - .withDescription("Number of network buffers to use for each outgoing/incoming channel (subpartition/input channel)."); + .withDescription("Number of network buffers to use for each outgoing/incoming channel (subpartition/input channel)." + + "In credit-based flow control mode, this indicates how many credits are exclusive in each input channel. It should be" + + " configured at least 2 for good performance. 1 buffer is for receiving in-flight data in the subpartition and 1 buffer is" + + " for parallel serialization."); /** * Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). @@ -277,7 +280,12 @@ public class TaskManagerOptions { public static final ConfigOption NETWORK_EXTRA_BUFFERS_PER_GATE = key("taskmanager.network.memory.floating-buffers-per-gate") .defaultValue(8) - .withDescription("Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate)."); + .withDescription("Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate)." + + " In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels." + + " The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can" + + " help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be" + + " increased in case of higher round trip times between nodes and/or larger number of machines in the cluster."); + /** * Minimum backoff for partition requests of input channels. @@ -307,7 +315,7 @@ public class TaskManagerOptions { .withDescription("Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths."); /** - * Config parameter defining whether to enable credit-based flow control or not. + * Boolean flag to enable/disable network credit-based flow control. * * @deprecated Will be removed for Flink 1.6 when the old code will be dropped in favour of * credit-based flow control. @@ -315,7 +323,8 @@ public class TaskManagerOptions { @Deprecated public static final ConfigOption NETWORK_CREDIT_BASED_FLOW_CONTROL_ENABLED = key("taskmanager.network.credit-based-flow-control.enabled") - .defaultValue(true); + .defaultValue(true) + .withDescription("Boolean flag to enable/disable network credit-based flow control."); /** * Config parameter defining whether to spill data for channels with barrier or not in exactly-once From 131daa28bf0169f9e634325243c83e24aed9c514 Mon Sep 17 00:00:00 2001 From: sihuazhou Date: Mon, 5 Mar 2018 20:21:27 +0800 Subject: [PATCH 0106/2294] [FLINK-8859][checkpointing] RocksDB backend should pass WriteOption to Rocks.put() when restoring This closes #5635. --- .../contrib/streaming/state/RocksDBKeyedStateBackend.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 8f95b1812d844b..5444dee443851c 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -680,7 +680,7 @@ private void restoreKVStateData() throws IOException, RocksDBException { if (RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(key)) { //clear the signal bit in the key to make it ready for insertion again RocksDBFullSnapshotOperation.clearMetaDataFollowsFlag(key); - rocksDBKeyedStateBackend.db.put(handle, key, value); + rocksDBKeyedStateBackend.db.put(handle, rocksDBKeyedStateBackend.writeOptions, key, value); //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible kvStateId = RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK & compressedKgInputView.readShort(); @@ -690,7 +690,7 @@ private void restoreKVStateData() throws IOException, RocksDBException { handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); } } else { - rocksDBKeyedStateBackend.db.put(handle, key, value); + rocksDBKeyedStateBackend.db.put(handle, rocksDBKeyedStateBackend.writeOptions, key, value); } } } @@ -1091,6 +1091,7 @@ private void restoreKeyGroupsShardWithTemporaryHelperInstance( if (stateBackend.keyGroupRange.contains(keyGroup)) { stateBackend.db.put(targetColumnFamilyHandle, + stateBackend.writeOptions, iterator.key(), iterator.value()); } From d85fe58d768142ad07587f394ac933edac405cbd Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 2 Mar 2018 14:38:20 +0100 Subject: [PATCH 0107/2294] [FLINK-8517] Fix missing synchronization in TaskEventDispatcher This closes #5621. --- .../flink/runtime/io/network/TaskEventDispatcher.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/TaskEventDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/TaskEventDispatcher.java index 1ec4ade85ad431..c9de902e63aed6 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/TaskEventDispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/TaskEventDispatcher.java @@ -102,7 +102,10 @@ public void subscribeToEvent( checkNotNull(eventListener); checkNotNull(eventType); - TaskEventHandler taskEventHandler = registeredHandlers.get(partitionId); + TaskEventHandler taskEventHandler; + synchronized (registeredHandlers) { + taskEventHandler = registeredHandlers.get(partitionId); + } if (taskEventHandler == null) { throw new IllegalStateException( "Partition " + partitionId + " not registered at task event dispatcher."); @@ -123,7 +126,10 @@ public boolean publish(ResultPartitionID partitionId, TaskEvent event) { checkNotNull(partitionId); checkNotNull(event); - TaskEventHandler taskEventHandler = registeredHandlers.get(partitionId); + TaskEventHandler taskEventHandler; + synchronized (registeredHandlers) { + taskEventHandler = registeredHandlers.get(partitionId); + } if (taskEventHandler != null) { taskEventHandler.publish(event); From 4226bf22aab5b4359998422fe53755db19785515 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Fri, 2 Mar 2018 17:46:56 +0100 Subject: [PATCH 0108/2294] [FLINK-8807] Fix ZookeeperCompleted checkpoint store can get stuck in infinite loop Before, CompletedCheckpoint did not have proper equals()/hashCode(), which meant that the fixpoint condition in ZooKeeperCompletedCheckpointStore would never hold if at least on checkpoint became unreadable. We now compare the interesting fields of the checkpoints manually and extended the test to properly create new CompletedCheckpoints. Before, we were reusing the same CompletedCheckpoint instances, meaning that Objects.equals()/hashCode() would make the test succeed. --- .../checkpoint/CompletedCheckpoint.java | 26 ++++++++++ .../ZooKeeperCompletedCheckpointStore.java | 2 +- ...ZooKeeperCompletedCheckpointStoreTest.java | 51 ++++++++++--------- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java index df8c233ada5273..58424272bbc8de 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.checkpoint; import org.apache.flink.api.common.JobID; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation; @@ -37,7 +38,9 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -283,6 +286,29 @@ private void doDiscard() throws Exception { // Miscellaneous // ------------------------------------------------------------------------ + public static boolean checkpointsMatch( + Collection first, + Collection second) { + + Set> firstInterestingFields = + new HashSet<>(); + + for (CompletedCheckpoint checkpoint : first) { + firstInterestingFields.add( + new Tuple2<>(checkpoint.getCheckpointID(), checkpoint.getJobId())); + } + + Set> secondInterestingFields = + new HashSet<>(); + + for (CompletedCheckpoint checkpoint : second) { + secondInterestingFields.add( + new Tuple2<>(checkpoint.getCheckpointID(), checkpoint.getJobId())); + } + + return firstInterestingFields.equals(secondInterestingFields); + } + /** * Sets the callback for tracking when this checkpoint is discarded. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java index 73598e628b657e..0cbd4fb6c9e9f0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java @@ -199,7 +199,7 @@ public void recover() throws Exception { } } while (retrievedCheckpoints.size() != numberOfInitialCheckpoints && - !lastTryRetrievedCheckpoints.equals(retrievedCheckpoints)); + !CompletedCheckpoint.checkpointsMatch(lastTryRetrievedCheckpoints, retrievedCheckpoints)); // Clear local handles in order to prevent duplicates on // recovery. The local handles should reflect the state diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java index a54432788be2e8..08c73bae900450 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java @@ -83,31 +83,16 @@ public void testPathConversion() { /** * Tests that the completed checkpoint store can retrieve all checkpoints stored in ZooKeeper * and ignores those which cannot be retrieved via their state handles. + * + *

    We have a timeout in case the ZooKeeper store get's into a deadlock/livelock situation. */ - @Test + @Test(timeout = 50000) public void testCheckpointRecovery() throws Exception { + final JobID jobID = new JobID(); + final long checkpoint1Id = 1L; + final long checkpoint2Id = 2; final List, String>> checkpointsInZooKeeper = new ArrayList<>(4); - final CompletedCheckpoint completedCheckpoint1 = new CompletedCheckpoint( - new JobID(), - 1L, - 1L, - 1L, - new HashMap<>(), - null, - CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), - new TestCompletedCheckpointStorageLocation()); - - final CompletedCheckpoint completedCheckpoint2 = new CompletedCheckpoint( - new JobID(), - 2L, - 2L, - 2L, - new HashMap<>(), - null, - CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), - new TestCompletedCheckpointStorageLocation()); - final Collection expectedCheckpointIds = new HashSet<>(2); expectedCheckpointIds.add(1L); expectedCheckpointIds.add(2L); @@ -116,10 +101,28 @@ public void testCheckpointRecovery() throws Exception { when(failingRetrievableStateHandle.retrieveState()).thenThrow(new IOException("Test exception")); final RetrievableStateHandle retrievableStateHandle1 = mock(RetrievableStateHandle.class); - when(retrievableStateHandle1.retrieveState()).thenReturn(completedCheckpoint1); + when(retrievableStateHandle1.retrieveState()).then( + (invocation) -> new CompletedCheckpoint( + jobID, + checkpoint1Id, + 1L, + 1L, + new HashMap<>(), + null, + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), + new TestCompletedCheckpointStorageLocation())); final RetrievableStateHandle retrievableStateHandle2 = mock(RetrievableStateHandle.class); - when(retrievableStateHandle2.retrieveState()).thenReturn(completedCheckpoint2); + when(retrievableStateHandle2.retrieveState()).then( + (invocation -> new CompletedCheckpoint( + jobID, + checkpoint2Id, + 2L, + 2L, + new HashMap<>(), + null, + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), + new TestCompletedCheckpointStorageLocation()))); checkpointsInZooKeeper.add(Tuple2.of(retrievableStateHandle1, "/foobar1")); checkpointsInZooKeeper.add(Tuple2.of(failingRetrievableStateHandle, "/failing1")); @@ -185,7 +188,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { // check that we return the latest retrievable checkpoint // this should remove the latest checkpoint because it is broken - assertEquals(completedCheckpoint2.getCheckpointID(), latestCompletedCheckpoint.getCheckpointID()); + assertEquals(checkpoint2Id, latestCompletedCheckpoint.getCheckpointID()); // this should remove the second broken checkpoint because we're iterating over all checkpoints List completedCheckpoints = zooKeeperCompletedCheckpointStore.getAllCheckpoints(); From c1b805050946f851123bca0c2b02fc82d025da96 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 5 Mar 2018 15:45:36 +0100 Subject: [PATCH 0109/2294] Fix checkstyle in ZooKeeperCompletedCheckpointStoreTest --- ...ZooKeeperCompletedCheckpointStoreTest.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java index 08c73bae900450..0384733fdb1cfb 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStoreTest.java @@ -33,10 +33,8 @@ import org.apache.curator.framework.api.CuratorEventType; import org.apache.curator.framework.api.ErrorListenerPathable; import org.apache.curator.utils.EnsurePath; - import org.junit.Test; import org.junit.runner.RunWith; - import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -67,6 +65,9 @@ import static org.powermock.api.mockito.PowerMockito.doThrow; import static org.powermock.api.mockito.PowerMockito.whenNew; +/** + * Tests for {@link ZooKeeperCompletedCheckpointStore}. + */ @RunWith(PowerMockRunner.class) @PrepareForTest(ZooKeeperCompletedCheckpointStore.class) public class ZooKeeperCompletedCheckpointStoreTest extends TestLogger { @@ -209,7 +210,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable { // are subsumed should they be discarded. verify(failingRetrievableStateHandle, never()).discardState(); } - + /** * Tests that the checkpoint does not exist in the store when we fail to add * it into the store (i.e., there exists an exception thrown by the method). @@ -218,29 +219,29 @@ public Void answer(InvocationOnMock invocation) throws Throwable { public void testAddCheckpointWithFailedRemove() throws Exception { final CuratorFramework client = mock(CuratorFramework.class, Mockito.RETURNS_DEEP_STUBS); final RetrievableStateStorageHelper storageHelperMock = mock(RetrievableStateStorageHelper.class); - - ZooKeeperStateHandleStore zookeeperStateHandleStoreMock = + + ZooKeeperStateHandleStore zookeeperStateHandleStoreMock = spy(new ZooKeeperStateHandleStore<>(client, storageHelperMock, Executors.directExecutor())); whenNew(ZooKeeperStateHandleStore.class).withAnyArguments().thenReturn(zookeeperStateHandleStoreMock); - + doAnswer(new Answer>() { @Override public RetrievableStateHandle answer(InvocationOnMock invocationOnMock) throws Throwable { - CompletedCheckpoint checkpoint = (CompletedCheckpoint)invocationOnMock.getArguments()[1]; - + CompletedCheckpoint checkpoint = (CompletedCheckpoint) invocationOnMock.getArguments()[1]; + RetrievableStateHandle retrievableStateHandle = mock(RetrievableStateHandle.class); when(retrievableStateHandle.retrieveState()).thenReturn(checkpoint); - + return retrievableStateHandle; } }).when(zookeeperStateHandleStoreMock).addAndLock(anyString(), any(CompletedCheckpoint.class)); - + doThrow(new Exception()).when(zookeeperStateHandleStoreMock).releaseAndTryRemove(anyString(), any(ZooKeeperStateHandleStore.RemoveCallback.class)); - + final int numCheckpointsToRetain = 1; final String checkpointsPath = "foobar"; final RetrievableStateStorageHelper stateSotrage = mock(RetrievableStateStorageHelper.class); - + ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore( numCheckpointsToRetain, client, @@ -252,10 +253,10 @@ public RetrievableStateHandle answer(InvocationOnMock invoc CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class); doReturn(i).when(checkpointToAdd).getCheckpointID(); doReturn(Collections.emptyMap()).when(checkpointToAdd).getOperatorStates(); - + try { zooKeeperCompletedCheckpointStore.addCheckpoint(checkpointToAdd); - + // The checkpoint should be in the store if we successfully add it into the store. List addedCheckpoints = zooKeeperCompletedCheckpointStore.getAllCheckpoints(); assertTrue(addedCheckpoints.contains(checkpointToAdd)); From 10d52f268db3eda7ee1511ea30afb9a982644148 Mon Sep 17 00:00:00 2001 From: Matrix42 <934336389@qq.com> Date: Wed, 28 Feb 2018 10:51:42 +0800 Subject: [PATCH 0110/2294] [hotfix] Fix javadoc link in ClusterClient#triggerSavepoint This closes #5592. --- .../java/org/apache/flink/client/program/ClusterClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index 18000b7f127d1f..1a783fc2213849 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -692,7 +692,7 @@ public void stop(final JobID jobId) throws Exception { /** * Triggers a savepoint for the job identified by the job id. The savepoint will be written to the given savepoint - * directory, or {@link org.apache.flink.configuration.CoreOptions#SAVEPOINT_DIRECTORY} if it is null. + * directory, or {@link org.apache.flink.configuration.CheckpointingOptions#SAVEPOINT_DIRECTORY} if it is null. * * @param jobId job id * @param savepointDirectory directory the savepoint should be written to From 92cf9378d788c449e855a98b8d1c049c0e0d2609 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 27 Feb 2018 16:04:57 +0100 Subject: [PATCH 0111/2294] [hotfix][REST] Fix CONTENT_TYPE header This closes #5590. --- .../flink/runtime/webmonitor/WebRuntimeMonitorITCase.java | 2 +- .../apache/flink/runtime/rest/handler/util/HandlerUtils.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java index e6cfdda968c7c9..d4fe93af24dca6 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitorITCase.java @@ -334,7 +334,7 @@ public void testLeaderNotAvailable() throws Exception { HttpTestClient.SimpleHttpResponse response = client.getNextResponse(); assertEquals(HttpResponseStatus.SERVICE_UNAVAILABLE, response.getStatus()); - assertEquals(MimeTypes.getMimeTypeForExtension("json"), response.getType()); + assertEquals("application/json; charset=UTF-8", response.getType()); assertTrue(response.getContent().contains("refresh")); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java index 604c0b891f3453..a69f4aaf4576bf 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java @@ -21,6 +21,7 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.runtime.rest.messages.ErrorResponseBody; import org.apache.flink.runtime.rest.messages.ResponseBody; +import org.apache.flink.runtime.rest.util.RestConstants; import org.apache.flink.runtime.rest.util.RestMapperUtils; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -149,7 +150,7 @@ public static void sendResponse( @Nonnull Map headers) { HttpResponse response = new DefaultHttpResponse(HTTP_1_1, statusCode); - response.headers().set(CONTENT_TYPE, "application/json"); + response.headers().set(CONTENT_TYPE, RestConstants.REST_CONTENT_TYPE); for (Map.Entry headerEntry : headers.entrySet()) { response.headers().set(headerEntry.getKey(), headerEntry.getValue()); From 128127a2410c84eef1442093e51b7e79e53f6b1b Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Thu, 1 Mar 2018 10:40:20 +0100 Subject: [PATCH 0112/2294] [hotfix][docs] Drop the incorrect parallel remark in windowAll This closes #5607. --- .../streaming/api/datastream/DataStream.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java index a357eda118457b..7fecdb051954b3 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java @@ -717,9 +717,8 @@ public JoinedStreams join(DataStream otherStream) { * {@code .window(TumblingProcessingTimeWindows.of(size))} depending on the time characteristic * set using * - *

    Note: This operation can be inherently non-parallel since all elements have to pass through - * the same operator instance. (Only for special cases, such as aligned time windows is - * it possible to perform this operation in parallel). + *

    Note: This operation is inherently non-parallel since all elements have to pass through + * the same operator instance. * * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)} * @@ -741,9 +740,8 @@ public AllWindowedStream timeWindowAll(Time size) { * set using * {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#setStreamTimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic)} * - *

    Note: This operation can be inherently non-parallel since all elements have to pass through - * the same operator instance. (Only for special cases, such as aligned time windows is - * it possible to perform this operation in parallel). + *

    Note: This operation is inherently non-parallel since all elements have to pass through + * the same operator instance. * * @param size The size of the window. */ @@ -758,9 +756,8 @@ public AllWindowedStream timeWindowAll(Time size, Time slide) { /** * Windows this {@code DataStream} into tumbling count windows. * - *

    Note: This operation can be inherently non-parallel since all elements have to pass through - * the same operator instance. (Only for special cases, such as aligned time windows is - * it possible to perform this operation in parallel). + *

    Note: This operation is inherently non-parallel since all elements have to pass through + * the same operator instance. * * @param size The size of the windows in number of elements. */ @@ -771,9 +768,8 @@ public AllWindowedStream countWindowAll(long size) { /** * Windows this {@code DataStream} into sliding count windows. * - *

    Note: This operation can be inherently non-parallel since all elements have to pass through - * the same operator instance. (Only for special cases, such as aligned time windows is - * it possible to perform this operation in parallel). + *

    Note: This operation is inherently non-parallel since all elements have to pass through + * the same operator instance. * * @param size The size of the windows in number of elements. * @param slide The slide interval in number of elements. @@ -794,9 +790,8 @@ public AllWindowedStream countWindowAll(long size, long slide) * when windows are evaluated. However, {@code WindowAssigners} have a default {@code Trigger} * that is used if a {@code Trigger} is not specified. * - *

    Note: This operation can be inherently non-parallel since all elements have to pass through - * the same operator instance. (Only for special cases, such as aligned time windows is - * it possible to perform this operation in parallel). + *

    Note: This operation is inherently non-parallel since all elements have to pass through + * the same operator instance. * * @param assigner The {@code WindowAssigner} that assigns elements to windows. * @return The trigger windows data stream. From d14cbe97139c3173fa4be0804ea63fdfcaec6a27 Mon Sep 17 00:00:00 2001 From: Stephen Parente Date: Fri, 2 Mar 2018 14:20:10 -0800 Subject: [PATCH 0113/2294] [hotfix][docs] Remove reference to CheckpointedRestoring This closes #5627. --- docs/dev/stream/state/state.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/dev/stream/state/state.md b/docs/dev/stream/state/state.md index 3ea9b7a7b0e931..a26e24d87027a9 100644 --- a/docs/dev/stream/state/state.md +++ b/docs/dev/stream/state/state.md @@ -386,8 +386,7 @@ public class BufferingSink {% highlight scala %} class BufferingSink(threshold: Int = 0) extends SinkFunction[(String, Int)] - with CheckpointedFunction - with CheckpointedRestoring[List[(String, Int)]] { + with CheckpointedFunction { @transient private var checkpointedState: ListState[(String, Int)] = _ @@ -426,9 +425,6 @@ class BufferingSink(threshold: Int = 0) } } - override def restoreState(state: List[(String, Int)]): Unit = { - bufferedElements ++= state - } } {% endhighlight %} From 72bba50aa3382dbfe904cdb36797bc716f76a129 Mon Sep 17 00:00:00 2001 From: neoremind Date: Mon, 5 Mar 2018 16:50:37 +0800 Subject: [PATCH 0114/2294] [FLINK-8857][hbase] Remove redundant execute() call in hbase example This closes #5633. --- .../apache/flink/addons/hbase/example/HBaseReadExample.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java index 817ae090c4ef01..8475fb81ec0dd8 100644 --- a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java +++ b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseReadExample.java @@ -86,9 +86,6 @@ public boolean filter(Tuple2 t) throws Exception { hbaseDs.print(); - // kick off execution. - env.execute(); - } } From 38785a0072b58b0238615a3bdf8f6da579f98154 Mon Sep 17 00:00:00 2001 From: Ken Krugler Date: Sun, 4 Mar 2018 09:27:11 -0800 Subject: [PATCH 0115/2294] [FLINK-8849][docs] Fix links to chaining docs This closes #5630. --- docs/concepts/runtime.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts/runtime.md b/docs/concepts/runtime.md index cb6d58f0c1e97b..1c7c2816a0e145 100644 --- a/docs/concepts/runtime.md +++ b/docs/concepts/runtime.md @@ -31,7 +31,7 @@ under the License. For distributed execution, Flink *chains* operator subtasks together into *tasks*. Each task is executed by one thread. Chaining operators together into tasks is a useful optimization: it reduces the overhead of thread-to-thread handover and buffering, and increases overall throughput while decreasing latency. -The chaining behavior can be configured; see the [chaining docs](../dev/datastream_api.html#task-chaining-and-resource-groups) for details. +The chaining behavior can be configured; see the [chaining docs](../dev/stream/operators/#task-chaining-and-resource-groups) for details. The sample dataflow in the figure below is executed with five subtasks, and hence with five parallel threads. @@ -98,7 +98,7 @@ job. Allowing this *slot sharing* has two main benefits: TaskManagers with shared Task Slots -The APIs also include a *[resource group](../dev/datastream_api.html#task-chaining-and-resource-groups)* mechanism which can be used to prevent undesirable slot sharing. +The APIs also include a *[resource group](../dev/stream/operators/#task-chaining-and-resource-groups)* mechanism which can be used to prevent undesirable slot sharing. As a rule-of-thumb, a good default number of task slots would be the number of CPU cores. With hyper-threading, each slot then takes 2 or more hardware thread contexts. From 159986292e35a71737bcc434d5f20f385973fafa Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Thu, 15 Feb 2018 21:37:44 +0100 Subject: [PATCH 0116/2294] [FLINK-8560] Add KeyedProcessFunction exposing key in onTimer(). This closes #5481. --- docs/dev/stream/operators/process_function.md | 29 +- ...yedProcessOperatorWithWatermarkDelay.scala | 6 +- .../harness/NonWindowHarnessTest.scala | 6 +- .../harness/OverWindowHarnessTest.scala | 16 +- .../SortProcessFunctionHarnessTest.scala | 6 +- .../streaming/api/datastream/KeyedStream.java | 72 ++- .../api/functions/KeyedProcessFunction.java | 130 +++++ .../api/operators/KeyedProcessOperator.java | 46 +- .../operators/LegacyKeyedProcessOperator.java | 178 +++++++ .../flink/streaming/api/DataStreamTest.java | 43 +- .../operators/KeyedProcessOperatorTest.java | 82 +-- .../LegacyKeyedProcessOperatorTest.java | 483 ++++++++++++++++++ .../streaming/api/scala/KeyedStream.scala | 37 +- .../streaming/api/scala/DataStreamTest.scala | 41 +- 14 files changed, 1078 insertions(+), 97 deletions(-) create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/KeyedProcessFunction.java create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperator.java create mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperatorTest.java diff --git a/docs/dev/stream/operators/process_function.md b/docs/dev/stream/operators/process_function.md index a52c5bfeb48580..d96798323ceed5 100644 --- a/docs/dev/stream/operators/process_function.md +++ b/docs/dev/stream/operators/process_function.md @@ -242,4 +242,31 @@ class CountWithTimeoutFunction extends ProcessFunction[(String, String), (String the current processing time as event-time timestamp. This behavior is very subtle and might not be noticed by users. Well, it's harmful because processing-time timestamps are indeterministic and not aligned with watermarks. Besides, user-implemented logic depends on this wrong timestamp highly likely is unintendedly faulty. So we've decided to fix it. Upon upgrading to 1.4.0, Flink jobs -that are using this incorrect event-time timestamp will fail, and users should adapt their jobs to the correct logic. \ No newline at end of file +that are using this incorrect event-time timestamp will fail, and users should adapt their jobs to the correct logic. + +## The KeyedProcessFunction + +`KeyedProcessFunction`, as an extension of `ProcessFunction`, gives access to the key of timers in its `onTimer(...)` +method. + +

    +
    +{% highlight java %} +@Override +public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + K key = ctx.getCurrentKey(); + // ... +} + +{% endhighlight %} +
    + +
    +{% highlight scala %} +override def onTimer(timestamp: Long, ctx: OnTimerContext, out: Collector[OUT]): Unit = { + var key = ctx.getCurrentKey + // ... +} +{% endhighlight %} +
    +
    \ No newline at end of file diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala index 74b4773005a658..f63bdb5acd663b 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala @@ -19,16 +19,16 @@ package org.apache.flink.table.runtime.operators import org.apache.flink.streaming.api.functions.ProcessFunction -import org.apache.flink.streaming.api.operators.KeyedProcessOperator +import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator import org.apache.flink.streaming.api.watermark.Watermark /** - * A [[KeyedProcessOperator]] that supports holding back watermarks with a static delay. + * A [[LegacyKeyedProcessOperator]] that supports holding back watermarks with a static delay. */ class KeyedProcessOperatorWithWatermarkDelay[KEY, IN, OUT]( private val function: ProcessFunction[IN, OUT], private var watermarkDelay: Long = 0L) - extends KeyedProcessOperator[KEY, IN, OUT](function) { + extends LegacyKeyedProcessOperator[KEY, IN, OUT](function) { /** emits watermark without delay */ def emitWithoutDelay(mark: Watermark): Unit = output.emitWatermark(mark) diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala index ad507618abf5e6..5c31cb246306af 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/NonWindowHarnessTest.scala @@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentLinkedQueue import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.BasicTypeInfo -import org.apache.flink.streaming.api.operators.KeyedProcessOperator +import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator import org.apache.flink.streaming.runtime.streamrecord.StreamRecord import org.apache.flink.table.api.StreamQueryConfig import org.apache.flink.table.runtime.aggregate._ @@ -39,7 +39,7 @@ class NonWindowHarnessTest extends HarnessTestBase { @Test def testNonWindow(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new GroupAggProcessFunction( genSumAggFunction, sumAggregationStateType, @@ -99,7 +99,7 @@ class NonWindowHarnessTest extends HarnessTestBase { @Test def testNonWindowWithRetract(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new GroupAggProcessFunction( genSumAggFunction, sumAggregationStateType, diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala index def1972866a072..6f6fc0edb13245 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/OverWindowHarnessTest.scala @@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentLinkedQueue import org.apache.flink.api.common.time.Time import org.apache.flink.api.common.typeinfo.BasicTypeInfo -import org.apache.flink.streaming.api.operators.KeyedProcessOperator +import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator import org.apache.flink.streaming.runtime.streamrecord.StreamRecord import org.apache.flink.table.api.{StreamQueryConfig, Types} import org.apache.flink.table.runtime.aggregate._ @@ -40,7 +40,7 @@ class OverWindowHarnessTest extends HarnessTestBase{ @Test def testProcTimeBoundedRowsOver(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new ProcTimeBoundedRowsOver( genMinMaxAggFunction, 2, @@ -141,7 +141,7 @@ class OverWindowHarnessTest extends HarnessTestBase{ @Test def testProcTimeBoundedRangeOver(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new ProcTimeBoundedRangeOver( genMinMaxAggFunction, 4000, @@ -250,7 +250,7 @@ class OverWindowHarnessTest extends HarnessTestBase{ @Test def testProcTimeUnboundedOver(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new ProcTimeUnboundedOver( genMinMaxAggFunction, minMaxAggregationStateType, @@ -342,7 +342,7 @@ class OverWindowHarnessTest extends HarnessTestBase{ @Test def testRowTimeBoundedRangeOver(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new RowTimeBoundedRangeOver( genMinMaxAggFunction, minMaxAggregationStateType, @@ -492,7 +492,7 @@ class OverWindowHarnessTest extends HarnessTestBase{ @Test def testRowTimeBoundedRowsOver(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new RowTimeBoundedRowsOver( genMinMaxAggFunction, minMaxAggregationStateType, @@ -640,7 +640,7 @@ class OverWindowHarnessTest extends HarnessTestBase{ @Test def testRowTimeUnboundedRangeOver(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new RowTimeUnboundedRangeOver( genMinMaxAggFunction, minMaxAggregationStateType, @@ -776,7 +776,7 @@ class OverWindowHarnessTest extends HarnessTestBase{ @Test def testRowTimeUnboundedRowsOver(): Unit = { - val processFunction = new KeyedProcessOperator[String, CRow, CRow]( + val processFunction = new LegacyKeyedProcessOperator[String, CRow, CRow]( new RowTimeUnboundedRowsOver( genMinMaxAggFunction, minMaxAggregationStateType, diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala index 94900391378225..457bde2ef7aa74 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala @@ -28,7 +28,7 @@ import org.apache.flink.api.java.functions.KeySelector import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.api.java.typeutils.runtime.RowComparator import org.apache.flink.streaming.api.TimeCharacteristic -import org.apache.flink.streaming.api.operators.KeyedProcessOperator +import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.streaming.runtime.streamrecord.StreamRecord import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil} @@ -71,7 +71,7 @@ class SortProcessFunctionHarnessTest { val inputCRowType = CRowTypeInfo(rT) - val processFunction = new KeyedProcessOperator[Integer,CRow,CRow]( + val processFunction = new LegacyKeyedProcessOperator[Integer,CRow,CRow]( new ProcTimeSortProcessFunction( inputCRowType, collectionRowComparator)) @@ -170,7 +170,7 @@ class SortProcessFunctionHarnessTest { val inputCRowType = CRowTypeInfo(rT) - val processFunction = new KeyedProcessOperator[Integer,CRow,CRow]( + val processFunction = new LegacyKeyedProcessOperator[Integer,CRow,CRow]( new RowTimeSortProcessFunction( inputCRowType, 4, diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java index 7beaa0369d00e2..a948ae20d8cb42 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/KeyedStream.java @@ -37,6 +37,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfoBase; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.functions.KeyedProcessFunction; import org.apache.flink.streaming.api.functions.ProcessFunction; import org.apache.flink.streaming.api.functions.aggregation.AggregationFunction; import org.apache.flink.streaming.api.functions.aggregation.ComparableAggregator; @@ -46,6 +47,7 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.api.graph.StreamGraphGenerator; import org.apache.flink.streaming.api.operators.KeyedProcessOperator; +import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.StreamGroupedFold; import org.apache.flink.streaming.api.operators.StreamGroupedReduce; @@ -272,8 +274,7 @@ public DataStreamSink addSink(SinkFunction sinkFunction) { } /** - * Applies the given {@link ProcessFunction} on the input stream, thereby - * creating a transformed output stream. + * Applies the given {@link ProcessFunction} on the input stream, thereby creating a transformed output stream. * *

    The function will be called for every element in the input streams and can produce zero * or more output elements. Contrary to the {@link DataStream#flatMap(FlatMapFunction)} @@ -286,7 +287,10 @@ public DataStreamSink addSink(SinkFunction sinkFunction) { * @param The type of elements emitted by the {@code ProcessFunction}. * * @return The transformed {@link DataStream}. + * + * @deprecated Use {@link KeyedStream#process(KeyedProcessFunction)} */ + @Deprecated @Override @PublicEvolving public SingleOutputStreamOperator process(ProcessFunction processFunction) { @@ -306,8 +310,7 @@ public SingleOutputStreamOperator process(ProcessFunction processFu } /** - * Applies the given {@link ProcessFunction} on the input stream, thereby - * creating a transformed output stream. + * Applies the given {@link ProcessFunction} on the input stream, thereby creating a transformed output stream. * *

    The function will be called for every element in the input streams and can produce zero * or more output elements. Contrary to the {@link DataStream#flatMap(FlatMapFunction)} @@ -321,19 +324,76 @@ public SingleOutputStreamOperator process(ProcessFunction processFu * @param The type of elements emitted by the {@code ProcessFunction}. * * @return The transformed {@link DataStream}. + * + * @deprecated Use {@link KeyedStream#process(KeyedProcessFunction, TypeInformation)} */ + @Deprecated @Override @Internal public SingleOutputStreamOperator process( ProcessFunction processFunction, TypeInformation outputType) { - KeyedProcessOperator operator = - new KeyedProcessOperator<>(clean(processFunction)); + LegacyKeyedProcessOperator operator = new LegacyKeyedProcessOperator<>(clean(processFunction)); return transform("Process", outputType, operator); } + /** + * Applies the given {@link KeyedProcessFunction} on the input stream, thereby creating a transformed output stream. + * + *

    The function will be called for every element in the input streams and can produce zero + * or more output elements. Contrary to the {@link DataStream#flatMap(FlatMapFunction)} + * function, this function can also query the time and set timers. When reacting to the firing + * of set timers the function can directly emit elements and/or register yet more timers. + * + * @param keyedProcessFunction The {@link KeyedProcessFunction} that is called for each element in the stream. + * + * @param The type of elements emitted by the {@code KeyedProcessFunction}. + * + * @return The transformed {@link DataStream}. + */ + @PublicEvolving + public SingleOutputStreamOperator process(KeyedProcessFunction keyedProcessFunction) { + + TypeInformation outType = TypeExtractor.getUnaryOperatorReturnType( + keyedProcessFunction, + KeyedProcessFunction.class, + 1, + 2, + TypeExtractor.NO_INDEX, + TypeExtractor.NO_INDEX, + getType(), + Utils.getCallLocationName(), + true); + + return process(keyedProcessFunction, outType); + } + + /** + * Applies the given {@link KeyedProcessFunction} on the input stream, thereby creating a transformed output stream. + * + *

    The function will be called for every element in the input streams and can produce zero + * or more output elements. Contrary to the {@link DataStream#flatMap(FlatMapFunction)} + * function, this function can also query the time and set timers. When reacting to the firing + * of set timers the function can directly emit elements and/or register yet more timers. + * + * @param keyedProcessFunction The {@link KeyedProcessFunction} that is called for each element in the stream. + * + * @param outputType {@link TypeInformation} for the result type of the function. + * + * @param The type of elements emitted by the {@code KeyedProcessFunction}. + * + * @return The transformed {@link DataStream}. + */ + @Internal + public SingleOutputStreamOperator process( + KeyedProcessFunction keyedProcessFunction, + TypeInformation outputType) { + + KeyedProcessOperator operator = new KeyedProcessOperator<>(clean(keyedProcessFunction)); + return transform("KeyedProcess", outputType, operator); + } // ------------------------------------------------------------------------ // Windowing diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/KeyedProcessFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/KeyedProcessFunction.java new file mode 100644 index 00000000000000..a03480bc682592 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/KeyedProcessFunction.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.functions.AbstractRichFunction; +import org.apache.flink.streaming.api.TimeDomain; +import org.apache.flink.streaming.api.TimerService; +import org.apache.flink.util.Collector; +import org.apache.flink.util.OutputTag; + +/** + * A keyed function that processes elements of a stream. + * + *

    For every element in the input stream {@link #processElement(Object, Context, Collector)} + * is invoked. This can produce zero or more elements as output. Implementations can also + * query the time and set timers through the provided {@link Context}. For firing timers + * {@link #onTimer(long, OnTimerContext, Collector)} will be invoked. This can again produce + * zero or more elements as output and register further timers. + * + *

    NOTE: Access to keyed state and timers (which are also scoped to a key) is only + * available if the {@code KeyedProcessFunction} is applied on a {@code KeyedStream}. + * + *

    NOTE: A {@code KeyedProcessFunction} is always a + * {@link org.apache.flink.api.common.functions.RichFunction}. Therefore, access to the + * {@link org.apache.flink.api.common.functions.RuntimeContext} is always available and setup and + * teardown methods can be implemented. See + * {@link org.apache.flink.api.common.functions.RichFunction#open(org.apache.flink.configuration.Configuration)} + * and {@link org.apache.flink.api.common.functions.RichFunction#close()}. + * + * @param Type of the key. + * @param Type of the input elements. + * @param Type of the output elements. + */ +@PublicEvolving +public abstract class KeyedProcessFunction extends AbstractRichFunction { + + private static final long serialVersionUID = 1L; + + /** + * Process one element from the input stream. + * + *

    This function can output zero or more elements using the {@link Collector} parameter + * and also update internal state or set timers using the {@link Context} parameter. + * + * @param value The input value. + * @param ctx A {@link Context} that allows querying the timestamp of the element and getting + * a {@link TimerService} for registering timers and querying the time. The + * context is only valid during the invocation of this method, do not store it. + * @param out The collector for returning result values. + * + * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation + * to fail and may trigger recovery. + */ + public abstract void processElement(I value, Context ctx, Collector out) throws Exception; + + /** + * Called when a timer set using {@link TimerService} fires. + * + * @param timestamp The timestamp of the firing timer. + * @param ctx An {@link OnTimerContext} that allows querying the timestamp, the {@link TimeDomain}, and the key + * of the firing timer and getting a {@link TimerService} for registering timers and querying the time. + * The context is only valid during the invocation of this method, do not store it. + * @param out The collector for returning result values. + * + * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation + * to fail and may trigger recovery. + */ + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception {} + + /** + * Information available in an invocation of {@link #processElement(Object, Context, Collector)} + * or {@link #onTimer(long, OnTimerContext, Collector)}. + */ + public abstract class Context { + + /** + * Timestamp of the element currently being processed or timestamp of a firing timer. + * + *

    This might be {@code null}, for example if the time characteristic of your program + * is set to {@link org.apache.flink.streaming.api.TimeCharacteristic#ProcessingTime}. + */ + public abstract Long timestamp(); + + /** + * A {@link TimerService} for querying time and registering timers. + */ + public abstract TimerService timerService(); + + /** + * Emits a record to the side output identified by the {@link OutputTag}. + * + * @param outputTag the {@code OutputTag} that identifies the side output to emit to. + * @param value The record to emit. + */ + public abstract void output(OutputTag outputTag, X value); + } + + /** + * Information available in an invocation of {@link #onTimer(long, OnTimerContext, Collector)}. + */ + public abstract class OnTimerContext extends Context { + /** + * The {@link TimeDomain} of the firing timer. + */ + public abstract TimeDomain timeDomain(); + + /** + * Get key of the firing timer. + */ + public abstract K getCurrentKey(); + } + +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java index 6501a9de7d586a..b74fdf3492eeca 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java @@ -23,7 +23,7 @@ import org.apache.flink.streaming.api.SimpleTimerService; import org.apache.flink.streaming.api.TimeDomain; import org.apache.flink.streaming.api.TimerService; -import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.streaming.api.functions.KeyedProcessFunction; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.OutputTag; @@ -31,12 +31,11 @@ import static org.apache.flink.util.Preconditions.checkState; /** - * A {@link org.apache.flink.streaming.api.operators.StreamOperator} for executing keyed - * {@link ProcessFunction ProcessFunctions}. + * A {@link StreamOperator} for executing {@link KeyedProcessFunction KeyedProcessFunctions}. */ @Internal public class KeyedProcessOperator - extends AbstractUdfStreamOperator> + extends AbstractUdfStreamOperator> implements OneInputStreamOperator, Triggerable { private static final long serialVersionUID = 1L; @@ -47,7 +46,7 @@ public class KeyedProcessOperator private transient OnTimerContextImpl onTimerContext; - public KeyedProcessOperator(ProcessFunction function) { + public KeyedProcessOperator(KeyedProcessFunction function) { super(function); chainingStrategy = ChainingStrategy.ALWAYS; @@ -70,21 +69,13 @@ public void open() throws Exception { @Override public void onEventTime(InternalTimer timer) throws Exception { collector.setAbsoluteTimestamp(timer.getTimestamp()); - onTimerContext.timeDomain = TimeDomain.EVENT_TIME; - onTimerContext.timer = timer; - userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector); - onTimerContext.timeDomain = null; - onTimerContext.timer = null; + invokeUserFunction(TimeDomain.EVENT_TIME, timer); } @Override public void onProcessingTime(InternalTimer timer) throws Exception { collector.eraseTimestamp(); - onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME; - onTimerContext.timer = timer; - userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector); - onTimerContext.timeDomain = null; - onTimerContext.timer = null; + invokeUserFunction(TimeDomain.PROCESSING_TIME, timer); } @Override @@ -95,13 +86,23 @@ public void processElement(StreamRecord element) throws Exception { context.element = null; } - private class ContextImpl extends ProcessFunction.Context { + private void invokeUserFunction( + TimeDomain timeDomain, + InternalTimer timer) throws Exception { + onTimerContext.timeDomain = timeDomain; + onTimerContext.timer = timer; + userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector); + onTimerContext.timeDomain = null; + onTimerContext.timer = null; + } + + private class ContextImpl extends KeyedProcessFunction.Context { private final TimerService timerService; private StreamRecord element; - ContextImpl(ProcessFunction function, TimerService timerService) { + ContextImpl(KeyedProcessFunction function, TimerService timerService) { function.super(); this.timerService = checkNotNull(timerService); } @@ -132,15 +133,15 @@ public void output(OutputTag outputTag, X value) { } } - private class OnTimerContextImpl extends ProcessFunction.OnTimerContext{ + private class OnTimerContextImpl extends KeyedProcessFunction.OnTimerContext { private final TimerService timerService; private TimeDomain timeDomain; - private InternalTimer timer; + private InternalTimer timer; - OnTimerContextImpl(ProcessFunction function, TimerService timerService) { + OnTimerContextImpl(KeyedProcessFunction function, TimerService timerService) { function.super(); this.timerService = checkNotNull(timerService); } @@ -170,5 +171,10 @@ public TimeDomain timeDomain() { checkState(timeDomain != null); return timeDomain; } + + @Override + public K getCurrentKey() { + return timer.getKey(); + } } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperator.java new file mode 100644 index 00000000000000..8481c4680bed43 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperator.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.operators; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.runtime.state.VoidNamespace; +import org.apache.flink.runtime.state.VoidNamespaceSerializer; +import org.apache.flink.streaming.api.SimpleTimerService; +import org.apache.flink.streaming.api.TimeDomain; +import org.apache.flink.streaming.api.TimerService; +import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; + +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; + +/** + * A {@link StreamOperator} for executing keyed {@link ProcessFunction ProcessFunctions}. + * + * @deprecated Replaced by {@link KeyedProcessOperator} which takes {@code KeyedProcessFunction} + */ +@Deprecated +@Internal +public class LegacyKeyedProcessOperator + extends AbstractUdfStreamOperator> + implements OneInputStreamOperator, Triggerable { + + private static final long serialVersionUID = 1L; + + private transient TimestampedCollector collector; + + private transient ContextImpl context; + + private transient OnTimerContextImpl onTimerContext; + + public LegacyKeyedProcessOperator(ProcessFunction function) { + super(function); + + chainingStrategy = ChainingStrategy.ALWAYS; + } + + @Override + public void open() throws Exception { + super.open(); + collector = new TimestampedCollector<>(output); + + InternalTimerService internalTimerService = + getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this); + + TimerService timerService = new SimpleTimerService(internalTimerService); + + context = new ContextImpl(userFunction, timerService); + onTimerContext = new OnTimerContextImpl(userFunction, timerService); + } + + @Override + public void onEventTime(InternalTimer timer) throws Exception { + collector.setAbsoluteTimestamp(timer.getTimestamp()); + invokeUserFunction(TimeDomain.EVENT_TIME, timer); + } + + @Override + public void onProcessingTime(InternalTimer timer) throws Exception { + collector.eraseTimestamp(); + invokeUserFunction(TimeDomain.PROCESSING_TIME, timer); + } + + @Override + public void processElement(StreamRecord element) throws Exception { + collector.setTimestamp(element); + context.element = element; + userFunction.processElement(element.getValue(), context, collector); + context.element = null; + } + + private void invokeUserFunction( + TimeDomain timeDomain, + InternalTimer timer) throws Exception { + onTimerContext.timeDomain = timeDomain; + onTimerContext.timer = timer; + userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector); + onTimerContext.timeDomain = null; + onTimerContext.timer = null; + } + + private class ContextImpl extends ProcessFunction.Context { + + private final TimerService timerService; + + private StreamRecord element; + + ContextImpl(ProcessFunction function, TimerService timerService) { + function.super(); + this.timerService = checkNotNull(timerService); + } + + @Override + public Long timestamp() { + checkState(element != null); + + if (element.hasTimestamp()) { + return element.getTimestamp(); + } else { + return null; + } + } + + @Override + public TimerService timerService() { + return timerService; + } + + @Override + public void output(OutputTag outputTag, X value) { + if (outputTag == null) { + throw new IllegalArgumentException("OutputTag must not be null."); + } + + output.collect(outputTag, new StreamRecord<>(value, element.getTimestamp())); + } + } + + private class OnTimerContextImpl extends ProcessFunction.OnTimerContext{ + + private final TimerService timerService; + + private TimeDomain timeDomain; + + private InternalTimer timer; + + OnTimerContextImpl(ProcessFunction function, TimerService timerService) { + function.super(); + this.timerService = checkNotNull(timerService); + } + + @Override + public Long timestamp() { + checkState(timer != null); + return timer.getTimestamp(); + } + + @Override + public TimerService timerService() { + return timerService; + } + + @Override + public void output(OutputTag outputTag, X value) { + if (outputTag == null) { + throw new IllegalArgumentException("OutputTag must not be null."); + } + + output.collect(outputTag, new StreamRecord<>(value, timer.getTimestamp())); + } + + @Override + public TimeDomain timeDomain() { + checkState(timeDomain != null); + return timeDomain; + } + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java index ec8a134e8248be..4fa3fc84ff2daa 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java @@ -49,6 +49,7 @@ import org.apache.flink.streaming.api.datastream.SplitStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; +import org.apache.flink.streaming.api.functions.KeyedProcessFunction; import org.apache.flink.streaming.api.functions.ProcessFunction; import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction; import org.apache.flink.streaming.api.functions.co.CoFlatMapFunction; @@ -61,6 +62,7 @@ import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator; import org.apache.flink.streaming.api.operators.KeyedProcessOperator; +import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator; import org.apache.flink.streaming.api.operators.ProcessOperator; import org.apache.flink.streaming.api.operators.StreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; @@ -689,11 +691,11 @@ public CustomPOJO fold(CustomPOJO accumulator, String value) throws Exception { } /** - * Verify that a {@link KeyedStream#process(ProcessFunction)} call is correctly translated to - * an operator. + * Verify that a {@link KeyedStream#process(ProcessFunction)} call is correctly translated to an operator. */ @Test - public void testKeyedProcessTranslation() { + @Deprecated + public void testKeyedStreamProcessTranslation() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource src = env.generateSequence(0, 0); @@ -724,12 +726,43 @@ public void onTimer( processed.addSink(new DiscardingSink()); assertEquals(processFunction, getFunctionForDataStream(processed)); + assertTrue(getOperatorForDataStream(processed) instanceof LegacyKeyedProcessOperator); + } + + /** + * Verify that a {@link KeyedStream#process(KeyedProcessFunction)} call is correctly translated to an operator. + */ + @Test + public void testKeyedStreamKeyedProcessTranslation() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + DataStreamSource src = env.generateSequence(0, 0); + + KeyedProcessFunction keyedProcessFunction = new KeyedProcessFunction() { + private static final long serialVersionUID = 1L; + + @Override + public void processElement(Long value, Context ctx, Collector out) throws Exception { + // Do nothing + } + + @Override + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + // Do nothing + } + }; + + DataStream processed = src + .keyBy(new IdentityKeySelector()) + .process(keyedProcessFunction); + + processed.addSink(new DiscardingSink()); + + assertEquals(keyedProcessFunction, getFunctionForDataStream(processed)); assertTrue(getOperatorForDataStream(processed) instanceof KeyedProcessOperator); } /** - * Verify that a {@link DataStream#process(ProcessFunction)} call is correctly translated to - * an operator. + * Verify that a {@link DataStream#process(ProcessFunction)} call is correctly translated to an operator. */ @Test public void testProcessTranslation() { diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/KeyedProcessOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/KeyedProcessOperatorTest.java index e1986f37edee29..c5f478cc8c510a 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/KeyedProcessOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/KeyedProcessOperatorTest.java @@ -24,7 +24,7 @@ import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; import org.apache.flink.streaming.api.TimeDomain; -import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.streaming.api.functions.KeyedProcessFunction; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; @@ -111,8 +111,10 @@ public void testTimestampAndProcessingTimeQuerying() throws Exception { @Test public void testEventTimeTimers() throws Exception { + final int expectedKey = 17; + KeyedProcessOperator operator = - new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME)); + new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME, expectedKey)); OneInputStreamOperatorTestHarness testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); @@ -122,14 +124,14 @@ public void testEventTimeTimers() throws Exception { testHarness.processWatermark(new Watermark(0)); - testHarness.processElement(new StreamRecord<>(17, 42L)); + testHarness.processElement(new StreamRecord<>(expectedKey, 42L)); testHarness.processWatermark(new Watermark(5)); ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(0L)); - expectedOutput.add(new StreamRecord<>(17, 42L)); + expectedOutput.add(new StreamRecord<>(expectedKey, 42L)); expectedOutput.add(new StreamRecord<>(1777, 5L)); expectedOutput.add(new Watermark(5L)); @@ -141,8 +143,10 @@ public void testEventTimeTimers() throws Exception { @Test public void testProcessingTimeTimers() throws Exception { + final int expectedKey = 17; + KeyedProcessOperator operator = - new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME)); + new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME, expectedKey)); OneInputStreamOperatorTestHarness testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); @@ -150,13 +154,13 @@ public void testProcessingTimeTimers() throws Exception { testHarness.setup(); testHarness.open(); - testHarness.processElement(new StreamRecord<>(17)); + testHarness.processElement(new StreamRecord<>(expectedKey)); testHarness.setProcessingTime(5); ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); - expectedOutput.add(new StreamRecord<>(17)); + expectedOutput.add(new StreamRecord<>(expectedKey)); expectedOutput.add(new StreamRecord<>(1777)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); @@ -243,8 +247,10 @@ public void testProcessingTimeTimerWithState() throws Exception { @Test public void testSnapshotAndRestore() throws Exception { + final int expectedKey = 5; + KeyedProcessOperator operator = - new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); + new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey)); OneInputStreamOperatorTestHarness testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); @@ -252,14 +258,14 @@ public void testSnapshotAndRestore() throws Exception { testHarness.setup(); testHarness.open(); - testHarness.processElement(new StreamRecord<>(5, 12L)); + testHarness.processElement(new StreamRecord<>(expectedKey, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); - operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); + operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey)); testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); @@ -283,8 +289,7 @@ public void testSnapshotAndRestore() throws Exception { @Test public void testNullOutputTagRefusal() throws Exception { - KeyedProcessOperator operator = - new KeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); + KeyedProcessOperator operator = new KeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness testHarness = new KeyedOneInputStreamOperatorTestHarness<>( @@ -307,8 +312,7 @@ public void testNullOutputTagRefusal() throws Exception { */ @Test public void testSideOutput() throws Exception { - KeyedProcessOperator operator = - new KeyedProcessOperator<>(new SideOutputProcessFunction()); + KeyedProcessOperator operator = new KeyedProcessOperator<>(new SideOutputProcessFunction()); OneInputStreamOperatorTestHarness testHarness = new KeyedOneInputStreamOperatorTestHarness<>( @@ -346,7 +350,7 @@ public void testSideOutput() throws Exception { testHarness.close(); } - private static class NullOutputTagEmittingProcessFunction extends ProcessFunction { + private static class NullOutputTagEmittingProcessFunction extends KeyedProcessFunction { @Override public void processElement(Integer value, Context ctx, Collector out) throws Exception { @@ -354,7 +358,7 @@ public void processElement(Integer value, Context ctx, Collector out) th } } - private static class SideOutputProcessFunction extends ProcessFunction { + private static class SideOutputProcessFunction extends KeyedProcessFunction { static final OutputTag INTEGER_OUTPUT_TAG = new OutputTag("int-out") {}; static final OutputTag LONG_OUTPUT_TAG = new OutputTag("long-out") {}; @@ -377,19 +381,19 @@ public T getKey(T value) throws Exception { } } - private static class QueryingFlatMapFunction extends ProcessFunction { + private static class QueryingFlatMapFunction extends KeyedProcessFunction { private static final long serialVersionUID = 1L; - private final TimeDomain timeDomain; + private final TimeDomain expectedTimeDomain; public QueryingFlatMapFunction(TimeDomain timeDomain) { - this.timeDomain = timeDomain; + this.expectedTimeDomain = timeDomain; } @Override public void processElement(Integer value, Context ctx, Collector out) throws Exception { - if (timeDomain.equals(TimeDomain.EVENT_TIME)) { + if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) { out.collect(value + "TIME:" + ctx.timerService().currentWatermark() + " TS:" + ctx.timestamp()); } else { out.collect(value + "TIME:" + ctx.timerService().currentProcessingTime() + " TS:" + ctx.timestamp()); @@ -401,23 +405,26 @@ public void onTimer( long timestamp, OnTimerContext ctx, Collector out) throws Exception { + // Do nothing } } - private static class TriggeringFlatMapFunction extends ProcessFunction { + private static class TriggeringFlatMapFunction extends KeyedProcessFunction { private static final long serialVersionUID = 1L; - private final TimeDomain timeDomain; + private final TimeDomain expectedTimeDomain; + private final Integer expectedKey; - public TriggeringFlatMapFunction(TimeDomain timeDomain) { - this.timeDomain = timeDomain; + public TriggeringFlatMapFunction(TimeDomain timeDomain, Integer expectedKey) { + this.expectedTimeDomain = timeDomain; + this.expectedKey = expectedKey; } @Override public void processElement(Integer value, Context ctx, Collector out) throws Exception { out.collect(value); - if (timeDomain.equals(TimeDomain.EVENT_TIME)) { + if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) { ctx.timerService().registerEventTimeTimer(ctx.timerService().currentWatermark() + 5); } else { ctx.timerService().registerProcessingTimeTimer(ctx.timerService().currentProcessingTime() + 5); @@ -429,30 +436,30 @@ public void onTimer( long timestamp, OnTimerContext ctx, Collector out) throws Exception { - - assertEquals(this.timeDomain, ctx.timeDomain()); + assertEquals(expectedKey, ctx.getCurrentKey()); + assertEquals(expectedTimeDomain, ctx.timeDomain()); out.collect(1777); } } - private static class TriggeringStatefulFlatMapFunction extends ProcessFunction { + private static class TriggeringStatefulFlatMapFunction extends KeyedProcessFunction { private static final long serialVersionUID = 1L; private final ValueStateDescriptor state = new ValueStateDescriptor<>("seen-element", IntSerializer.INSTANCE); - private final TimeDomain timeDomain; + private final TimeDomain expectedTimeDomain; public TriggeringStatefulFlatMapFunction(TimeDomain timeDomain) { - this.timeDomain = timeDomain; + this.expectedTimeDomain = timeDomain; } @Override public void processElement(Integer value, Context ctx, Collector out) throws Exception { out.collect("INPUT:" + value); getRuntimeContext().getState(state).update(value); - if (timeDomain.equals(TimeDomain.EVENT_TIME)) { + if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) { ctx.timerService().registerEventTimeTimer(ctx.timerService().currentWatermark() + 5); } else { ctx.timerService().registerProcessingTimeTimer(ctx.timerService().currentProcessingTime() + 5); @@ -464,15 +471,21 @@ public void onTimer( long timestamp, OnTimerContext ctx, Collector out) throws Exception { - assertEquals(this.timeDomain, ctx.timeDomain()); + assertEquals(expectedTimeDomain, ctx.timeDomain()); out.collect("STATE:" + getRuntimeContext().getState(state).value()); } } - private static class BothTriggeringFlatMapFunction extends ProcessFunction { + private static class BothTriggeringFlatMapFunction extends KeyedProcessFunction { private static final long serialVersionUID = 1L; + private final Integer expectedKey; + + public BothTriggeringFlatMapFunction(Integer expectedKey) { + this.expectedKey = expectedKey; + } + @Override public void processElement(Integer value, Context ctx, Collector out) throws Exception { ctx.timerService().registerProcessingTimeTimer(5); @@ -484,6 +497,8 @@ public void onTimer( long timestamp, OnTimerContext ctx, Collector out) throws Exception { + assertEquals(expectedKey, ctx.getCurrentKey()); + if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) { out.collect("EVENT:1777"); } else { @@ -491,5 +506,4 @@ public void onTimer( } } } - } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperatorTest.java new file mode 100644 index 00000000000000..970bb35ba75038 --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/LegacyKeyedProcessOperatorTest.java @@ -0,0 +1,483 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.operators; + +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.api.common.typeutils.base.IntSerializer; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.TimeDomain; +import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.flink.streaming.util.TestHarnessUtil; +import org.apache.flink.util.Collector; +import org.apache.flink.util.OutputTag; +import org.apache.flink.util.TestLogger; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.concurrent.ConcurrentLinkedQueue; + +import static org.junit.Assert.assertEquals; + +/** + * Tests {@link LegacyKeyedProcessOperator}. + */ +@Deprecated +public class LegacyKeyedProcessOperatorTest extends TestLogger { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testTimestampAndWatermarkQuerying() throws Exception { + + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.processWatermark(new Watermark(17)); + testHarness.processElement(new StreamRecord<>(5, 12L)); + + testHarness.processWatermark(new Watermark(42)); + testHarness.processElement(new StreamRecord<>(6, 13L)); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new Watermark(17L)); + expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); + expectedOutput.add(new Watermark(42L)); + expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testTimestampAndProcessingTimeQuerying() throws Exception { + + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.PROCESSING_TIME)); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.setProcessingTime(17); + testHarness.processElement(new StreamRecord<>(5)); + + testHarness.setProcessingTime(42); + testHarness.processElement(new StreamRecord<>(6)); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); + expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testEventTimeTimers() throws Exception { + + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME)); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.processWatermark(new Watermark(0)); + + testHarness.processElement(new StreamRecord<>(17, 42L)); + + testHarness.processWatermark(new Watermark(5)); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new Watermark(0L)); + expectedOutput.add(new StreamRecord<>(17, 42L)); + expectedOutput.add(new StreamRecord<>(1777, 5L)); + expectedOutput.add(new Watermark(5L)); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testProcessingTimeTimers() throws Exception { + + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME)); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(17)); + + testHarness.setProcessingTime(5); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new StreamRecord<>(17)); + expectedOutput.add(new StreamRecord<>(1777)); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + /** + * Verifies that we don't have leakage between different keys. + */ + @Test + public void testEventTimeTimerWithState() throws Exception { + + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME)); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.processWatermark(new Watermark(1)); + testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6 + + testHarness.processWatermark(new Watermark(2)); + testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7 + + testHarness.processWatermark(new Watermark(6)); + testHarness.processWatermark(new Watermark(7)); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new Watermark(1L)); + expectedOutput.add(new StreamRecord<>("INPUT:17", 0L)); + expectedOutput.add(new Watermark(2L)); + expectedOutput.add(new StreamRecord<>("INPUT:42", 1L)); + expectedOutput.add(new StreamRecord<>("STATE:17", 6L)); + expectedOutput.add(new Watermark(6L)); + expectedOutput.add(new StreamRecord<>("STATE:42", 7L)); + expectedOutput.add(new Watermark(7L)); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + /** + * Verifies that we don't have leakage between different keys. + */ + @Test + public void testProcessingTimeTimerWithState() throws Exception { + + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME)); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.setProcessingTime(1); + testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6 + + testHarness.setProcessingTime(2); + testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7 + + testHarness.setProcessingTime(6); + testHarness.setProcessingTime(7); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new StreamRecord<>("INPUT:17")); + expectedOutput.add(new StreamRecord<>("INPUT:42")); + expectedOutput.add(new StreamRecord<>("STATE:17")); + expectedOutput.add(new StreamRecord<>("STATE:42")); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testSnapshotAndRestore() throws Exception { + + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(5, 12L)); + + // snapshot and restore from scratch + OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); + + testHarness.close(); + + operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); + + testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.initializeState(snapshot); + testHarness.open(); + + testHarness.setProcessingTime(5); + testHarness.processWatermark(new Watermark(6)); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new StreamRecord<>("PROC:1777")); + expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L)); + expectedOutput.add(new Watermark(6)); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testNullOutputTagRefusal() throws Exception { + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>( + operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.setProcessingTime(17); + try { + expectedException.expect(IllegalArgumentException.class); + testHarness.processElement(new StreamRecord<>(5)); + } finally { + testHarness.close(); + } + } + + /** + * This also verifies that the timestamps ouf side-emitted records is correct. + */ + @Test + public void testSideOutput() throws Exception { + LegacyKeyedProcessOperator operator = + new LegacyKeyedProcessOperator<>(new SideOutputProcessFunction()); + + OneInputStreamOperatorTestHarness testHarness = + new KeyedOneInputStreamOperatorTestHarness<>( + operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); + + testHarness.setup(); + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */)); + + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */)); + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + ConcurrentLinkedQueue> expectedIntSideOutput = new ConcurrentLinkedQueue<>(); + expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */)); + ConcurrentLinkedQueue> intSideOutput = + testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); + TestHarnessUtil.assertOutputEquals( + "Side output was not correct.", + expectedIntSideOutput, + intSideOutput); + + ConcurrentLinkedQueue> expectedLongSideOutput = new ConcurrentLinkedQueue<>(); + expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); + ConcurrentLinkedQueue> longSideOutput = + testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); + TestHarnessUtil.assertOutputEquals( + "Side output was not correct.", + expectedLongSideOutput, + longSideOutput); + + testHarness.close(); + } + + private static class NullOutputTagEmittingProcessFunction extends ProcessFunction { + + @Override + public void processElement(Integer value, Context ctx, Collector out) throws Exception { + ctx.output(null, value); + } + } + + private static class SideOutputProcessFunction extends ProcessFunction { + + static final OutputTag INTEGER_OUTPUT_TAG = new OutputTag("int-out") {}; + static final OutputTag LONG_OUTPUT_TAG = new OutputTag("long-out") {}; + + @Override + public void processElement(Integer value, Context ctx, Collector out) throws Exception { + out.collect("IN:" + value); + + ctx.output(INTEGER_OUTPUT_TAG, value); + ctx.output(LONG_OUTPUT_TAG, value.longValue()); + } + } + + private static class IdentityKeySelector implements KeySelector { + private static final long serialVersionUID = 1L; + + @Override + public T getKey(T value) throws Exception { + return value; + } + } + + private static class QueryingFlatMapFunction extends ProcessFunction { + + private static final long serialVersionUID = 1L; + + private final TimeDomain timeDomain; + + public QueryingFlatMapFunction(TimeDomain timeDomain) { + this.timeDomain = timeDomain; + } + + @Override + public void processElement(Integer value, Context ctx, Collector out) throws Exception { + if (timeDomain.equals(TimeDomain.EVENT_TIME)) { + out.collect(value + "TIME:" + ctx.timerService().currentWatermark() + " TS:" + ctx.timestamp()); + } else { + out.collect(value + "TIME:" + ctx.timerService().currentProcessingTime() + " TS:" + ctx.timestamp()); + } + } + + @Override + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + // Do nothing + } + } + + private static class TriggeringFlatMapFunction extends ProcessFunction { + + private static final long serialVersionUID = 1L; + + private final TimeDomain timeDomain; + + public TriggeringFlatMapFunction(TimeDomain timeDomain) { + this.timeDomain = timeDomain; + } + + @Override + public void processElement(Integer value, Context ctx, Collector out) throws Exception { + out.collect(value); + if (timeDomain.equals(TimeDomain.EVENT_TIME)) { + ctx.timerService().registerEventTimeTimer(ctx.timerService().currentWatermark() + 5); + } else { + ctx.timerService().registerProcessingTimeTimer(ctx.timerService().currentProcessingTime() + 5); + } + } + + @Override + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + assertEquals(this.timeDomain, ctx.timeDomain()); + out.collect(1777); + } + } + + private static class TriggeringStatefulFlatMapFunction extends ProcessFunction { + + private static final long serialVersionUID = 1L; + + private final ValueStateDescriptor state = + new ValueStateDescriptor<>("seen-element", IntSerializer.INSTANCE); + + private final TimeDomain timeDomain; + + public TriggeringStatefulFlatMapFunction(TimeDomain timeDomain) { + this.timeDomain = timeDomain; + } + + @Override + public void processElement(Integer value, Context ctx, Collector out) throws Exception { + out.collect("INPUT:" + value); + getRuntimeContext().getState(state).update(value); + if (timeDomain.equals(TimeDomain.EVENT_TIME)) { + ctx.timerService().registerEventTimeTimer(ctx.timerService().currentWatermark() + 5); + } else { + ctx.timerService().registerProcessingTimeTimer(ctx.timerService().currentProcessingTime() + 5); + } + } + + @Override + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + assertEquals(this.timeDomain, ctx.timeDomain()); + out.collect("STATE:" + getRuntimeContext().getState(state).value()); + } + } + + private static class BothTriggeringFlatMapFunction extends ProcessFunction { + + private static final long serialVersionUID = 1L; + + @Override + public void processElement(Integer value, Context ctx, Collector out) throws Exception { + ctx.timerService().registerProcessingTimeTimer(5); + ctx.timerService().registerEventTimeTimer(6); + } + + @Override + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) { + out.collect("EVENT:1777"); + } else { + out.collect("PROC:1777"); + } + } + } +} diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala index 49bdbd9e70c413..51def984898eee 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala @@ -24,7 +24,7 @@ import org.apache.flink.api.common.state.{FoldingStateDescriptor, ReducingStateD import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.api.common.typeutils.TypeSerializer import org.apache.flink.streaming.api.datastream.{QueryableStateStream, DataStream => JavaStream, KeyedStream => KeyedJavaStream, WindowedStream => WindowedJavaStream} -import org.apache.flink.streaming.api.functions.ProcessFunction +import org.apache.flink.streaming.api.functions.{KeyedProcessFunction, ProcessFunction} import org.apache.flink.streaming.api.functions.aggregation.AggregationFunction.AggregationType import org.apache.flink.streaming.api.functions.aggregation.{ComparableAggregator, SumAggregator} import org.apache.flink.streaming.api.functions.query.{QueryableAppendingStateOperator, QueryableValueStateOperator} @@ -66,9 +66,11 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T] * function, this function can also query the time and set timers. When reacting to the firing * of set timers the function can directly emit elements and/or register yet more timers. * - * @param processFunction The [[ProcessFunction]] that is called for each element - * in the stream. + * @param processFunction The [[ProcessFunction]] that is called for each element in the stream. + * + * @deprecated Use [[KeyedStream#process(KeyedProcessFunction)]] */ + @deprecated("will be removed in a future version") @PublicEvolving override def process[R: TypeInformation]( processFunction: ProcessFunction[T, R]): DataStream[R] = { @@ -79,7 +81,34 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T] asScalaStream(javaStream.process(processFunction, implicitly[TypeInformation[R]])) } - + + /** + * Applies the given [[KeyedProcessFunction]] on the input stream, thereby + * creating a transformed output stream. + * + * The function will be called for every element in the stream and can produce + * zero or more output. The function can also query the time and set timers. When + * reacting to the firing of set timers the function can emit yet more elements. + * + * The function will be called for every element in the input streams and can produce zero + * or more output elements. Contrary to the [[DataStream#flatMap(FlatMapFunction)]] + * function, this function can also query the time and set timers. When reacting to the firing + * of set timers the function can directly emit elements and/or register yet more timers. + * + * @param keyedProcessFunction The [[KeyedProcessFunction]] that is called for each element + * in the stream. + */ + @PublicEvolving + def process[R: TypeInformation]( + keyedProcessFunction: KeyedProcessFunction[K, T, R]): DataStream[R] = { + + if (keyedProcessFunction == null) { + throw new NullPointerException("KeyedProcessFunction must not be null.") + } + + asScalaStream(javaStream.process(keyedProcessFunction, implicitly[TypeInformation[R]])) + } + // ------------------------------------------------------------------------ // Windowing // ------------------------------------------------------------------------ diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala index e2c5b416ec783a..51ec5e382307ae 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala @@ -23,13 +23,11 @@ import java.lang import org.apache.flink.api.common.functions._ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.api.collector.selector.OutputSelector -import org.apache.flink.streaming.api.functions.ProcessFunction +import org.apache.flink.streaming.api.functions.{KeyedProcessFunction, ProcessFunction} import org.apache.flink.streaming.api.functions.co.CoMapFunction -import org.apache.flink.streaming.api.functions.sink.DiscardingSink import org.apache.flink.streaming.api.graph.{StreamEdge, StreamGraph} -import org.apache.flink.streaming.api.operators.{AbstractUdfStreamOperator, KeyedProcessOperator, ProcessOperator, StreamOperator} +import org.apache.flink.streaming.api.operators._ import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows -import org.apache.flink.streaming.api.windowing.time.Time import org.apache.flink.streaming.api.windowing.triggers.{CountTrigger, PurgingTrigger} import org.apache.flink.streaming.api.windowing.windows.GlobalWindow import org.apache.flink.streaming.runtime.partitioner._ @@ -430,10 +428,11 @@ class DataStreamTest extends AbstractTestBase { } /** - * Verify that a [[KeyedStream.process()]] call is correctly translated to an operator. + * Verify that a [[KeyedStream.process(ProcessFunction)]] call is correctly + * translated to an operator. */ @Test - def testKeyedProcessTranslation(): Unit = { + def testKeyedStreamProcessTranslation(): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment val src = env.generateSequence(0, 0) @@ -448,12 +447,36 @@ class DataStreamTest extends AbstractTestBase { val flatMapped = src.keyBy(x => x).process(processFunction) assert(processFunction == getFunctionForDataStream(flatMapped)) + assert(getOperatorForDataStream(flatMapped).isInstanceOf[LegacyKeyedProcessOperator[_, _, _]]) + } + + /** + * Verify that a [[KeyedStream.process(KeyedProcessFunction)]] call is correctly + * translated to an operator. + */ + @Test + def testKeyedStreamKeyedProcessTranslation(): Unit = { + val env = StreamExecutionEnvironment.getExecutionEnvironment + + val src = env.generateSequence(0, 0) + + val keyedProcessFunction = new KeyedProcessFunction[Long, Long, Int] { + override def processElement( + value: Long, + ctx: KeyedProcessFunction[Long, Long, Int]#Context, + out: Collector[Int]): Unit = ??? + } + + val flatMapped = src.keyBy(x => x).process(keyedProcessFunction) + + assert(keyedProcessFunction == getFunctionForDataStream(flatMapped)) assert(getOperatorForDataStream(flatMapped).isInstanceOf[KeyedProcessOperator[_, _, _]]) } /** - * Verify that a [[DataStream.process()]] call is correctly translated to an operator. - */ + * Verify that a [[DataStream.process(ProcessFunction)]] call is correctly + * translated to an operator. + */ @Test def testProcessTranslation(): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -473,7 +496,6 @@ class DataStreamTest extends AbstractTestBase { assert(getOperatorForDataStream(flatMapped).isInstanceOf[ProcessOperator[_, _]]) } - @Test def operatorTest() { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -688,5 +710,4 @@ class DataStreamTest extends AbstractTestBase { m.print() m.getId } - } From 836998bd65ef2d0d0276faed189a0dfe8a7a6dc3 Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Thu, 15 Feb 2018 21:37:44 +0100 Subject: [PATCH 0117/2294] [FLINK-8667] Expose key in KeyedBroadcastProcessFunction#onTimer() This closes #5500. --- .../co/KeyedBroadcastProcessFunction.java | 5 ++ .../co/CoBroadcastWithKeyedOperator.java | 5 ++ .../flink/streaming/api/DataStreamTest.java | 8 +- .../co/CoBroadcastWithKeyedOperatorTest.java | 83 +++++++++++-------- .../api/scala/BroadcastStateITCase.scala | 14 +++- .../runtime/BroadcastStateITCase.java | 24 +++--- 6 files changed, 86 insertions(+), 53 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/KeyedBroadcastProcessFunction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/KeyedBroadcastProcessFunction.java index de9cb324dc3077..6e6ae5cb62cf7d 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/KeyedBroadcastProcessFunction.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/KeyedBroadcastProcessFunction.java @@ -170,5 +170,10 @@ public abstract class OnTimerContext extends KeyedReadOnlyContext { * event or processing time timer. */ public abstract TimeDomain timeDomain(); + + /** + * Get the key of the firing timer. + */ + public abstract KS getCurrentKey(); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperator.java index 2bdb6832b8c2d9..871363b68d5e5f 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperator.java @@ -324,6 +324,11 @@ public TimeDomain timeDomain() { return timeDomain; } + @Override + public KS getCurrentKey() { + return timer.getKey(); + } + @Override public TimerService timerService() { return timerService; diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java index 4fa3fc84ff2daa..632667217633f5 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/DataStreamTest.java @@ -707,7 +707,7 @@ public void processElement( Long value, Context ctx, Collector out) throws Exception { - + // Do nothing } @Override @@ -715,7 +715,7 @@ public void onTimer( long timestamp, OnTimerContext ctx, Collector out) throws Exception { - + // Do nothing } }; @@ -777,7 +777,7 @@ public void processElement( Long value, Context ctx, Collector out) throws Exception { - + // Do nothing } @Override @@ -785,7 +785,7 @@ public void onTimer( long timestamp, OnTimerContext ctx, Collector out) throws Exception { - + // Do nothing } }; diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperatorTest.java index 96607d404d26b0..b923b751d33b96 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperatorTest.java @@ -38,7 +38,6 @@ import org.apache.flink.util.OutputTag; import org.apache.flink.util.Preconditions; -import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; @@ -54,6 +53,11 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Function; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Tests for the {@link CoBroadcastWithKeyedOperator}. */ @@ -148,7 +152,7 @@ public void process(String key, ListState state) throws Exception { while (it.hasNext()) { list.add(it.next()); } - Assert.assertEquals(expectedKeyedStates.get(key), list); + assertEquals(expectedKeyedStates.get(key), list); } }); } @@ -161,12 +165,13 @@ public void processElement(String value, KeyedReadOnlyContext ctx, Collector testHarness = getInitializedTestHarness( BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), - new FunctionWithTimerOnKeyed(41L)) + new FunctionWithTimerOnKeyed(41L, expectedKey)) ) { testHarness.processWatermark1(new Watermark(10L)); testHarness.processWatermark2(new Watermark(10L)); @@ -174,8 +179,8 @@ public void testFunctionWithTimer() throws Exception { testHarness.processWatermark1(new Watermark(40L)); testHarness.processWatermark2(new Watermark(40L)); - testHarness.processElement1(new StreamRecord<>("6", 13L)); - testHarness.processElement1(new StreamRecord<>("6", 15L)); + testHarness.processElement1(new StreamRecord<>(expectedKey, 13L)); + testHarness.processElement1(new StreamRecord<>(expectedKey, 15L)); testHarness.processWatermark1(new Watermark(50L)); testHarness.processWatermark2(new Watermark(50L)); @@ -203,9 +208,11 @@ private static class FunctionWithTimerOnKeyed extends KeyedBroadcastProcessFunct private static final long serialVersionUID = 7496674620398203933L; private final long timerTS; + private final String expectedKey; - FunctionWithTimerOnKeyed(long timerTS) { + FunctionWithTimerOnKeyed(long timerTS, String expectedKey) { this.timerTS = timerTS; + this.expectedKey = expectedKey; } @Override @@ -221,6 +228,7 @@ public void processElement(String value, KeyedReadOnlyContext ctx, Collector out) throws Exception { + assertEquals(expectedKey, ctx.getCurrentKey()); out.collect("TIMER:" + timestamp); } } @@ -293,7 +301,6 @@ public void processElement(String value, KeyedReadOnlyContext ctx, Collector expectedBroadcastState = new HashMap<>(); expectedBroadcastState.put("5.key", 5); expectedBroadcastState.put("34.key", 34); @@ -301,11 +308,13 @@ public void testFunctionWithBroadcastState() throws Exception { expectedBroadcastState.put("12.key", 12); expectedBroadcastState.put("98.key", 98); + final String expectedKey = "trigger"; + try ( TwoInputStreamOperatorTestHarness testHarness = getInitializedTestHarness( BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), - new FunctionWithBroadcastState("key", expectedBroadcastState, 41L)) + new FunctionWithBroadcastState("key", expectedBroadcastState, 41L, expectedKey)) ) { testHarness.processWatermark1(new Watermark(10L)); testHarness.processWatermark2(new Watermark(10L)); @@ -316,7 +325,7 @@ public void testFunctionWithBroadcastState() throws Exception { testHarness.processElement2(new StreamRecord<>(12, 16L)); testHarness.processElement2(new StreamRecord<>(98, 19L)); - testHarness.processElement1(new StreamRecord<>("trigger", 13L)); + testHarness.processElement1(new StreamRecord<>(expectedKey, 13L)); testHarness.processElement2(new StreamRecord<>(51, 21L)); @@ -324,29 +333,29 @@ public void testFunctionWithBroadcastState() throws Exception { testHarness.processWatermark2(new Watermark(50L)); Queue output = testHarness.getOutput(); - Assert.assertEquals(3L, output.size()); + assertEquals(3L, output.size()); Object firstRawWm = output.poll(); - Assert.assertTrue(firstRawWm instanceof Watermark); + assertTrue(firstRawWm instanceof Watermark); Watermark firstWm = (Watermark) firstRawWm; - Assert.assertEquals(10L, firstWm.getTimestamp()); + assertEquals(10L, firstWm.getTimestamp()); Object rawOutputElem = output.poll(); - Assert.assertTrue(rawOutputElem instanceof StreamRecord); + assertTrue(rawOutputElem instanceof StreamRecord); StreamRecord outputRec = (StreamRecord) rawOutputElem; - Assert.assertTrue(outputRec.getValue() instanceof String); + assertTrue(outputRec.getValue() instanceof String); String outputElem = (String) outputRec.getValue(); expectedBroadcastState.put("51.key", 51); List> expectedEntries = new ArrayList<>(); expectedEntries.addAll(expectedBroadcastState.entrySet()); String expected = "TS:41 " + mapToString(expectedEntries); - Assert.assertEquals(expected, outputElem); + assertEquals(expected, outputElem); Object secondRawWm = output.poll(); - Assert.assertTrue(secondRawWm instanceof Watermark); + assertTrue(secondRawWm instanceof Watermark); Watermark secondWm = (Watermark) secondRawWm; - Assert.assertEquals(50L, secondWm.getTimestamp()); + assertEquals(50L, secondWm.getTimestamp()); } } @@ -357,15 +366,17 @@ private static class FunctionWithBroadcastState extends KeyedBroadcastProcessFun private final String keyPostfix; private final Map expectedBroadcastState; private final long timerTs; + private final String expectedKey; FunctionWithBroadcastState( final String keyPostfix, final Map expectedBroadcastState, - final long timerTs - ) { + final long timerTs, + final String expectedKey) { this.keyPostfix = Preconditions.checkNotNull(keyPostfix); this.expectedBroadcastState = Preconditions.checkNotNull(expectedBroadcastState); this.timerTs = timerTs; + this.expectedKey = expectedKey; } @Override @@ -381,14 +392,14 @@ public void processElement(String value, KeyedReadOnlyContext ctx, Collector> iter = broadcastStateIt.iterator(); for (int i = 0; i < expectedBroadcastState.size(); i++) { - Assert.assertTrue(iter.hasNext()); + assertTrue(iter.hasNext()); Map.Entry entry = iter.next(); - Assert.assertTrue(expectedBroadcastState.containsKey(entry.getKey())); - Assert.assertEquals(expectedBroadcastState.get(entry.getKey()), entry.getValue()); + assertTrue(expectedBroadcastState.containsKey(entry.getKey())); + assertEquals(expectedBroadcastState.get(entry.getKey()), entry.getValue()); } - Assert.assertFalse(iter.hasNext()); + assertFalse(iter.hasNext()); ctx.timerService().registerEventTimeTimer(timerTs); } @@ -401,6 +412,8 @@ public void onTimer(long timestamp, OnTimerContext ctx, Collector out) t while (iter.hasNext()) { map.add(iter.next()); } + + assertEquals(expectedKey, ctx.getCurrentKey()); final String mapToStr = mapToString(map); out.collect("TS:" + timestamp + " " + mapToStr); } @@ -485,22 +498,22 @@ public void testScaleUp() throws Exception { Queue output2 = testHarness2.getOutput(); Queue output3 = testHarness3.getOutput(); - Assert.assertEquals(expected.size(), output1.size()); + assertEquals(expected.size(), output1.size()); for (Object o: output1) { StreamRecord rec = (StreamRecord) o; - Assert.assertTrue(expected.contains(rec.getValue())); + assertTrue(expected.contains(rec.getValue())); } - Assert.assertEquals(expected.size(), output2.size()); + assertEquals(expected.size(), output2.size()); for (Object o: output2) { StreamRecord rec = (StreamRecord) o; - Assert.assertTrue(expected.contains(rec.getValue())); + assertTrue(expected.contains(rec.getValue())); } - Assert.assertEquals(expected.size(), output3.size()); + assertEquals(expected.size(), output3.size()); for (Object o: output3) { StreamRecord rec = (StreamRecord) o; - Assert.assertTrue(expected.contains(rec.getValue())); + assertTrue(expected.contains(rec.getValue())); } } } @@ -583,16 +596,16 @@ public void testScaleDown() throws Exception { Queue output1 = testHarness1.getOutput(); Queue output2 = testHarness2.getOutput(); - Assert.assertEquals(expected.size(), output1.size()); + assertEquals(expected.size(), output1.size()); for (Object o: output1) { StreamRecord rec = (StreamRecord) o; - Assert.assertTrue(expected.contains(rec.getValue())); + assertTrue(expected.contains(rec.getValue())); } - Assert.assertEquals(expected.size(), output2.size()); + assertEquals(expected.size(), output2.size()); for (Object o: output2) { StreamRecord rec = (StreamRecord) o; - Assert.assertTrue(expected.contains(rec.getValue())); + assertTrue(expected.contains(rec.getValue())); } } } @@ -653,12 +666,12 @@ public void processElement(String value, KeyedReadOnlyContext ctx, Collector(5, 12L)); } catch (NullPointerException e) { - Assert.assertEquals("No key set. This method should not be called outside of a keyed context.", e.getMessage()); + assertEquals("No key set. This method should not be called outside of a keyed context.", e.getMessage()); exceptionThrown = true; } if (!exceptionThrown) { - Assert.fail("No exception thrown"); + fail("No exception thrown"); } } diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/BroadcastStateITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/BroadcastStateITCase.scala index 6c382d5f4c5d6c..55bb3ba420152f 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/BroadcastStateITCase.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/BroadcastStateITCase.scala @@ -28,7 +28,7 @@ import org.apache.flink.streaming.api.watermark.Watermark import org.apache.flink.test.util.AbstractTestBase import org.apache.flink.util.Collector import org.junit.Assert.assertEquals -import org.junit.{Assert, Test} +import org.junit.{Test} /** * ITCase for the [[org.apache.flink.api.common.state.BroadcastState]]. @@ -103,13 +103,19 @@ class TestBroadcastProcessFunction( BasicTypeInfo.LONG_TYPE_INFO.asInstanceOf[TypeInformation[Long]], BasicTypeInfo.STRING_TYPE_INFO) + var timerToExpectedKey = Map[Long, Long]() + var nextTimerTimestamp :Long = expectedTimestamp + @throws[Exception] override def processElement( value: Long, ctx: KeyedBroadcastProcessFunction[Long, Long, String, String]#KeyedReadOnlyContext, out: Collector[String]): Unit = { - ctx.timerService.registerEventTimeTimer(expectedTimestamp) + val currentTime = nextTimerTimestamp + nextTimerTimestamp += 1 + ctx.timerService.registerEventTimeTimer(currentTime) + timerToExpectedKey += (currentTime -> value) } @throws[Exception] @@ -128,6 +134,8 @@ class TestBroadcastProcessFunction( ctx: KeyedBroadcastProcessFunction[Long, Long, String, String]#OnTimerContext, out: Collector[String]): Unit = { + assertEquals(timerToExpectedKey(timestamp), ctx.getCurrentKey) + var map = Map[Long, String]() import scala.collection.JavaConversions._ @@ -137,7 +145,7 @@ class TestBroadcastProcessFunction( map += (entry.getKey -> entry.getValue) } - Assert.assertEquals(expectedBroadcastState, map) + assertEquals(expectedBroadcastState, map) out.collect(timestamp.toString) } diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/BroadcastStateITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/BroadcastStateITCase.java index 868aca91a9d8f2..7ccba3337e6f73 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/BroadcastStateITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/BroadcastStateITCase.java @@ -32,7 +32,6 @@ import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.util.Collector; -import org.junit.Assert; import org.junit.Test; import javax.annotation.Nullable; @@ -40,6 +39,8 @@ import java.util.HashMap; import java.util.Map; +import static org.junit.Assert.assertEquals; + /** * ITCase for the {@link org.apache.flink.api.common.state.BroadcastState}. */ @@ -120,7 +121,7 @@ public void close() throws Exception { super.close(); // make sure that all the timers fired - Assert.assertEquals(expectedOutputCounter, outputCounter); + assertEquals(expectedOutputCounter, outputCounter); } } @@ -145,17 +146,15 @@ private static class TestBroadcastProcessFunction extends KeyedBroadcastProcessF private static final long serialVersionUID = 7616910653561100842L; private final Map expectedState; + private final Map timerToExpectedKey = new HashMap<>(); - private final long timerTimestamp; + private long nextTimerTimestamp; private transient MapStateDescriptor descriptor; - TestBroadcastProcessFunction( - final long timerTS, - final Map expectedBroadcastState - ) { + TestBroadcastProcessFunction(final long initialTimerTimestamp, final Map expectedBroadcastState) { expectedState = expectedBroadcastState; - timerTimestamp = timerTS; + nextTimerTimestamp = initialTimerTimestamp; } @Override @@ -169,7 +168,10 @@ public void open(Configuration parameters) throws Exception { @Override public void processElement(Long value, KeyedReadOnlyContext ctx, Collector out) throws Exception { - ctx.timerService().registerEventTimeTimer(timerTimestamp); + long currentTime = nextTimerTimestamp; + nextTimerTimestamp++; + ctx.timerService().registerEventTimeTimer(currentTime); + timerToExpectedKey.put(currentTime, value); } @Override @@ -180,14 +182,14 @@ public void processBroadcastElement(String value, KeyedContext ctx, Collector out) throws Exception { - Assert.assertEquals(timerTimestamp, timestamp); + assertEquals(timerToExpectedKey.get(timestamp), ctx.getCurrentKey()); Map map = new HashMap<>(); for (Map.Entry entry : ctx.getBroadcastState(descriptor).immutableEntries()) { map.put(entry.getKey(), entry.getValue()); } - Assert.assertEquals(expectedState, map); + assertEquals(expectedState, map); out.collect(Long.toString(timestamp)); } From fb72a802b01ac9c4cb5fd3d3ca2032e9af8a1064 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 7 Mar 2018 11:58:07 +0100 Subject: [PATCH 0118/2294] [FLINK-8890] Compare checkpoints with order in CompletedCheckpoint.checkpointsMatch() This method is used, among other things, to check if a list of restored checkpoints is stable after several restore attempts in the ZooKeeper checkpoint store. The order of checkpoints is somewhat important because we want the latest checkpoint to stay the latest checkpoint. --- .../checkpoint/CompletedCheckpoint.java | 12 +- .../checkpoint/CompletedCheckpointTest.java | 123 ++++++++++++++++++ 2 files changed, 129 insertions(+), 6 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java index 58424272bbc8de..1932b19925cb4d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpoint.java @@ -38,9 +38,8 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.List; import java.util.Map; -import java.util.Set; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -289,17 +288,18 @@ private void doDiscard() throws Exception { public static boolean checkpointsMatch( Collection first, Collection second) { + if (first.size() != second.size()) { + return false; + } - Set> firstInterestingFields = - new HashSet<>(); + List> firstInterestingFields = new ArrayList<>(first.size()); for (CompletedCheckpoint checkpoint : first) { firstInterestingFields.add( new Tuple2<>(checkpoint.getCheckpointID(), checkpoint.getJobId())); } - Set> secondInterestingFields = - new HashSet<>(); + List> secondInterestingFields = new ArrayList<>(second.size()); for (CompletedCheckpoint checkpoint : second) { secondInterestingFields.add( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java index 69003cd984c2ae..5af7c76ec6eff7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java @@ -31,8 +31,10 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -50,6 +52,127 @@ public class CompletedCheckpointTest { @Rule public final TemporaryFolder tmpFolder = new TemporaryFolder(); + @Test + public void testCompareCheckpointsWithDifferentOrder() { + + CompletedCheckpoint checkpoint1 = new CompletedCheckpoint( + new JobID(), 0, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + CompletedCheckpoint checkpoint2 = new CompletedCheckpoint( + new JobID(), 1, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + List checkpoints1= new ArrayList<>(); + checkpoints1.add(checkpoint1); + checkpoints1.add(checkpoint2); + checkpoints1.add(checkpoint1); + + List checkpoints2 = new ArrayList<>(); + checkpoints2.add(checkpoint2); + checkpoints2.add(checkpoint1); + checkpoints2.add(checkpoint2); + + assertFalse(CompletedCheckpoint.checkpointsMatch(checkpoints1, checkpoints2)); + } + + @Test + public void testCompareCheckpointsWithSameOrder() { + + CompletedCheckpoint checkpoint1 = new CompletedCheckpoint( + new JobID(), 0, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + CompletedCheckpoint checkpoint2 = new CompletedCheckpoint( + new JobID(), 1, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + List checkpoints1= new ArrayList<>(); + checkpoints1.add(checkpoint1); + checkpoints1.add(checkpoint2); + checkpoints1.add(checkpoint1); + + List checkpoints2 = new ArrayList<>(); + checkpoints2.add(checkpoint1); + checkpoints2.add(checkpoint2); + checkpoints2.add(checkpoint1); + + assertTrue(CompletedCheckpoint.checkpointsMatch(checkpoints1, checkpoints2)); + } + + /** + * Verify that both JobID and checkpoint id are taken into account when comparing. + */ + @Test + public void testCompareCheckpointsWithSameJobID() { + JobID jobID = new JobID(); + + CompletedCheckpoint checkpoint1 = new CompletedCheckpoint( + jobID, 0, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + CompletedCheckpoint checkpoint2 = new CompletedCheckpoint( + jobID, 1, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + List checkpoints1= new ArrayList<>(); + checkpoints1.add(checkpoint1); + + List checkpoints2 = new ArrayList<>(); + checkpoints2.add(checkpoint2); + + assertFalse(CompletedCheckpoint.checkpointsMatch(checkpoints1, checkpoints2)); + } + + /** + * Verify that both JobID and checkpoint id are taken into account when comparing. + */ + @Test + public void testCompareCheckpointsWithSameCheckpointId() { + JobID jobID1 = new JobID(); + JobID jobID2 = new JobID(); + + CompletedCheckpoint checkpoint1 = new CompletedCheckpoint( + jobID1, 0, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + CompletedCheckpoint checkpoint2 = new CompletedCheckpoint( + jobID2, 0, 0, 1, + new HashMap<>(), + Collections.emptyList(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), + new TestCompletedCheckpointStorageLocation()); + + List checkpoints1= new ArrayList<>(); + checkpoints1.add(checkpoint1); + + List checkpoints2 = new ArrayList<>(); + checkpoints2.add(checkpoint2); + + assertFalse(CompletedCheckpoint.checkpointsMatch(checkpoints1, checkpoints2)); + } + @Test public void testRegisterStatesAtRegistry() { OperatorState state = mock(OperatorState.class); From 6b7e9896eb6a4e2fc628403c9159b7652db2e311 Mon Sep 17 00:00:00 2001 From: davidxdh Date: Fri, 2 Mar 2018 10:55:23 +0800 Subject: [PATCH 0119/2294] [FLINK-8827] [scripts] When FLINK_CONF_DIR contains spaces, ZooKeeper related scripts fail This closes #5614 --- flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh | 6 +++--- flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh | 6 +++--- flink-dist/src/main/flink-bin/bin/zookeeper.sh | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh b/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh index cc8917b054737b..d5a7593a61f3a4 100755 --- a/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh +++ b/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh @@ -24,8 +24,8 @@ bin=`cd "$bin"; pwd` # Starts a ZooKeeper quorum as configured in $FLINK_CONF/zoo.cfg -ZK_CONF=$FLINK_CONF_DIR/zoo.cfg -if [ ! -f $ZK_CONF ]; then +ZK_CONF="$FLINK_CONF_DIR/zoo.cfg" +if [ ! -f "$ZK_CONF" ]; then echo "[ERROR] No ZooKeeper configuration file found in '$ZK_CONF'." exit 1 fi @@ -43,4 +43,4 @@ while read server ; do else echo "[WARN] Parse error. Skipping config entry '$server'." fi -done < <(grep "^server\." $ZK_CONF) +done < <(grep "^server\." "$ZK_CONF") diff --git a/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh b/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh index 29ddae4193119b..ad79de83a12963 100755 --- a/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh +++ b/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh @@ -24,8 +24,8 @@ bin=`cd "$bin"; pwd` # Stops a ZooKeeper quorum as configured in $FLINK_CONF/zoo.cfg -ZK_CONF=$FLINK_CONF_DIR/zoo.cfg -if [ ! -f $ZK_CONF ]; then +ZK_CONF="$FLINK_CONF_DIR/zoo.cfg" +if [ ! -f "$ZK_CONF" ]; then echo "[ERROR] No ZooKeeper configuration file found in '$ZK_CONF'." exit 1 fi @@ -43,4 +43,4 @@ while read server ; do else echo "[WARN] Parse error. Skipping config entry '$server'." fi -done < <(grep "^server\." $ZK_CONF) +done < <(grep "^server\." "$ZK_CONF") diff --git a/flink-dist/src/main/flink-bin/bin/zookeeper.sh b/flink-dist/src/main/flink-bin/bin/zookeeper.sh index ca72bb7eae53ed..53d709833656cf 100755 --- a/flink-dist/src/main/flink-bin/bin/zookeeper.sh +++ b/flink-dist/src/main/flink-bin/bin/zookeeper.sh @@ -33,8 +33,8 @@ bin=`cd "$bin"; pwd` . "$bin"/config.sh -ZK_CONF=$FLINK_CONF_DIR/zoo.cfg -if [ ! -f $ZK_CONF ]; then +ZK_CONF="$FLINK_CONF_DIR/zoo.cfg" +if [ ! -f "$ZK_CONF" ]; then echo "[ERROR] No ZooKeeper configuration file found in '$ZK_CONF'." exit 1 fi From 75a4aaea8b051aa6dae52d421f7b4d7eeab99486 Mon Sep 17 00:00:00 2001 From: zhangminglei Date: Fri, 2 Mar 2018 20:31:49 +0800 Subject: [PATCH 0120/2294] [FLINK-8824] [kafka connector] Replace Class.getCanonicalName() with Class.getName() This closes #5620 --- .../streaming/connectors/kafka/FlinkKafkaProducer011.java | 4 ++-- .../streaming/connectors/kafka/FlinkKafkaConsumer09.java | 2 +- .../streaming/connectors/kafka/FlinkKafkaProducerBase.java | 4 ++-- .../connectors/kafka/FlinkKafkaProducerBaseTest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java index 3fe4bc6b015f2c..e92f38b3ea0633 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java @@ -492,13 +492,13 @@ public FlinkKafkaProducer011( // set the producer configuration properties for kafka record key value serializers. if (!producerConfig.containsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG)) { - this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); + this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); } else { LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG); } if (!producerConfig.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG)) { - this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); + this.producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); } else { LOG.warn("Overwriting the '{}' is not recommended", ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG); } diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java index 497003293415f4..00b7da401e8d11 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java @@ -296,7 +296,7 @@ protected Map fetchOffsetsWithTimestamp(Collection Date: Tue, 6 Mar 2018 18:04:05 +0000 Subject: [PATCH 0121/2294] [hotfix] [javadoc] Minor javadoc fix in TimestampAssigner.java This closes #5646 Also close unrelated lingering pull request: This closes #5643 --- .../flink/streaming/api/functions/TimestampAssigner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java index 60debb966d07ef..ba7bdd87e16c2e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/TimestampAssigner.java @@ -40,9 +40,9 @@ public interface TimestampAssigner extends Function { * by ingestion time. If the element did not carry a timestamp before, this value is * {@code Long.MIN_VALUE}. * - * @param element The element that the timestamp is wil be assigned to. + * @param element The element that the timestamp will be assigned to. * @param previousElementTimestamp The previous internal timestamp of the element, - * or a negative value, if no timestamp has been assigned, yet. + * or a negative value, if no timestamp has been assigned yet. * @return The new timestamp. */ long extractTimestamp(T element, long previousElementTimestamp); From 06110d27d5fcbf939610e2adc780e7ad1c467f6f Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Sun, 4 Mar 2018 12:11:29 +0100 Subject: [PATCH 0122/2294] [FLINK-8877] [core] Set Kryo trace if Flink log level is TRACE --- .../runtime/kryo/KryoSerializer.java | 14 +++++ .../runtime/kryo/MinlogForwarder.java | 61 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/MinlogForwarder.java diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java index f60ce460e6d765..06ba906c241b0a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java @@ -74,6 +74,10 @@ public class KryoSerializer extends TypeSerializer { private static final Logger LOG = LoggerFactory.getLogger(KryoSerializer.class); + static { + configureKryoLogging(); + } + // ------------------------------------------------------------------------ private final LinkedHashMap, ExecutionConfig.SerializableSerializer> defaultSerializers; @@ -483,6 +487,16 @@ private static LinkedHashMap buildKryoRegistrations( return kryoRegistrations; } + static void configureKryoLogging() { + // Kryo uses only DEBUG and TRACE levels + // we only forward TRACE level, because even DEBUG levels results in + // a logging for each object, which is infeasible in Flink. + if (LOG.isTraceEnabled()) { + com.esotericsoftware.minlog.Log.setLogger(new MinlogForwarder(LOG)); + com.esotericsoftware.minlog.Log.TRACE(); + } + } + // -------------------------------------------------------------------------------------------- private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/MinlogForwarder.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/MinlogForwarder.java new file mode 100644 index 00000000000000..3467923f9fc034 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/MinlogForwarder.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.java.typeutils.runtime.kryo; + +import org.apache.flink.annotation.Internal; + +import com.esotericsoftware.minlog.Log; +import com.esotericsoftware.minlog.Log.Logger; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * An implementation of the Minlog Logger that forwards to slf4j. + */ +@Internal +class MinlogForwarder extends Logger { + + private final org.slf4j.Logger log; + + MinlogForwarder(org.slf4j.Logger log) { + this.log = checkNotNull(log); + } + + @Override + public void log (int level, String category, String message, Throwable ex) { + final String logString = "[KRYO " + category + "] " + message; + switch (level) { + case Log.LEVEL_ERROR: + log.error(logString, ex); + break; + case Log.LEVEL_WARN: + log.warn(logString, ex); + break; + case Log.LEVEL_INFO: + log.info(logString, ex); + break; + case Log.LEVEL_DEBUG: + log.debug(logString, ex); + break; + case Log.LEVEL_TRACE: + log.trace(logString, ex); + break; + } + } +} From 420122452f12c96a03d5eb0e66e074c607e1af0c Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 6 Mar 2018 11:30:54 +0100 Subject: [PATCH 0123/2294] [FLINK-8878] [tests] Add BlockerSync utility This helps to synchronize two threads of which one is expected to block while holding a resource. --- .../flink/core/testutils/BlockerSync.java | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/BlockerSync.java diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/BlockerSync.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/BlockerSync.java new file mode 100644 index 00000000000000..fb854f87d54bba --- /dev/null +++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/core/testutils/BlockerSync.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.core.testutils; + +/** + * A utility to help synchronize two threads in cases where one of them is supposed to reach + * a blocking state before the other may continue. + * + *

    Use as follows: + * 

    + * {@code
+ *
+ * final BlockerSync sync = new BlockerSync();
+ *
+ * // thread to be blocked
+ * Runnable toBeBlocked = () -> {
+ *     // do something, like acquire a shared resource
+ *     sync.blockNonInterruptible();
+ *     // release resource
+ * }
+ *
+ * new Thread(toBeBlocked).start();
+ * sync.awaitBlocker();
+ *
+ * // do stuff that requires the other thread to still hold the resource
+ * sync.releaseBlocker();
+ * }
+ *
    + */ +public class BlockerSync { + + private final Object lock = new Object(); + + private boolean blockerReady; + + private boolean blockerReleased; + + /** + * Waits until the blocking thread has entered the method {@link #block()} + * or {@link #blockNonInterruptible()}. + */ + public void awaitBlocker() throws InterruptedException { + synchronized (lock) { + while (!blockerReady) { + lock.wait(); + } + } + } + + /** + * Blocks until {@link #releaseBlocker()} is called or this thread is interrupted. + * Notifies the awaiting thread that waits in the method {@link #awaitBlocker()}. + */ + public void block() throws InterruptedException { + synchronized (lock) { + blockerReady = true; + lock.notifyAll(); + + while (!blockerReleased) { + lock.wait(); + } + } + } + + /** + * Blocks until {@link #releaseBlocker()} is called. + * Notifies the awaiting thread that waits in the method {@link #awaitBlocker()}. + */ + public void blockNonInterruptible() { + synchronized (lock) { + blockerReady = true; + lock.notifyAll(); + + while (!blockerReleased) { + try { + lock.wait(); + } catch (InterruptedException ignored) {} + } + } + } + + /** + * Lets the blocked thread continue. + */ + public void releaseBlocker() { + synchronized (lock) { + blockerReleased = true; + lock.notifyAll(); + } + } +} From 57ff6e8930db0bfdd8e7cbb8418d9a4b46ca4a61 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Sun, 4 Mar 2018 12:20:17 +0100 Subject: [PATCH 0124/2294] [FLINK-8878] [core] Add concurrency check Kryo Serializer on DEBUG level --- .../runtime/kryo/KryoSerializer.java | 191 ++++++++++++------ .../kryo/KryoSerializerDebugInitHelper.java | 47 +++++ ...ializerConcurrencyCheckInactiveITCase.java | 62 ++++++ .../kryo/KryoSerializerConcurrencyTest.java | 95 +++++++++ 4 files changed, 338 insertions(+), 57 deletions(-) create mode 100644 flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerDebugInitHelper.java create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyCheckInactiveITCase.java create mode 100644 flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyTest.java diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java index 06ba906c241b0a..7c97c5c44e18db 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java @@ -74,6 +74,12 @@ public class KryoSerializer extends TypeSerializer { private static final Logger LOG = LoggerFactory.getLogger(KryoSerializer.class); + /** Flag whether to check for concurrent thread access. + * Because this flag is static final, a value of 'false' allows the JIT compiler to eliminate + * the guarded code sections. */ + private static final boolean CONCURRENT_ACCESS_CHECK = + LOG.isDebugEnabled() || KryoSerializerDebugInitHelper.setToDebug; + static { configureKryoLogging(); } @@ -112,6 +118,9 @@ public class KryoSerializer extends TypeSerializer { private LinkedHashMap, Class>> registeredTypesWithSerializerClasses; private LinkedHashSet> registeredTypes; + // for debugging purposes + private transient volatile Thread currentThread; + // ------------------------------------------------------------------------ public KryoSerializer(Class type, ExecutionConfig executionConfig){ @@ -174,26 +183,38 @@ public T copy(T from) { if (from == null) { return null; } - checkKryoInitialized(); - try { - return kryo.copy(from); + + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); } - catch(KryoException ke) { - // kryo was unable to copy it, so we do it through serialization: - ByteArrayOutputStream baout = new ByteArrayOutputStream(); - Output output = new Output(baout); - kryo.writeObject(output, from); + try { + checkKryoInitialized(); + try { + return kryo.copy(from); + } + catch (KryoException ke) { + // kryo was unable to copy it, so we do it through serialization: + ByteArrayOutputStream baout = new ByteArrayOutputStream(); + Output output = new Output(baout); + + kryo.writeObject(output, from); - output.close(); + output.close(); - ByteArrayInputStream bain = new ByteArrayInputStream(baout.toByteArray()); - Input input = new Input(bain); + ByteArrayInputStream bain = new ByteArrayInputStream(baout.toByteArray()); + Input input = new Input(bain); - return (T)kryo.readObject(input, from.getClass()); + return (T)kryo.readObject(input, from.getClass()); + } + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); + } } } - + @Override public T copy(T from, T reuse) { return copy(from); @@ -206,35 +227,47 @@ public int getLength() { @Override public void serialize(T record, DataOutputView target) throws IOException { - checkKryoInitialized(); - if (target != previousOut) { - DataOutputViewStream outputStream = new DataOutputViewStream(target); - output = new Output(outputStream); - previousOut = target; - } - - // Sanity check: Make sure that the output is cleared/has been flushed by the last call - // otherwise data might be written multiple times in case of a previous EOFException - if (output.position() != 0) { - throw new IllegalStateException("The Kryo Output still contains data from a previous " + - "serialize call. It has to be flushed or cleared at the end of the serialize call."); + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); } try { - kryo.writeClassAndObject(output, record); - output.flush(); - } - catch (KryoException ke) { - // make sure that the Kryo output buffer is cleared in case that we can recover from - // the exception (e.g. EOFException which denotes buffer full) - output.clear(); - - Throwable cause = ke.getCause(); - if (cause instanceof EOFException) { - throw (EOFException) cause; + checkKryoInitialized(); + + if (target != previousOut) { + DataOutputViewStream outputStream = new DataOutputViewStream(target); + output = new Output(outputStream); + previousOut = target; + } + + // Sanity check: Make sure that the output is cleared/has been flushed by the last call + // otherwise data might be written multiple times in case of a previous EOFException + if (output.position() != 0) { + throw new IllegalStateException("The Kryo Output still contains data from a previous " + + "serialize call. It has to be flushed or cleared at the end of the serialize call."); + } + + try { + kryo.writeClassAndObject(output, record); + output.flush(); + } + catch (KryoException ke) { + // make sure that the Kryo output buffer is cleared in case that we can recover from + // the exception (e.g. EOFException which denotes buffer full) + output.clear(); + + Throwable cause = ke.getCause(); + if (cause instanceof EOFException) { + throw (EOFException) cause; + } + else { + throw ke; + } } - else { - throw ke; + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); } } } @@ -242,26 +275,38 @@ public void serialize(T record, DataOutputView target) throws IOException { @SuppressWarnings("unchecked") @Override public T deserialize(DataInputView source) throws IOException { - checkKryoInitialized(); - if (source != previousIn) { - DataInputViewStream inputStream = new DataInputViewStream(source); - input = new NoFetchingInput(inputStream); - previousIn = source; + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); } try { - return (T) kryo.readClassAndObject(input); - } catch (KryoException ke) { - Throwable cause = ke.getCause(); - - if (cause instanceof EOFException) { - throw (EOFException) cause; - } else { - throw ke; + checkKryoInitialized(); + + if (source != previousIn) { + DataInputViewStream inputStream = new DataInputViewStream(source); + input = new NoFetchingInput(inputStream); + previousIn = source; + } + + try { + return (T) kryo.readClassAndObject(input); + } catch (KryoException ke) { + Throwable cause = ke.getCause(); + + if (cause instanceof EOFException) { + throw (EOFException) cause; + } else { + throw ke; + } + } + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); } } } - + @Override public T deserialize(T reuse, DataInputView source) throws IOException { return deserialize(source); @@ -269,13 +314,24 @@ public T deserialize(T reuse, DataInputView source) throws IOException { @Override public void copy(DataInputView source, DataOutputView target) throws IOException { - checkKryoInitialized(); - if(this.copyInstance == null){ - this.copyInstance = createInstance(); + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); } - T tmp = deserialize(copyInstance, source); - serialize(tmp, target); + try { + checkKryoInitialized(); + if (this.copyInstance == null){ + this.copyInstance = createInstance(); + } + + T tmp = deserialize(copyInstance, source); + serialize(tmp, target); + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); + } + } } // -------------------------------------------------------------------------------------------- @@ -516,6 +572,27 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE // For testing // -------------------------------------------------------------------------------------------- + private void enterExclusiveThread() { + // we use simple get, check, set here, rather than CAS + // we don't need lock-style correctness, this is only a sanity-check and we thus + // favor speed at the cost of some false negatives in this check + Thread previous = currentThread; + Thread thisThread = Thread.currentThread(); + + if (previous == null) { + currentThread = thisThread; + } + else if (previous != thisThread) { + throw new IllegalStateException( + "Concurrent access to KryoSerializer. Thread 1: " + thisThread.getName() + + " , Thread 2: " + previous.getName()); + } + } + + private void exitExclusiveThread() { + currentThread = null; + } + @VisibleForTesting public Kryo getKryo() { checkKryoInitialized(); diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerDebugInitHelper.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerDebugInitHelper.java new file mode 100644 index 00000000000000..ac918d645f84a3 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerDebugInitHelper.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.java.typeutils.runtime.kryo; + +import org.apache.flink.annotation.Internal; + +/** + * Simple helper class to initialize the concurrency checks for tests. + * + *

    The flag is automatically set to true when assertions are activated (tests) + * and can be set to true manually in other tests as well; + */ +@Internal +class KryoSerializerDebugInitHelper { + + /** This captures the initial setting after initialization. It is used to + * validate in tests that we never change the default to true. */ + static final boolean INITIAL_SETTING; + + /** The flag that is used to initialize the KryoSerializer's concurrency check flag. */ + static boolean setToDebug = false; + + static { + // capture the default setting, for tests + INITIAL_SETTING = setToDebug; + + // if assertions are active, the check should be activated + //noinspection AssertWithSideEffects,ConstantConditions + assert setToDebug = true; + } +} diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyCheckInactiveITCase.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyCheckInactiveITCase.java new file mode 100644 index 00000000000000..522bf9e9b303b3 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyCheckInactiveITCase.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.java.typeutils.runtime.kryo; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * A test that validates that the concurrency checks in the Kryo Serializer + * are not hard coded to active. + * + *

    The debug initialization in the KryoSerializer happens together with class + * initialization (that makes it peak efficient), which is why this test needs to + * run in a fresh JVM fork, and the JVM fork of this test should not be reused. + * + *

    Important: If you see this test fail and the initial settings are still + * correct, check the assumptions above (on fresh JVM fork). + */ +public class KryoSerializerConcurrencyCheckInactiveITCase { + + // this sets the debug initialization back to its default, even if + // by default tests modify it (implicitly via assertion loading) + static { + KryoSerializerDebugInitHelper.setToDebug = KryoSerializerDebugInitHelper.INITIAL_SETTING; + } + + /** + * This test checks that concurrent access is not detected by default, meaning that + * the thread concurrency checks are off by default. + */ + @Test + public void testWithNoConcurrencyCheck() throws Exception { + boolean assertionError; + try { + new KryoSerializerConcurrencyTest().testConcurrentUseOfSerializer(); + assertionError = false; + } + catch (AssertionError e) { + assertionError = true; + } + + assertTrue("testConcurrentUseOfSerializer() should have failed if " + + "concurrency checks are off by default", assertionError); + } +} diff --git a/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyTest.java b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyTest.java new file mode 100644 index 00000000000000..ca81fd4e6fa719 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializerConcurrencyTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.java.typeutils.runtime.kryo; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.core.testutils.BlockerSync; +import org.apache.flink.core.testutils.CheckedThread; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +/** + * This tests that the {@link KryoSerializer} properly fails when accessed by two threads + * concurrently. + * + *

    Important: This test only works if assertions are activated (-ea) on the JVM + * when running tests. + */ +public class KryoSerializerConcurrencyTest { + + @Test + public void testConcurrentUseOfSerializer() throws Exception { + final KryoSerializer serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); + + final BlockerSync sync = new BlockerSync(); + + final DataOutputView regularOut = new DataOutputSerializer(32); + final DataOutputView lockingOut = new LockingView(sync); + + // this thread serializes and gets stuck there + final CheckedThread thread = new CheckedThread("serializer") { + @Override + public void go() throws Exception { + serializer.serialize("a value", lockingOut); + } + }; + + thread.start(); + sync.awaitBlocker(); + + // this should fail with an exception + try { + serializer.serialize("value", regularOut); + fail("should have failed with an exception"); + } + catch (IllegalStateException e) { + // expected + } + finally { + // release the thread that serializes + sync.releaseBlocker(); + } + + // this propagates exceptions from the spawned thread + thread.sync(); + } + + // ------------------------------------------------------------------------ + + private static class LockingView extends DataOutputSerializer { + + private final BlockerSync blocker; + + LockingView(BlockerSync blocker) { + super(32); + this.blocker = blocker; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + blocker.blockNonInterruptible(); + } + } +} From be7c89596a3b9cd8805a90aaf32336ec2759a1f7 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 6 Mar 2018 11:21:08 +0100 Subject: [PATCH 0125/2294] [FLINK-8879] [avro] Add concurrency check Avro Serializer on DEBUG level. --- .../avro/typeutils/AvroSerializer.java | 114 ++++++++++++++++-- .../AvroSerializerDebugInitHelper.java | 47 ++++++++ ...ializerConcurrencyCheckInactiveITCase.java | 62 ++++++++++ .../AvroSerializerConcurrencyTest.java | 94 +++++++++++++++ .../AvroSerializerSerializabilityTest.java | 70 +++++++++++ .../flink-1.4-serializer-java-serialized | Bin 0 -> 202 bytes 6 files changed, 374 insertions(+), 13 deletions(-) create mode 100644 flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializerDebugInitHelper.java create mode 100644 flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyCheckInactiveITCase.java create mode 100644 flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyTest.java create mode 100644 flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerSerializabilityTest.java create mode 100644 flink-formats/flink-avro/src/test/resources/flink-1.4-serializer-java-serialized diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java index bc3369fcf593a7..75f298893e0e38 100644 --- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializer.java @@ -39,6 +39,8 @@ import org.apache.avro.specific.SpecificDatumReader; import org.apache.avro.specific.SpecificDatumWriter; import org.apache.avro.specific.SpecificRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; @@ -52,12 +54,24 @@ * (ReflectDatumReader / -Writer). The serializer instantiates them depending on * the class of the type it should serialize. * + *

    Important: This serializer is NOT THREAD SAFE, because it reuses the data encoders + * and decoders which have buffers that would be shared between the threads if used concurrently + * * @param The type to be serialized. */ public class AvroSerializer extends TypeSerializer { private static final long serialVersionUID = 1L; + /** Logger instance. */ + private static final Logger LOG = LoggerFactory.getLogger(AvroSerializer.class); + + /** Flag whether to check for concurrent thread access. + * Because this flag is static final, a value of 'false' allows the JIT compiler to eliminate + * the guarded code sections. */ + private static final boolean CONCURRENT_ACCESS_CHECK = + LOG.isDebugEnabled() || AvroSerializerDebugInitHelper.setToDebug; + // -------- configuration fields, serializable ----------- /** The class of the type that is serialized by this serializer. */ @@ -78,6 +92,9 @@ public class AvroSerializer extends TypeSerializer { /** The serializer configuration snapshot, cached for efficiency. */ private transient AvroSchemaSerializerConfigSnapshot configSnapshot; + /** The currently accessing thread, set and checked on debug level only. */ + private transient volatile Thread currentThread; + // ------------------------------------------------------------------------ /** @@ -127,23 +144,56 @@ public T createInstance() { @Override public void serialize(T value, DataOutputView target) throws IOException { - checkAvroInitialized(); - this.encoder.setOut(target); - this.writer.write(value, this.encoder); + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); + } + + try { + checkAvroInitialized(); + this.encoder.setOut(target); + this.writer.write(value, this.encoder); + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); + } + } } @Override public T deserialize(DataInputView source) throws IOException { - checkAvroInitialized(); - this.decoder.setIn(source); - return this.reader.read(null, this.decoder); + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); + } + + try { + checkAvroInitialized(); + this.decoder.setIn(source); + return this.reader.read(null, this.decoder); + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); + } + } } @Override public T deserialize(T reuse, DataInputView source) throws IOException { - checkAvroInitialized(); - this.decoder.setIn(source); - return this.reader.read(reuse, this.decoder); + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); + } + + try { + checkAvroInitialized(); + this.decoder.setIn(source); + return this.reader.read(reuse, this.decoder); + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); + } + } } // ------------------------------------------------------------------------ @@ -152,8 +202,19 @@ public T deserialize(T reuse, DataInputView source) throws IOException { @Override public T copy(T from) { - checkAvroInitialized(); - return avroData.deepCopy(schema, from); + if (CONCURRENT_ACCESS_CHECK) { + enterExclusiveThread(); + } + + try { + checkAvroInitialized(); + return avroData.deepCopy(schema, from); + } + finally { + if (CONCURRENT_ACCESS_CHECK) { + exitExclusiveThread(); + } + } } @Override @@ -163,8 +224,10 @@ public T copy(T from, T reuse) { @Override public void copy(DataInputView source, DataOutputView target) throws IOException { - T value = deserialize(source); - serialize(value, target); + // we do not have concurrency checks here, because serialize() and + // deserialize() do the checks and the current concurrency check mechanism + // does provide additional safety in cases of re-entrant calls + serialize(deserialize(source), target); } // ------------------------------------------------------------------------ @@ -277,6 +340,31 @@ private void initializeAvro() { this.decoder = new DataInputDecoder(); } + // -------------------------------------------------------------------------------------------- + // Concurrency checks + // -------------------------------------------------------------------------------------------- + + private void enterExclusiveThread() { + // we use simple get, check, set here, rather than CAS + // we don't need lock-style correctness, this is only a sanity-check and we thus + // favor speed at the cost of some false negatives in this check + Thread previous = currentThread; + Thread thisThread = Thread.currentThread(); + + if (previous == null) { + currentThread = thisThread; + } + else if (previous != thisThread) { + throw new IllegalStateException( + "Concurrent access to KryoSerializer. Thread 1: " + thisThread.getName() + + " , Thread 2: " + previous.getName()); + } + } + + private void exitExclusiveThread() { + currentThread = null; + } + // ------------------------------------------------------------------------ // Serializer Snapshots // ------------------------------------------------------------------------ diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializerDebugInitHelper.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializerDebugInitHelper.java new file mode 100644 index 00000000000000..c65709278e61b1 --- /dev/null +++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/typeutils/AvroSerializerDebugInitHelper.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.typeutils; + +import org.apache.flink.annotation.Internal; + +/** + * Simple helper class to initialize the concurrency checks for tests. + * + *

    The flag is automatically set to true when assertions are activated (tests) + * and can be set to true manually in other tests as well; + */ +@Internal +class AvroSerializerDebugInitHelper { + + /** This captures the initial setting after initialization. It is used to + * validate in tests that we never change the default to true. */ + static final boolean INITIAL_SETTING; + + /** The flag that is used to initialize the KryoSerializer's concurrency check flag. */ + static boolean setToDebug = false; + + static { + // capture the default setting, for tests + INITIAL_SETTING = setToDebug; + + // if assertions are active, the check should be activated + //noinspection AssertWithSideEffects,ConstantConditions + assert setToDebug = true; + } +} diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyCheckInactiveITCase.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyCheckInactiveITCase.java new file mode 100644 index 00000000000000..9b98e44447ccfa --- /dev/null +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyCheckInactiveITCase.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.typeutils; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * A test that validates that the concurrency checks in the Avro Serializer + * are not hard coded to active. + * + *

    The debug initialization in the AvroSerializer happens together with class + * initialization (that makes it peak efficient), which is why this test needs to + * run in a fresh JVM fork, and the JVM fork of this test should not be reused. + * + *

    Important: If you see this test fail and the initial settings are still + * correct, check the assumptions above (on fresh JVM fork). + */ +public class AvroSerializerConcurrencyCheckInactiveITCase { + + // this sets the debug initialization back to its default, even if + // by default tests modify it (implicitly via assertion loading) + static { + AvroSerializerDebugInitHelper.setToDebug = AvroSerializerDebugInitHelper.INITIAL_SETTING; + } + + /** + * This test checks that concurrent access is not detected by default, meaning that + * the thread concurrency checks are off by default. + */ + @Test + public void testWithNoConcurrencyCheck() throws Exception { + boolean assertionError; + try { + new AvroSerializerConcurrencyTest().testConcurrentUseOfSerializer(); + assertionError = false; + } + catch (AssertionError e) { + assertionError = true; + } + + assertTrue("testConcurrentUseOfSerializer() should have failed if " + + "concurrency checks are off by default", assertionError); + } +} diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyTest.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyTest.java new file mode 100644 index 00000000000000..aaa9b4b08b70cc --- /dev/null +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerConcurrencyTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.typeutils; + +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.core.testutils.BlockerSync; +import org.apache.flink.core.testutils.CheckedThread; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.fail; + +/** + * This tests that the {@link AvroSerializer} properly fails when accessed by two threads + * concurrently. + * + *

    Important: This test only works if assertions are activated (-ea) on the JVM + * when running tests. + */ +public class AvroSerializerConcurrencyTest { + + @Test + public void testConcurrentUseOfSerializer() throws Exception { + final AvroSerializer serializer = new AvroSerializer<>(String.class); + + final BlockerSync sync = new BlockerSync(); + + final DataOutputView regularOut = new DataOutputSerializer(32); + final DataOutputView lockingOut = new LockingView(sync); + + // this thread serializes and gets stuck there + final CheckedThread thread = new CheckedThread("serializer") { + @Override + public void go() throws Exception { + serializer.serialize("a value", lockingOut); + } + }; + + thread.start(); + sync.awaitBlocker(); + + // this should fail with an exception + try { + serializer.serialize("value", regularOut); + fail("should have failed with an exception"); + } + catch (IllegalStateException e) { + // expected + } + finally { + // release the thread that serializes + sync.releaseBlocker(); + } + + // this propagates exceptions from the spawned thread + thread.sync(); + } + + // ------------------------------------------------------------------------ + + private static class LockingView extends DataOutputSerializer { + + private final BlockerSync blocker; + + LockingView(BlockerSync blocker) { + super(32); + this.blocker = blocker; + } + + @Override + public void writeInt(int v) throws IOException { + blocker.blockNonInterruptible(); + } + } +} diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerSerializabilityTest.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerSerializabilityTest.java new file mode 100644 index 00000000000000..c15aa7c01414b9 --- /dev/null +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/typeutils/AvroSerializerSerializabilityTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.typeutils; + +import org.junit.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +import static org.junit.Assert.assertEquals; + +/** + * Test that validates that the serialized form of the AvroSerializer is the same as in + * previous Flink versions. + * + *

    While that is not strictly necessary for FLink to work, it increases user experience + * in job upgrade situations. + */ +public class AvroSerializerSerializabilityTest { + + private static final String RESOURCE_NAME = "flink-1.4-serializer-java-serialized"; + + @Test + public void testDeserializeSerializer() throws Exception { + final AvroSerializer currentSerializer = new AvroSerializer<>(String.class); + + try (ObjectInputStream in = new ObjectInputStream( + getClass().getClassLoader().getResourceAsStream(RESOURCE_NAME))) { + + @SuppressWarnings("unchecked") + AvroSerializer deserialized = (AvroSerializer) in.readObject(); + + assertEquals(currentSerializer, deserialized); + } + } + + // ------------------------------------------------------------------------ + // To create a serialized serializer file + // ------------------------------------------------------------------------ + + public static void main(String[] args) throws Exception { + final AvroSerializer serializer = new AvroSerializer<>(String.class); + + final File file = new File("flink-formats/flink-avro/src/test/resources/" + RESOURCE_NAME).getAbsoluteFile(); + + try (FileOutputStream fos = new FileOutputStream(file); + ObjectOutputStream out = new ObjectOutputStream(fos)) { + + out.writeObject(serializer); + } + } +} diff --git a/flink-formats/flink-avro/src/test/resources/flink-1.4-serializer-java-serialized b/flink-formats/flink-avro/src/test/resources/flink-1.4-serializer-java-serialized new file mode 100644 index 0000000000000000000000000000000000000000..63fef0a3dabfc6a49a1441de4e6c72352d981073 GIT binary patch literal 202 zcmZw9F$%&k6vpw_qPt$?brBIdY1eM`0*{i$XqtqV7-~=B;OOSL1dm{Yi=f~3!Jps# z33Jb|wk(kxwFiNnR_27A<(`5^ifl1loHz%iJ#MGl2v(%kC>TD=a>#0!r7FO(K2oGY xlaWQG$@_9(SpE3q6t=eSt@+aTlkiW6!9|8ef-%9^1Xe~~U$@OzK6W`w!5i#!Li+#! literal 0 HcmV?d00001 From 00773492ad22aba7eb4c2194dfa65bdd1a94f887 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Thu, 8 Mar 2018 11:22:19 +0100 Subject: [PATCH 0126/2294] [hotfix] [build] Change REST port to 8081 for end-to-end testing scripts Now that the FLIP-6 code uses 8081, we need to probe that port to check Flink's status in the end-to-end tests. --- flink-end-to-end-tests/test-scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index e6d21a26b0907d..ef4856f561b542 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -45,7 +45,7 @@ function start_cluster { # wait at most 10 seconds until the dispatcher is up for i in {1..10}; do # without the || true this would exit our script if the JobManager is not yet up - QUERY_RESULT=$(curl "http://localhost:9065/taskmanagers" 2> /dev/null || true) + QUERY_RESULT=$(curl "http://localhost:8081/taskmanagers" 2> /dev/null || true) if [[ "$QUERY_RESULT" == "" ]]; then echo "Dispatcher/TaskManagers are not yet up" From 236ceb58ff5f189389d60b8c66c243acf0136047 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 5 Mar 2018 13:46:41 +0100 Subject: [PATCH 0127/2294] [FLINK-8839] [sql-client] Fix table source factory discovery This closes #5640. --- flink-libraries/flink-sql-client/pom.xml | 23 +++ .../flink/table/client/gateway/Executor.java | 18 +- .../table/client/gateway/SessionContext.java | 20 ++ .../gateway/local/ExecutionContext.java | 166 +++++++++++++++ .../client/gateway/local/LocalExecutor.java | 189 ++++++------------ .../assembly/test-table-source-factory.xml | 47 +++++ .../client/gateway/local/DependencyTest.java | 72 +++++++ .../gateway/local/LocalExecutorITCase.java | 59 +++--- .../gateway/utils/EnvironmentFileUtil.java | 56 ++++++ .../gateway/utils/TestTableSourceFactory.java | 112 +++++++++++ .../test/resources/test-factory-services-file | 20 ++ .../resources/test-sql-client-defaults.yaml | 1 + .../resources/test-sql-client-factory.yaml | 45 +++++ .../flink/table/descriptors/Rowtime.scala | 2 +- .../sources/TableSourceFactoryService.scala | 27 ++- .../flink/table/descriptors/RowtimeTest.scala | 2 +- 16 files changed, 683 insertions(+), 176 deletions(-) create mode 100644 flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java create mode 100644 flink-libraries/flink-sql-client/src/test/assembly/test-table-source-factory.xml create mode 100644 flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java create mode 100644 flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/EnvironmentFileUtil.java create mode 100644 flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java create mode 100644 flink-libraries/flink-sql-client/src/test/resources/test-factory-services-file create mode 100644 flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml diff --git a/flink-libraries/flink-sql-client/pom.xml b/flink-libraries/flink-sql-client/pom.xml index 03fca24c917c43..300f6ceadf213f 100644 --- a/flink-libraries/flink-sql-client/pom.xml +++ b/flink-libraries/flink-sql-client/pom.xml @@ -133,6 +133,7 @@ under the License. + org.apache.maven.plugins maven-shade-plugin @@ -167,6 +168,28 @@ under the License. + + + + maven-assembly-plugin + 2.4 + + + create-table-source-factory-jar + process-test-classes + + single + + + table-source-factory + false + + src/test/assembly/test-table-source-factory.xml + + + + + diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java index 512c1943937138..4a41222700e26a 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java @@ -38,38 +38,38 @@ public interface Executor { /** * Lists all session properties that are defined by the executor and the session. */ - Map getSessionProperties(SessionContext context) throws SqlExecutionException; + Map getSessionProperties(SessionContext session) throws SqlExecutionException; /** * Lists all tables known to the executor. */ - List listTables(SessionContext context) throws SqlExecutionException; + List listTables(SessionContext session) throws SqlExecutionException; /** * Returns the schema of a table. Throws an exception if the table could not be found. */ - TableSchema getTableSchema(SessionContext context, String name) throws SqlExecutionException; + TableSchema getTableSchema(SessionContext session, String name) throws SqlExecutionException; /** * Returns a string-based explanation about AST and execution plan of the given statement. */ - String explainStatement(SessionContext context, String statement) throws SqlExecutionException; + String explainStatement(SessionContext session, String statement) throws SqlExecutionException; /** * Submits a Flink job (detached) and returns the result descriptor. */ - ResultDescriptor executeQuery(SessionContext context, String query) throws SqlExecutionException; + ResultDescriptor executeQuery(SessionContext session, String query) throws SqlExecutionException; /** * Asks for the next changelog results (non-blocking). */ - TypedResult>> retrieveResultChanges(SessionContext context, String resultId) throws SqlExecutionException; + TypedResult>> retrieveResultChanges(SessionContext session, String resultId) throws SqlExecutionException; /** * Creates an immutable result snapshot of the running Flink job. Throws an exception if no Flink job can be found. * Returns the number of pages. */ - TypedResult snapshotResult(SessionContext context, String resultId, int pageSize) throws SqlExecutionException; + TypedResult snapshotResult(SessionContext session, String resultId, int pageSize) throws SqlExecutionException; /** * Returns the rows that are part of the current page or throws an exception if the snapshot has been expired. @@ -79,10 +79,10 @@ public interface Executor { /** * Cancels a table program and stops the result retrieval. */ - void cancelQuery(SessionContext context, String resultId) throws SqlExecutionException; + void cancelQuery(SessionContext session, String resultId) throws SqlExecutionException; /** * Stops the executor. */ - void stop(SessionContext context); + void stop(SessionContext session); } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/SessionContext.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/SessionContext.java index 1058eb6a3f60f3..0b6ee2c8ef5626 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/SessionContext.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/SessionContext.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Objects; /** * Context describing a session. @@ -54,4 +55,23 @@ public Environment getEnvironment() { // enrich with session properties return Environment.enrich(defaultEnvironment, sessionProperties); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SessionContext)) { + return false; + } + SessionContext context = (SessionContext) o; + return Objects.equals(name, context.name) && + Objects.equals(defaultEnvironment, context.defaultEnvironment) && + Objects.equals(sessionProperties, context.sessionProperties); + } + + @Override + public int hashCode() { + return Objects.hash(name, defaultEnvironment, sessionProperties); + } } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java new file mode 100644 index 00000000000000..15a3c129bb8018 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.client.gateway.local; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.Plan; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.optimizer.DataStatistics; +import org.apache.flink.optimizer.Optimizer; +import org.apache.flink.optimizer.costs.DefaultCostEstimator; +import org.apache.flink.optimizer.plan.FlinkPlan; +import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.graph.StreamGraph; +import org.apache.flink.table.api.BatchQueryConfig; +import org.apache.flink.table.api.QueryConfig; +import org.apache.flink.table.api.StreamQueryConfig; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.client.config.Environment; +import org.apache.flink.table.client.gateway.SessionContext; +import org.apache.flink.table.sources.TableSource; +import org.apache.flink.table.sources.TableSourceFactoryService; + +import java.net.URL; +import java.util.List; + +/** + * Context for executing table programs. It contains configured environments and environment + * specific logic such as plan translation. + */ +public class ExecutionContext { + + private final SessionContext sessionContext; + private final Environment mergedEnv; + private final ExecutionEnvironment execEnv; + private final StreamExecutionEnvironment streamExecEnv; + private final TableEnvironment tableEnv; + private final ClassLoader classLoader; + private final QueryConfig queryConfig; + + public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List dependencies) { + this.sessionContext = sessionContext; + this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment()); + + // create environments + if (mergedEnv.getExecution().isStreamingExecution()) { + streamExecEnv = createStreamExecutionEnvironment(); + execEnv = null; + tableEnv = TableEnvironment.getTableEnvironment(streamExecEnv); + } else { + streamExecEnv = null; + execEnv = createExecutionEnvironment(); + tableEnv = TableEnvironment.getTableEnvironment(execEnv); + } + + // create class loader + classLoader = FlinkUserCodeClassLoaders.parentFirst( + dependencies.toArray(new URL[dependencies.size()]), + this.getClass().getClassLoader()); + + // create table sources + mergedEnv.getSources().forEach((name, source) -> { + TableSource tableSource = TableSourceFactoryService.findAndCreateTableSource(source, classLoader); + tableEnv.registerTableSource(name, tableSource); + }); + + // create query config + queryConfig = createQueryConfig(); + } + + public SessionContext getSessionContext() { + return sessionContext; + } + + public ExecutionEnvironment getExecutionEnvironment() { + return execEnv; + } + + public StreamExecutionEnvironment getStreamExecutionEnvironment() { + return streamExecEnv; + } + + public TableEnvironment getTableEnvironment() { + return tableEnv; + } + + public ClassLoader getClassLoader() { + return classLoader; + } + + public Environment getMergedEnvironment() { + return mergedEnv; + } + + public QueryConfig getQueryConfig() { + return queryConfig; + } + + public ExecutionConfig getExecutionConfig() { + if (streamExecEnv != null) { + return streamExecEnv.getConfig(); + } else { + return execEnv.getConfig(); + } + } + + public FlinkPlan createPlan(String name, Configuration flinkConfig) { + if (streamExecEnv != null) { + final StreamGraph graph = streamExecEnv.getStreamGraph(); + graph.setJobName(name); + return graph; + } else { + final int parallelism = execEnv.getParallelism(); + final Plan unoptimizedPlan = execEnv.createProgramPlan(); + unoptimizedPlan.setJobName(name); + final Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); + return ClusterClient.getOptimizedPlan(compiler, unoptimizedPlan, parallelism); + } + } + + // -------------------------------------------------------------------------------------------- + + private ExecutionEnvironment createExecutionEnvironment() { + final ExecutionEnvironment execEnv = ExecutionEnvironment.getExecutionEnvironment(); + execEnv.setParallelism(mergedEnv.getExecution().getParallelism()); + return execEnv; + } + + private StreamExecutionEnvironment createStreamExecutionEnvironment() { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(mergedEnv.getExecution().getParallelism()); + env.setMaxParallelism(mergedEnv.getExecution().getMaxParallelism()); + return env; + } + + private QueryConfig createQueryConfig() { + if (streamExecEnv != null) { + final StreamQueryConfig config = new StreamQueryConfig(); + final long minRetention = mergedEnv.getExecution().getMinStateRetention(); + final long maxRetention = mergedEnv.getExecution().getMaxStateRetention(); + config.withIdleStateRetentionTime(Time.milliseconds(minRetention), Time.milliseconds(maxRetention)); + return config; + } else { + return new BatchQueryConfig(); + } + } +} diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java index 8c40885d36a72f..35d7da9bbfd285 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java @@ -18,11 +18,7 @@ package org.apache.flink.table.client.gateway.local; -import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.Plan; -import org.apache.flink.api.common.time.Time; -import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.client.cli.CliFrontend; import org.apache.flink.client.deployment.ClusterDescriptor; @@ -36,34 +32,21 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.core.fs.Path; -import org.apache.flink.optimizer.DataStatistics; -import org.apache.flink.optimizer.Optimizer; -import org.apache.flink.optimizer.costs.DefaultCostEstimator; import org.apache.flink.optimizer.plan.FlinkPlan; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.graph.StreamGraph; -import org.apache.flink.table.api.BatchQueryConfig; -import org.apache.flink.table.api.QueryConfig; -import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.api.java.BatchTableEnvironment; -import org.apache.flink.table.api.java.StreamTableEnvironment; import org.apache.flink.table.client.SqlClientException; import org.apache.flink.table.client.config.Deployment; import org.apache.flink.table.client.config.Environment; -import org.apache.flink.table.client.config.Execution; import org.apache.flink.table.client.gateway.Executor; import org.apache.flink.table.client.gateway.ResultDescriptor; import org.apache.flink.table.client.gateway.SessionContext; import org.apache.flink.table.client.gateway.SqlExecutionException; import org.apache.flink.table.client.gateway.TypedResult; import org.apache.flink.table.sinks.TableSink; -import org.apache.flink.table.sources.TableSource; -import org.apache.flink.table.sources.TableSourceFactoryService; import org.apache.flink.types.Row; import org.apache.flink.util.StringUtils; @@ -86,11 +69,17 @@ public class LocalExecutor implements Executor { private static final String DEFAULT_ENV_FILE = "sql-client-defaults.yaml"; - private final Environment environment; + private final Environment defaultEnvironment; private final List dependencies; private final Configuration flinkConfig; private final ResultStore resultStore; + /** + * Cached execution context for unmodified sessions. Do not access this variable directly + * but through {@link LocalExecutor#getOrCreateExecutionContext}. + */ + private ExecutionContext executionContext; + /** * Creates a local executor for submitting table programs and retrieving results. */ @@ -129,12 +118,12 @@ public LocalExecutor(URL defaultEnv, List jars, List libraries) { if (defaultEnv != null) { System.out.println("Reading default environment from: " + defaultEnv); try { - environment = Environment.parse(defaultEnv); + defaultEnvironment = Environment.parse(defaultEnv); } catch (IOException e) { throw new SqlClientException("Could not read default environment file at: " + defaultEnv, e); } } else { - environment = new Environment(); + defaultEnvironment = new Environment(); } // discover dependencies @@ -175,8 +164,11 @@ public LocalExecutor(URL defaultEnv, List jars, List libraries) { resultStore = new ResultStore(flinkConfig); } - public LocalExecutor(Environment environment, List dependencies, Configuration flinkConfig) { - this.environment = environment; + /** + * Constructor for testing purposes. + */ + public LocalExecutor(Environment defaultEnvironment, List dependencies, Configuration flinkConfig) { + this.defaultEnvironment = defaultEnvironment; this.dependencies = dependencies; this.flinkConfig = flinkConfig; @@ -190,8 +182,8 @@ public void start() { } @Override - public Map getSessionProperties(SessionContext context) throws SqlExecutionException { - final Environment env = createEnvironment(context); + public Map getSessionProperties(SessionContext session) throws SqlExecutionException { + final Environment env = getOrCreateExecutionContext(session).getMergedEnvironment(); final Map properties = new HashMap<>(); properties.putAll(env.getExecution().toProperties()); properties.putAll(env.getDeployment().toProperties()); @@ -199,16 +191,14 @@ public Map getSessionProperties(SessionContext context) throws S } @Override - public List listTables(SessionContext context) throws SqlExecutionException { - final Environment env = createEnvironment(context); - final TableEnvironment tableEnv = createTableEnvironment(env); + public List listTables(SessionContext session) throws SqlExecutionException { + final TableEnvironment tableEnv = getOrCreateExecutionContext(session).getTableEnvironment(); return Arrays.asList(tableEnv.listTables()); } @Override - public TableSchema getTableSchema(SessionContext context, String name) throws SqlExecutionException { - final Environment env = createEnvironment(context); - final TableEnvironment tableEnv = createTableEnvironment(env); + public TableSchema getTableSchema(SessionContext session, String name) throws SqlExecutionException { + final TableEnvironment tableEnv = getOrCreateExecutionContext(session).getTableEnvironment(); try { return tableEnv.scan(name).getSchema(); } catch (Throwable t) { @@ -218,13 +208,13 @@ public TableSchema getTableSchema(SessionContext context, String name) throws Sq } @Override - public String explainStatement(SessionContext context, String statement) throws SqlExecutionException { - final Environment env = createEnvironment(context); + public String explainStatement(SessionContext session, String statement) throws SqlExecutionException { + final ExecutionContext context = getOrCreateExecutionContext(session); // translate try { - final Tuple2 table = createTable(env, statement); - return table.f0.explain(table.f1); + final Table table = createTable(context, statement); + return context.getTableEnvironment().explain(table); } catch (Throwable t) { // catch everything such that the query does not crash the executor throw new SqlExecutionException("Invalid SQL statement.", t); @@ -232,29 +222,26 @@ public String explainStatement(SessionContext context, String statement) throws } @Override - public ResultDescriptor executeQuery(SessionContext context, String query) throws SqlExecutionException { - final Environment env = createEnvironment(context); + public ResultDescriptor executeQuery(SessionContext session, String query) throws SqlExecutionException { + final ExecutionContext context = getOrCreateExecutionContext(session); + final Environment mergedEnv = context.getMergedEnvironment(); // create table here to fail quickly for wrong queries - final Tuple2 table = createTable(env, query); + final Table table = createTable(context, query); // deployment - final ClusterClient clusterClient = createDeployment(env.getDeployment()); + final ClusterClient clusterClient = createDeployment(mergedEnv.getDeployment()); // initialize result final DynamicResult result = resultStore.createResult( - env, - table.f1.getSchema(), - getExecutionConfig(table.f0)); + mergedEnv, + table.getSchema(), + context.getExecutionConfig()); // create job graph with jars final JobGraph jobGraph; try { - jobGraph = createJobGraph( - context.getName() + ": " + query, - env.getExecution(), - table.f0, - table.f1, + jobGraph = createJobGraph(context, context.getSessionContext().getName() + ": " + query, table, result.getTableSink(), clusterClient); } catch (Throwable t) { @@ -268,18 +255,12 @@ public ResultDescriptor executeQuery(SessionContext context, String query) throw final String resultId = jobGraph.getJobID().toString(); resultStore.storeResult(resultId, result); - // create class loader - final ClassLoader classLoader = JobWithJars.buildUserCodeClassLoader( - dependencies, - Collections.emptyList(), - this.getClass().getClassLoader()); - // create execution final Runnable program = () -> { // we need to submit the job attached for now // otherwise it is not possible to retrieve the reason why an execution failed try { - clusterClient.run(jobGraph, classLoader); + clusterClient.run(jobGraph, context.getClassLoader()); } catch (ProgramInvocationException e) { throw new SqlExecutionException("Could not execute table program.", e); } finally { @@ -294,11 +275,12 @@ public ResultDescriptor executeQuery(SessionContext context, String query) throw // start result retrieval result.startRetrieval(program); - return new ResultDescriptor(resultId, table.f1.getSchema(), result.isMaterialized()); + return new ResultDescriptor(resultId, table.getSchema(), result.isMaterialized()); } @Override - public TypedResult>> retrieveResultChanges(SessionContext context, String resultId) throws SqlExecutionException { + public TypedResult>> retrieveResultChanges(SessionContext session, + String resultId) throws SqlExecutionException { final DynamicResult result = resultStore.getResult(resultId); if (result == null) { throw new SqlExecutionException("Could not find a result with result identifier '" + resultId + "'."); @@ -310,7 +292,7 @@ public TypedResult>> retrieveResultChanges(SessionCont } @Override - public TypedResult snapshotResult(SessionContext context, String resultId, int pageSize) throws SqlExecutionException { + public TypedResult snapshotResult(SessionContext session, String resultId, int pageSize) throws SqlExecutionException { final DynamicResult result = resultStore.getResult(resultId); if (result == null) { throw new SqlExecutionException("Could not find a result with result identifier '" + resultId + "'."); @@ -334,7 +316,7 @@ public List retrieveResultPage(String resultId, int page) throws SqlExecuti } @Override - public void cancelQuery(SessionContext context, String resultId) throws SqlExecutionException { + public void cancelQuery(SessionContext session, String resultId) throws SqlExecutionException { final DynamicResult result = resultStore.getResult(resultId); if (result == null) { throw new SqlExecutionException("Could not find a result with result identifier '" + resultId + "'."); @@ -345,8 +327,8 @@ public void cancelQuery(SessionContext context, String resultId) throws SqlExecu resultStore.removeResult(resultId); // stop Flink job - final Environment env = createEnvironment(context); - final ClusterClient clusterClient = createDeployment(env.getDeployment()); + final Environment mergedEnv = getOrCreateExecutionContext(session).getMergedEnvironment(); + final ClusterClient clusterClient = createDeployment(mergedEnv.getDeployment()); try { clusterClient.cancel(new JobID(StringUtils.hexStringToByte(resultId))); } catch (Throwable t) { @@ -361,10 +343,10 @@ public void cancelQuery(SessionContext context, String resultId) throws SqlExecu } @Override - public void stop(SessionContext context) { + public void stop(SessionContext session) { resultStore.getResults().forEach((resultId) -> { try { - cancelQuery(context, resultId); + cancelQuery(session, resultId); } catch (Throwable t) { // ignore any throwable to keep the clean up running } @@ -373,45 +355,29 @@ public void stop(SessionContext context) { // -------------------------------------------------------------------------------------------- - private Tuple2 createTable(Environment env, String query) { - final TableEnvironment tableEnv = createTableEnvironment(env); - + private Table createTable(ExecutionContext context, String query) { // parse and validate query try { - return Tuple2.of(tableEnv, tableEnv.sqlQuery(query)); + return context.getTableEnvironment().sqlQuery(query); } catch (Throwable t) { // catch everything such that the query does not crash the executor throw new SqlExecutionException("Invalid SQL statement.", t); } } - private JobGraph createJobGraph(String name, Execution exec, TableEnvironment tableEnv, - Table table, TableSink sink, ClusterClient clusterClient) { - - final QueryConfig queryConfig = createQueryConfig(exec); + private JobGraph createJobGraph(ExecutionContext context, String name, Table table, + TableSink sink, ClusterClient clusterClient) { // translate try { - table.writeToSink(sink, queryConfig); + table.writeToSink(sink, context.getQueryConfig()); } catch (Throwable t) { // catch everything such that the query does not crash the executor throw new SqlExecutionException("Invalid SQL statement.", t); } // extract plan - final FlinkPlan plan; - if (exec.isStreamingExecution()) { - final StreamGraph graph = ((StreamTableEnvironment) tableEnv).execEnv().getStreamGraph(); - graph.setJobName(name); - plan = graph; - } else { - final int parallelism = exec.getParallelism(); - final Plan unoptimizedPlan = ((BatchTableEnvironment) tableEnv).execEnv().createProgramPlan(); - unoptimizedPlan.setJobName(name); - final Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), - clusterClient.getFlinkConfiguration()); - plan = ClusterClient.getOptimizedPlan(compiler, unoptimizedPlan, parallelism); - } + final FlinkPlan plan = context.createPlan(name, clusterClient.getFlinkConfiguration()); // create job graph return clusterClient.getJobGraph( @@ -421,15 +387,6 @@ private JobGraph createJobGraph(String name, Execution exec, TableEnvironment ta SavepointRestoreSettings.none()); } - @SuppressWarnings("unchecked") - private ExecutionConfig getExecutionConfig(TableEnvironment tableEnv) { - if (tableEnv instanceof StreamTableEnvironment) { - return ((StreamTableEnvironment) tableEnv).execEnv().getConfig(); - } else { - return ((BatchTableEnvironment) tableEnv).execEnv().getConfig(); - } - } - private ClusterClient createDeployment(Deployment deploy) { // change some configuration options for being more responsive @@ -457,44 +414,18 @@ private ClusterClient createStandaloneClusterClient(Configuration configurati } } - private Environment createEnvironment(SessionContext context) { - return Environment.merge(environment, context.getEnvironment()); - } - - private TableEnvironment createTableEnvironment(Environment env) { - try { - final TableEnvironment tableEnv; - if (env.getExecution().isStreamingExecution()) { - final StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); - execEnv.setParallelism(env.getExecution().getParallelism()); - execEnv.setMaxParallelism(env.getExecution().getMaxParallelism()); - tableEnv = StreamTableEnvironment.getTableEnvironment(execEnv); - } else { - final ExecutionEnvironment execEnv = ExecutionEnvironment.getExecutionEnvironment(); - execEnv.setParallelism(env.getExecution().getParallelism()); - tableEnv = BatchTableEnvironment.getTableEnvironment(execEnv); + /** + * Creates or reuses the execution context. + */ + private synchronized ExecutionContext getOrCreateExecutionContext(SessionContext session) throws SqlExecutionException { + if (executionContext == null || !executionContext.getSessionContext().equals(session)) { + try { + executionContext = new ExecutionContext(defaultEnvironment, session, dependencies); + } catch (Throwable t) { + // catch everything such that a configuration does not crash the executor + throw new SqlExecutionException("Could not create execution context.", t); } - - env.getSources().forEach((name, source) -> { - TableSource tableSource = TableSourceFactoryService.findAndCreateTableSource(source); - tableEnv.registerTableSource(name, tableSource); - }); - - return tableEnv; - } catch (Exception e) { - throw new SqlExecutionException("Could not create table environment.", e); - } - } - - private QueryConfig createQueryConfig(Execution exec) { - if (exec.isStreamingExecution()) { - final StreamQueryConfig config = new StreamQueryConfig(); - final long minRetention = exec.getMinStateRetention(); - final long maxRetention = exec.getMaxStateRetention(); - config.withIdleStateRetentionTime(Time.milliseconds(minRetention), Time.milliseconds(maxRetention)); - return config; - } else { - return new BatchQueryConfig(); } + return executionContext; } } diff --git a/flink-libraries/flink-sql-client/src/test/assembly/test-table-source-factory.xml b/flink-libraries/flink-sql-client/src/test/assembly/test-table-source-factory.xml new file mode 100644 index 00000000000000..fb9673c593ed34 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/assembly/test-table-source-factory.xml @@ -0,0 +1,47 @@ + + + + test-jar + + jar + + false + + + ${project.build.testOutputDirectory} + / + + + org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.class + org/apache/flink/table/client/gateway/utils/TestTableSourceFactory$*.class + + + + + + + src/test/resources/test-factory-services-file + META-INF/services + org.apache.flink.table.sources.TableSourceFactory + 0755 + + + diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java new file mode 100644 index 00000000000000..715d2db5c39c2a --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.client.gateway.local; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.Types; +import org.apache.flink.table.client.config.Environment; +import org.apache.flink.table.client.gateway.SessionContext; +import org.apache.flink.table.client.gateway.utils.EnvironmentFileUtil; + +import org.junit.Test; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * Dependency tests for {@link LocalExecutor}. Mainly for testing classloading of dependencies. + */ +public class DependencyTest { + + private static final String FACTORY_ENVIRONMENT_FILE = "test-sql-client-factory.yaml"; + private static final String TABLE_SOURCE_FACTORY_JAR_FILE = "table-source-factory-test-jar.jar"; + + @Test + public void testTableSourceFactoryDiscovery() throws Exception { + // create environment + final Map replaceVars = new HashMap<>(); + replaceVars.put("$VAR_0", "test-table-source-factory"); + replaceVars.put("$VAR_1", "test-property"); + replaceVars.put("$VAR_2", "test-value"); + final Environment env = EnvironmentFileUtil.parseModified(FACTORY_ENVIRONMENT_FILE, replaceVars); + + // create executor with dependencies + final URL dependency = Paths.get("target", TABLE_SOURCE_FACTORY_JAR_FILE).toUri().toURL(); + final LocalExecutor executor = new LocalExecutor( + env, + Collections.singletonList(dependency), + new Configuration()); + + final SessionContext session = new SessionContext("test-session", new Environment()); + + final TableSchema result = executor.getTableSchema(session, "TableNumber1"); + final TableSchema expected = TableSchema.builder() + .field("IntegerField1", Types.INT()) + .field("StringField1", Types.STRING()) + .build(); + + assertEquals(expected, result); + } +} diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java index f30cafe890ee9e..a2ae28108bfb96 100644 --- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java @@ -31,16 +31,15 @@ import org.apache.flink.table.client.gateway.ResultDescriptor; import org.apache.flink.table.client.gateway.SessionContext; import org.apache.flink.table.client.gateway.TypedResult; +import org.apache.flink.table.client.gateway.utils.EnvironmentFileUtil; import org.apache.flink.test.util.TestBaseUtils; import org.apache.flink.types.Row; -import org.apache.flink.util.FileUtils; import org.apache.flink.util.TestLogger; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -61,6 +60,8 @@ */ public class LocalExecutorITCase extends TestLogger { + private static final String DEFAULTS_ENVIRONMENT_FILE = "test-sql-client-defaults.yaml"; + private static StandaloneMiniCluster cluster; @BeforeClass @@ -80,9 +81,9 @@ public static void after() throws Exception { @Test public void testListTables() throws Exception { final Executor executor = createDefaultExecutor(); - final SessionContext context = new SessionContext("test-session", new Environment()); + final SessionContext session = new SessionContext("test-session", new Environment()); - final List actualTables = executor.listTables(context); + final List actualTables = executor.listTables(session); final List expectedTables = Arrays.asList("TableNumber1", "TableNumber2"); assertEquals(expectedTables, actualTables); @@ -91,12 +92,12 @@ public void testListTables() throws Exception { @Test public void testGetSessionProperties() throws Exception { final Executor executor = createDefaultExecutor(); - final SessionContext context = new SessionContext("test-session", new Environment()); + final SessionContext session = new SessionContext("test-session", new Environment()); // modify defaults - context.setSessionProperty("execution.result-mode", "table"); + session.setSessionProperty("execution.result-mode", "table"); - final Map actualProperties = executor.getSessionProperties(context); + final Map actualProperties = executor.getSessionProperties(session); final Map expectedProperties = new HashMap<>(); expectedProperties.put("execution.type", "streaming"); @@ -114,9 +115,9 @@ public void testGetSessionProperties() throws Exception { @Test public void testTableSchema() throws Exception { final Executor executor = createDefaultExecutor(); - final SessionContext context = new SessionContext("test-session", new Environment()); + final SessionContext session = new SessionContext("test-session", new Environment()); - final TableSchema actualTableSchema = executor.getTableSchema(context, "TableNumber2"); + final TableSchema actualTableSchema = executor.getTableSchema(session, "TableNumber2"); final TableSchema expectedTableSchema = new TableSchema( new String[] {"IntegerField2", "StringField2"}, @@ -135,11 +136,11 @@ public void testQueryExecutionChangelog() throws Exception { replaceVars.put("$VAR_2", "changelog"); final Executor executor = createModifiedExecutor(replaceVars); - final SessionContext context = new SessionContext("test-session", new Environment()); + final SessionContext session = new SessionContext("test-session", new Environment()); try { // start job and retrieval - final ResultDescriptor desc = executor.executeQuery(context, "SELECT * FROM TableNumber1"); + final ResultDescriptor desc = executor.executeQuery(session, "SELECT * FROM TableNumber1"); assertFalse(desc.isMaterialized()); @@ -148,7 +149,7 @@ public void testQueryExecutionChangelog() throws Exception { while (true) { Thread.sleep(50); // slow the processing down final TypedResult>> result = - executor.retrieveResultChanges(context, desc.getResultId()); + executor.retrieveResultChanges(session, desc.getResultId()); if (result.getType() == TypedResult.ResultType.PAYLOAD) { for (Tuple2 change : result.getPayload()) { actualResults.add(change.toString()); @@ -168,7 +169,7 @@ public void testQueryExecutionChangelog() throws Exception { TestBaseUtils.compareResultCollections(expectedResults, actualResults, Comparator.naturalOrder()); } finally { - executor.stop(context); + executor.stop(session); } } @@ -182,11 +183,11 @@ public void testQueryExecutionTable() throws Exception { replaceVars.put("$VAR_2", "table"); final Executor executor = createModifiedExecutor(replaceVars); - final SessionContext context = new SessionContext("test-session", new Environment()); + final SessionContext session = new SessionContext("test-session", new Environment()); try { // start job and retrieval - final ResultDescriptor desc = executor.executeQuery(context, "SELECT IntegerField1 FROM TableNumber1"); + final ResultDescriptor desc = executor.executeQuery(session, "SELECT IntegerField1 FROM TableNumber1"); assertTrue(desc.isMaterialized()); @@ -194,7 +195,7 @@ public void testQueryExecutionTable() throws Exception { while (true) { Thread.sleep(50); // slow the processing down - final TypedResult result = executor.snapshotResult(context, desc.getResultId(), 2); + final TypedResult result = executor.snapshotResult(session, desc.getResultId(), 2); if (result.getType() == TypedResult.ResultType.PAYLOAD) { actualResults.clear(); IntStream.rangeClosed(1, result.getPayload()).forEach((page) -> { @@ -217,29 +218,21 @@ public void testQueryExecutionTable() throws Exception { TestBaseUtils.compareResultCollections(expectedResults, actualResults, Comparator.naturalOrder()); } finally { - executor.stop(context); + executor.stop(session); } } private LocalExecutor createDefaultExecutor() throws Exception { - final URL url = getClass().getClassLoader().getResource("test-sql-client-defaults.yaml"); - Objects.requireNonNull(url); - final Environment env = Environment.parse(url); - - return new LocalExecutor(env, Collections.emptyList(), cluster.getConfiguration()); + return new LocalExecutor( + EnvironmentFileUtil.parseUnmodified(DEFAULTS_ENVIRONMENT_FILE), + Collections.emptyList(), + cluster.getConfiguration()); } private LocalExecutor createModifiedExecutor(Map replaceVars) throws Exception { - final URL url = getClass().getClassLoader().getResource("test-sql-client-defaults.yaml"); - Objects.requireNonNull(url); - String schema = FileUtils.readFileUtf8(new File(url.getFile())); - - for (Map.Entry replaceVar : replaceVars.entrySet()) { - schema = schema.replace(replaceVar.getKey(), replaceVar.getValue()); - } - - final Environment env = Environment.parse(schema); - - return new LocalExecutor(env, Collections.emptyList(), cluster.getConfiguration()); + return new LocalExecutor( + EnvironmentFileUtil.parseModified(DEFAULTS_ENVIRONMENT_FILE, replaceVars), + Collections.emptyList(), + cluster.getConfiguration()); } } diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/EnvironmentFileUtil.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/EnvironmentFileUtil.java new file mode 100644 index 00000000000000..4645b424e2d85a --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/EnvironmentFileUtil.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.client.gateway.utils; + +import org.apache.flink.table.client.config.Environment; +import org.apache.flink.util.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Map; +import java.util.Objects; + +/** + * Utilities for reading an environment file. + */ +public final class EnvironmentFileUtil { + + private EnvironmentFileUtil() { + // private + } + + public static Environment parseUnmodified(String fileName) throws IOException { + final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName); + Objects.requireNonNull(url); + return Environment.parse(url); + } + + public static Environment parseModified(String fileName, Map replaceVars) throws IOException { + final URL url = EnvironmentFileUtil.class.getClassLoader().getResource(fileName); + Objects.requireNonNull(url); + String schema = FileUtils.readFileUtf8(new File(url.getFile())); + + for (Map.Entry replaceVar : replaceVars.entrySet()) { + schema = schema.replace(replaceVar.getKey(), replaceVar.getValue()); + } + + return Environment.parse(schema); + } +} diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java new file mode 100644 index 00000000000000..40a7e7bac67859 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.client.gateway.utils; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.Types; +import org.apache.flink.table.client.gateway.local.DependencyTest; +import org.apache.flink.table.descriptors.DescriptorProperties; +import org.apache.flink.table.sources.StreamTableSource; +import org.apache.flink.table.sources.TableSource; +import org.apache.flink.table.sources.TableSourceFactory; +import org.apache.flink.types.Row; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_NAME; +import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_TYPE; + +/** + * Table source factory for testing the classloading in {@link DependencyTest}. + */ +public class TestTableSourceFactory implements TableSourceFactory { + + @Override + public Map requiredContext() { + final Map context = new HashMap<>(); + context.put(CONNECTOR_TYPE(), "test-table-source-factory"); + return context; + } + + @Override + public List supportedProperties() { + final List properties = new ArrayList<>(); + properties.add("connector.test-property"); + properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); + properties.add(SCHEMA() + ".#." + SCHEMA_NAME()); + return properties; + } + + @Override + public TableSource create(Map properties) { + final DescriptorProperties params = new DescriptorProperties(true); + params.putProperties(properties); + return new TestTableSource( + params.getTableSchema(SCHEMA()), + properties.get("connector.test-property")); + } + + // -------------------------------------------------------------------------------------------- + + /** + * Test table source. + */ + public static class TestTableSource implements StreamTableSource { + + private final TableSchema schema; + private final String property; + + public TestTableSource(TableSchema schema, String property) { + this.schema = schema; + this.property = property; + } + + public String getProperty() { + return property; + } + + @Override + public DataStream getDataStream(StreamExecutionEnvironment execEnv) { + return null; + } + + @Override + public TypeInformation getReturnType() { + return Types.ROW(schema.getColumnNames(), schema.getTypes()); + } + + @Override + public TableSchema getTableSchema() { + return schema; + } + + @Override + public String explainSource() { + return "TestTableSource"; + } + } +} diff --git a/flink-libraries/flink-sql-client/src/test/resources/test-factory-services-file b/flink-libraries/flink-sql-client/src/test/resources/test-factory-services-file new file mode 100644 index 00000000000000..41e7fb2cbd5611 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/resources/test-factory-services-file @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#============================================================================== +# Test file for org.apache.flink.table.client.gateway.local.DependencyTest. +#============================================================================== + +org.apache.flink.table.client.gateway.utils.TestTableSourceFactory diff --git a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml index 1c2a705ae57d42..9cbecb07903ab7 100644 --- a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml +++ b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml @@ -18,6 +18,7 @@ #============================================================================== # TEST ENVIRONMENT FILE +# Intended for org.apache.flink.table.client.gateway.local.LocalExecutorITCase. #============================================================================== # this file has variables that can be filled with content by replacing $VAR_XXX diff --git a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml new file mode 100644 index 00000000000000..1bb69e537f40a8 --- /dev/null +++ b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml @@ -0,0 +1,45 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +#============================================================================== +# TEST ENVIRONMENT FILE +# Intended for org.apache.flink.table.client.gateway.local.DependencyTest. +#============================================================================== + +# this file has variables that can be filled with content by replacing $VAR_XXX + +sources: + - name: TableNumber1 + schema: + - name: IntegerField1 + type: INT + - name: StringField1 + type: VARCHAR + connector: + type: "$VAR_0" + $VAR_1: "$VAR_2" + +execution: + type: streaming + parallelism: 1 + +deployment: + type: standalone + response-timeout: 5000 + + diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala index ed3854df36b647..1e28303d0e48e7 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala @@ -85,7 +85,7 @@ class Rowtime extends Descriptor { * * Emits watermarks which are the maximum observed timestamp minus the specified delay. */ - def watermarksPeriodicBounding(delay: Long): Rowtime = { + def watermarksPeriodicBounded(delay: Long): Rowtime = { watermarkStrategy = Some(new BoundedOutOfOrderTimestamps(delay)) this } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala index 877cb7b5f39122..0c81335f6b6fb9 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala @@ -36,18 +36,39 @@ import scala.collection.mutable */ object TableSourceFactoryService extends Logging { - private lazy val loader = ServiceLoader.load(classOf[TableSourceFactory[_]]) + private lazy val defaultLoader = ServiceLoader.load(classOf[TableSourceFactory[_]]) def findAndCreateTableSource(descriptor: TableSourceDescriptor): TableSource[_] = { + findAndCreateTableSource(descriptor, null) + } + + def findAndCreateTableSource( + descriptor: TableSourceDescriptor, + classLoader: ClassLoader) + : TableSource[_] = { + val properties = new DescriptorProperties() descriptor.addProperties(properties) - findAndCreateTableSource(properties.asMap.asScala.toMap) + findAndCreateTableSource(properties.asMap.asScala.toMap, classLoader) } def findAndCreateTableSource(properties: Map[String, String]): TableSource[_] = { + findAndCreateTableSource(properties, null) + } + + def findAndCreateTableSource( + properties: Map[String, String], + classLoader: ClassLoader) + : TableSource[_] = { + var matchingFactory: Option[(TableSourceFactory[_], Seq[String])] = None try { - val iter = loader.iterator() + val iter = if (classLoader == null) { + defaultLoader.iterator() + } else { + val customLoader = ServiceLoader.load(classOf[TableSourceFactory[_]], classLoader) + customLoader.iterator() + } while (iter.hasNext) { val factory = iter.next() diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala index 7968b481db3438..9e339d02adc83c 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala @@ -46,7 +46,7 @@ class RowtimeTest extends DescriptorTestBase { override def descriptors(): util.List[Descriptor] = { val desc1 = Rowtime() .timestampsFromField("otherField") - .watermarksPeriodicBounding(1000L) + .watermarksPeriodicBounded(1000L) val desc2 = Rowtime() .timestampsFromSource() From 53168d00cf07f1b93d048f3fc849ce354fa8c7d4 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 6 Mar 2018 15:54:13 +0100 Subject: [PATCH 0128/2294] [hotfix] [taskmanager] Fix checkstyle in Task and TaskTest --- .../flink/runtime/taskmanager/Task.java | 80 +++++------ .../flink/runtime/taskmanager/TaskTest.java | 126 +++++++++--------- 2 files changed, 106 insertions(+), 100 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java index 1ecb47a6288d78..00fbdfefca68b4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java @@ -127,10 +127,10 @@ public class Task implements Runnable, TaskActions, CheckpointListener { /** The class logger. */ private static final Logger LOG = LoggerFactory.getLogger(Task.class); - /** The tread group that contains all task threads */ + /** The tread group that contains all task threads. */ private static final ThreadGroup TASK_THREADS_GROUP = new ThreadGroup("Flink Task Threads"); - /** For atomic state updates */ + /** For atomic state updates. */ private static final AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(Task.class, ExecutionState.class, "executionState"); @@ -138,52 +138,52 @@ public class Task implements Runnable, TaskActions, CheckpointListener { // Constant fields that are part of the initial Task construction // ------------------------------------------------------------------------ - /** The job that the task belongs to */ + /** The job that the task belongs to. */ private final JobID jobId; - /** The vertex in the JobGraph whose code the task executes */ + /** The vertex in the JobGraph whose code the task executes. */ private final JobVertexID vertexId; - /** The execution attempt of the parallel subtask */ + /** The execution attempt of the parallel subtask. */ private final ExecutionAttemptID executionId; - /** ID which identifies the slot in which the task is supposed to run */ + /** ID which identifies the slot in which the task is supposed to run. */ private final AllocationID allocationId; - /** TaskInfo object for this task */ + /** TaskInfo object for this task. */ private final TaskInfo taskInfo; - /** The name of the task, including subtask indexes */ + /** The name of the task, including subtask indexes. */ private final String taskNameWithSubtask; - /** The job-wide configuration object */ + /** The job-wide configuration object. */ private final Configuration jobConfiguration; - /** The task-specific configuration */ + /** The task-specific configuration. */ private final Configuration taskConfiguration; - /** The jar files used by this task */ + /** The jar files used by this task. */ private final Collection requiredJarFiles; - /** The classpaths used by this task */ + /** The classpaths used by this task. */ private final Collection requiredClasspaths; - /** The name of the class that holds the invokable code */ + /** The name of the class that holds the invokable code. */ private final String nameOfInvokableClass; - /** Access to task manager configuration and host names*/ + /** Access to task manager configuration and host names. */ private final TaskManagerRuntimeInfo taskManagerConfig; - /** The memory manager to be used by this task */ + /** The memory manager to be used by this task. */ private final MemoryManager memoryManager; - /** The I/O manager to be used by this task */ + /** The I/O manager to be used by this task. */ private final IOManager ioManager; - /** The BroadcastVariableManager to be used by this task */ + /** The BroadcastVariableManager to be used by this task. */ private final BroadcastVariableManager broadcastVariableManager; - /** The manager for state of operators running in this task/slot */ + /** The manager for state of operators running in this task/slot. */ private final TaskStateManager taskStateManager; /** Serialized version of the job specific execution configuration (see {@link ExecutionConfig}). */ @@ -195,43 +195,43 @@ public class Task implements Runnable, TaskActions, CheckpointListener { private final Map inputGatesById; - /** Connection to the task manager */ + /** Connection to the task manager. */ private final TaskManagerActions taskManagerActions; - /** Input split provider for the task */ + /** Input split provider for the task. */ private final InputSplitProvider inputSplitProvider; - /** Checkpoint notifier used to communicate with the CheckpointCoordinator */ + /** Checkpoint notifier used to communicate with the CheckpointCoordinator. */ private final CheckpointResponder checkpointResponder; - /** All listener that want to be notified about changes in the task's execution state */ + /** All listener that want to be notified about changes in the task's execution state. */ private final List taskExecutionStateListeners; - /** The BLOB cache, from which the task can request BLOB files */ + /** The BLOB cache, from which the task can request BLOB files. */ private final BlobCacheService blobService; - /** The library cache, from which the task can request its class loader */ + /** The library cache, from which the task can request its class loader. */ private final LibraryCacheManager libraryCache; - /** The cache for user-defined files that the invokable requires */ + /** The cache for user-defined files that the invokable requires. */ private final FileCache fileCache; - /** The gateway to the network stack, which handles inputs and produced results */ + /** The gateway to the network stack, which handles inputs and produced results. */ private final NetworkEnvironment network; - /** The registry of this task which enables live reporting of accumulators */ + /** The registry of this task which enables live reporting of accumulators. */ private final AccumulatorRegistry accumulatorRegistry; - /** The thread that executes the task */ + /** The thread that executes the task. */ private final Thread executingThread; - /** Parent group for all metrics of this task */ + /** Parent group for all metrics of this task. */ private final TaskMetricGroup metrics; - /** Partition producer state checker to request partition states from */ + /** Partition producer state checker to request partition states from. */ private final PartitionProducerStateChecker partitionProducerStateChecker; - /** Executor to run future callbacks */ + /** Executor to run future callbacks. */ private final Executor executor; // ------------------------------------------------------------------------ @@ -240,19 +240,19 @@ public class Task implements Runnable, TaskActions, CheckpointListener { // proper happens-before semantics on parallel modification // ------------------------------------------------------------------------ - /** atomic flag that makes sure the invokable is canceled exactly once upon error */ + /** atomic flag that makes sure the invokable is canceled exactly once upon error. */ private final AtomicBoolean invokableHasBeenCanceled; - /** The invokable of this task, if initialized */ + /** The invokable of this task, if initialized. */ private volatile AbstractInvokable invokable; - /** The current execution state of the task */ + /** The current execution state of the task. */ private volatile ExecutionState executionState = ExecutionState.CREATED; - /** The observed exception, in case the task execution failed */ + /** The observed exception, in case the task execution failed. */ private volatile Throwable failureCause; - /** Serial executor for asynchronous calls (checkpoints, etc), lazily initialized */ + /** Serial executor for asynchronous calls (checkpoints, etc), lazily initialized. */ private volatile ExecutorService asyncCallDispatcher; /** Initialized from the Flink configuration. May also be set at the ExecutionConfig */ @@ -512,7 +512,7 @@ public void startTaskThread() { } /** - * The core work method that bootstraps the task and executes its code + * The core work method that bootstraps the task and executes its code. */ @Override public void run() { @@ -556,7 +556,7 @@ else if (current == ExecutionState.CANCELING) { // all resource acquisitions and registrations from here on // need to be undone in the end - Map> distributedCacheEntries = new HashMap>(); + Map> distributedCacheEntries = new HashMap<>(); AbstractInvokable invokable = null; try { @@ -743,8 +743,8 @@ else if (current == ExecutionState.CANCELING) { try { // check if the exception is unrecoverable if (ExceptionUtils.isJvmFatalError(t) || - (t instanceof OutOfMemoryError && taskManagerConfig.shouldExitJvmOnOutOfMemoryError())) - { + (t instanceof OutOfMemoryError && taskManagerConfig.shouldExitJvmOnOutOfMemoryError())) { + // terminate the JVM immediately // don't attempt a clean shutdown, because we cannot expect the clean shutdown to complete try { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java index 5b33d1922b13a5..1829e977489eb5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java @@ -29,8 +29,6 @@ import org.apache.flink.runtime.broadcast.BroadcastVariableManager; import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.concurrent.Executors; -import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor; -import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor; import org.apache.flink.runtime.execution.CancelTaskException; import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.execution.ExecutionState; @@ -103,12 +101,12 @@ * Tests for the Task, which make sure that correct state transitions happen, * and failures are correctly handled. * - * All tests here have a set of mock actors for TaskManager, JobManager, and + *

    All tests here have a set of mock actors for TaskManager, JobManager, and * execution listener, which simply put the messages in a queue to be picked * up by the test and validated. */ public class TaskTest extends TestLogger { - + private static OneShotLatch awaitLatch; private static OneShotLatch triggerLatch; private static OneShotLatch cancelLatch; @@ -123,7 +121,7 @@ public class TaskTest extends TestLogger { private BlockingQueue taskManagerMessages; private BlockingQueue jobManagerMessages; private BlockingQueue listenerMessages; - + @Before public void createQueuesAndActors() { taskManagerMessages = new LinkedBlockingQueue<>(); @@ -135,7 +133,7 @@ public void createQueuesAndActors() { listener = new ActorGatewayTaskExecutionStateListener(listenerGateway); taskManagerConnection = new ActorGatewayTaskManagerActions(taskManagerGateway); - + awaitLatch = new OneShotLatch(); triggerLatch = new OneShotLatch(); cancelLatch = new OneShotLatch(); @@ -155,32 +153,32 @@ public void clearActorsAndMessages() { // ------------------------------------------------------------------------ // Tests // ------------------------------------------------------------------------ - + @Test public void testRegularExecution() { try { Task task = createTask(TestInvokableCorrect.class); - + // task should be new and perfect assertEquals(ExecutionState.CREATED, task.getExecutionState()); assertFalse(task.isCanceledOrFailed()); assertNull(task.getFailureCause()); - + task.registerExecutionListener(listener); - + // go into the run method. we should switch to DEPLOYING, RUNNING, then // FINISHED, and all should be good task.run(); - + // verify final state assertEquals(ExecutionState.FINISHED, task.getExecutionState()); assertFalse(task.isCanceledOrFailed()); assertNull(task.getFailureCause()); - + // verify listener messages validateListenerMessage(ExecutionState.RUNNING, task, false); validateListenerMessage(ExecutionState.FINISHED, task, false); - + // make sure that the TaskManager received an message to unregister the task validateTaskManagerStateChange(ExecutionState.RUNNING, task, false); validateUnregisterTask(task.getExecutionId()); @@ -198,7 +196,7 @@ public void testCancelRightAway() { task.cancelExecution(); assertEquals(ExecutionState.CANCELING, task.getExecutionState()); - + task.run(); // verify final state @@ -274,7 +272,7 @@ public void testExecutionFailsInNetworkRegistration() { // mock a working library cache LibraryCacheManager libCache = mock(LibraryCacheManager.class); when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader()); - + // mock a network manager that rejects registration ResultPartitionManager partitionManager = mock(ResultPartitionManager.class); ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class); @@ -296,7 +294,7 @@ public void testExecutionFailsInNetworkRegistration() { assertEquals(ExecutionState.FAILED, task.getExecutionState()); assertTrue(task.isCanceledOrFailed()); assertTrue(task.getFailureCause().getMessage().contains("buffers")); - + validateUnregisterTask(task.getExecutionId()); validateListenerMessage(ExecutionState.FAILED, task, true); } @@ -326,13 +324,13 @@ public void testInvokableInstantiationFailed() { fail(e.getMessage()); } } - + @Test public void testExecutionFailsInInvoke() { try { Task task = createTask(InvokableWithExceptionInInvoke.class); task.registerExecutionListener(listener); - + task.run(); assertEquals(ExecutionState.FAILED, task.getExecutionState()); @@ -343,7 +341,7 @@ public void testExecutionFailsInInvoke() { validateTaskManagerStateChange(ExecutionState.RUNNING, task, false); validateUnregisterTask(task.getExecutionId()); - + validateListenerMessage(ExecutionState.RUNNING, task, false); validateListenerMessage(ExecutionState.FAILED, task, true); } @@ -378,7 +376,7 @@ public void testFailWithWrappedException() { fail(e.getMessage()); } } - + @Test public void testCancelDuringInvoke() { try { @@ -403,7 +401,7 @@ public void testCancelDuringInvoke() { validateTaskManagerStateChange(ExecutionState.RUNNING, task, false); validateUnregisterTask(task.getExecutionId()); - + validateListenerMessage(ExecutionState.RUNNING, task, false); validateCancelingAndCanceledListenerMessage(task); } @@ -463,7 +461,7 @@ public void testCanceledAfterExecutionFailedInInvoke() { validateTaskManagerStateChange(ExecutionState.RUNNING, task, false); validateUnregisterTask(task.getExecutionId()); - + validateListenerMessage(ExecutionState.RUNNING, task, false); validateListenerMessage(ExecutionState.FAILED, task, true); } @@ -472,7 +470,7 @@ public void testCanceledAfterExecutionFailedInInvoke() { fail(e.getMessage()); } } - + @Test public void testExecutionFailesAfterCanceling() { try { @@ -487,7 +485,7 @@ public void testExecutionFailesAfterCanceling() { task.cancelExecution(); assertEquals(ExecutionState.CANCELING, task.getExecutionState()); - + // this causes an exception triggerLatch.trigger(); @@ -497,7 +495,7 @@ public void testExecutionFailesAfterCanceling() { assertEquals(ExecutionState.CANCELED, task.getExecutionState()); assertTrue(task.isCanceledOrFailed()); assertNull(task.getFailureCause()); - + validateTaskManagerStateChange(ExecutionState.RUNNING, task, false); validateUnregisterTask(task.getExecutionId()); @@ -529,11 +527,11 @@ public void testExecutionFailsAfterTaskMarkedFailed() { triggerLatch.trigger(); task.getExecutingThread().join(); - + assertEquals(ExecutionState.FAILED, task.getExecutionState()); assertTrue(task.isCanceledOrFailed()); assertTrue(task.getFailureCause().getMessage().contains("external")); - + validateTaskManagerStateChange(ExecutionState.RUNNING, task, false); validateUnregisterTask(task.getExecutionId()); @@ -606,7 +604,7 @@ public void testOnPartitionStateUpdate() throws Exception { expected.put(ExecutionState.SCHEDULED, ExecutionState.RUNNING); expected.put(ExecutionState.DEPLOYING, ExecutionState.RUNNING); expected.put(ExecutionState.FINISHED, ExecutionState.RUNNING); - + expected.put(ExecutionState.CANCELED, ExecutionState.CANCELING); expected.put(ExecutionState.CANCELING, ExecutionState.CANCELING); expected.put(ExecutionState.FAILED, ExecutionState.CANCELING); @@ -901,10 +899,9 @@ private void setState(Task task, ExecutionState state) { * @return BlobCache mock with the bare minimum of implemented functions that work */ private BlobCacheService createBlobCache() { - BlobCacheService blobService = - new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class)); - - return blobService; + return new BlobCacheService( + mock(PermanentBlobCache.class), + mock(TransientBlobCache.class)); } private Task createTask(Class invokable) throws IOException { @@ -915,14 +912,14 @@ private Task createTask(Class invokable, Configurat BlobCacheService blobService = createBlobCache(); LibraryCacheManager libCache = mock(LibraryCacheManager.class); when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader()); - return createTask(invokable, blobService,libCache, config, new ExecutionConfig()); + return createTask(invokable, blobService, libCache, config, new ExecutionConfig()); } private Task createTask(Class invokable, Configuration config, ExecutionConfig execConfig) throws IOException { BlobCacheService blobService = createBlobCache(); LibraryCacheManager libCache = mock(LibraryCacheManager.class); when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader()); - return createTask(invokable, blobService,libCache, config, execConfig); + return createTask(invokable, blobService, libCache, config, execConfig); } private Task createTask( @@ -930,7 +927,7 @@ private Task createTask( BlobCacheService blobService, LibraryCacheManager libCache) throws IOException { - return createTask(invokable, blobService,libCache, new Configuration(), new ExecutionConfig()); + return createTask(invokable, blobService, libCache, new Configuration(), new ExecutionConfig()); } private Task createTask( @@ -965,7 +962,7 @@ private Task createTask( Executor executor) throws IOException { return createTask(invokable, blobService, libCache, networkEnvironment, consumableNotifier, partitionProducerStateChecker, executor, new Configuration(), new ExecutionConfig()); } - + private Task createTask( Class invokable, BlobCacheService blobService, @@ -976,7 +973,7 @@ private Task createTask( Executor executor, Configuration taskManagerConfig, ExecutionConfig execConfig) throws IOException { - + JobID jobId = new JobID(); JobVertexID jobVertexId = new JobVertexID(); ExecutionAttemptID executionAttemptId = new ExecutionAttemptID(); @@ -1010,7 +1007,7 @@ private Task createTask( TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class); when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class)); - + return new Task( jobInformation, taskInformation, @@ -1018,8 +1015,8 @@ private Task createTask( new AllocationID(), 0, 0, - Collections.emptyList(), - Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), 0, mock(MemoryManager.class), mock(IOManager.class), @@ -1042,18 +1039,18 @@ private Task createTask( // ------------------------------------------------------------------------ // Validation Methods // ------------------------------------------------------------------------ - + private void validateUnregisterTask(ExecutionAttemptID id) { try { // we may have to wait for a bit to give the actors time to receive the message // and put it into the queue Object rawMessage = taskManagerMessages.take(); - + assertNotNull("There is no additional TaskManager message", rawMessage); if (!(rawMessage instanceof TaskMessages.TaskInFinalState)) { fail("TaskManager message is not 'UnregisterTask', but " + rawMessage.getClass()); } - + TaskMessages.TaskInFinalState message = (TaskMessages.TaskInFinalState) rawMessage; assertEquals(id, message.executionID()); } @@ -1072,10 +1069,10 @@ private void validateTaskManagerStateChange(ExecutionState state, Task task, boo if (!(rawMessage instanceof TaskMessages.UpdateTaskExecutionState)) { fail("TaskManager message is not 'UpdateTaskExecutionState', but " + rawMessage.getClass()); } - + TaskMessages.UpdateTaskExecutionState message = (TaskMessages.UpdateTaskExecutionState) rawMessage; - + TaskExecutionState taskState = message.taskExecutionState(); assertEquals(task.getJobID(), taskState.getJobID()); @@ -1092,7 +1089,7 @@ private void validateTaskManagerStateChange(ExecutionState state, Task task, boo fail("interrupted"); } } - + private void validateListenerMessage(ExecutionState state, Task task, boolean hasError) { try { // we may have to wait for a bit to give the actors time to receive the message @@ -1100,13 +1097,13 @@ private void validateListenerMessage(ExecutionState state, Task task, boolean ha TaskMessages.UpdateTaskExecutionState message = (TaskMessages.UpdateTaskExecutionState) listenerMessages.take(); assertNotNull("There is no additional listener message", message); - + TaskExecutionState taskState = message.taskExecutionState(); assertEquals(task.getJobID(), taskState.getJobID()); assertEquals(task.getExecutionId(), taskState.getID()); assertEquals(state, taskState.getExecutionState()); - + if (hasError) { assertNotNull(taskState.getError(getClass().getClassLoader())); } else { @@ -1126,8 +1123,7 @@ private void validateCancelingAndCanceledListenerMessage(Task task) { (TaskMessages.UpdateTaskExecutionState) listenerMessages.take(); TaskMessages.UpdateTaskExecutionState message2 = (TaskMessages.UpdateTaskExecutionState) listenerMessages.take(); - - + assertNotNull("There is no additional listener message", message1); assertNotNull("There is no additional listener message", message2); @@ -1138,30 +1134,31 @@ private void validateCancelingAndCanceledListenerMessage(Task task) { assertEquals(task.getJobID(), taskState2.getJobID()); assertEquals(task.getExecutionId(), taskState1.getID()); assertEquals(task.getExecutionId(), taskState2.getID()); - + ExecutionState state1 = taskState1.getExecutionState(); ExecutionState state2 = taskState2.getExecutionState(); - + // it may be (very rarely) that the following race happens: // - OUTSIDE THREAD: call to cancel() // - OUTSIDE THREAD: atomic state change from running to canceling // - TASK THREAD: finishes, atomic change from canceling to canceled // - TASK THREAD: send notification that state is canceled // - OUTSIDE THREAD: send notification that state is canceling - + // for that reason, we allow the notification messages in any order. - assertTrue( (state1 == ExecutionState.CANCELING && state2 == ExecutionState.CANCELED) || + assertTrue((state1 == ExecutionState.CANCELING && state2 == ExecutionState.CANCELED) || (state2 == ExecutionState.CANCELING && state1 == ExecutionState.CANCELED)); } catch (InterruptedException e) { fail("interrupted"); } } - + // -------------------------------------------------------------------------------------------- // Mock invokable code // -------------------------------------------------------------------------------------------- - + + /** Test task class. */ public static final class TestInvokableCorrect extends AbstractInvokable { public TestInvokableCorrect(Environment environment) { @@ -1177,6 +1174,7 @@ public void cancel() throws Exception { } } + /** Test task class. */ public static final class InvokableWithExceptionInInvoke extends AbstractInvokable { public InvokableWithExceptionInInvoke(Environment environment) { @@ -1189,6 +1187,7 @@ public void invoke() throws Exception { } } + /** Test task class. */ public static final class InvokableWithExceptionOnTrigger extends AbstractInvokable { public InvokableWithExceptionOnTrigger(Environment environment) { @@ -1198,7 +1197,7 @@ public InvokableWithExceptionOnTrigger(Environment environment) { @Override public void invoke() { awaitLatch.trigger(); - + // make sure that the interrupt call does not // grab us out of the lock early while (true) { @@ -1215,13 +1214,15 @@ public void invoke() { } } - public static abstract class InvokableNonInstantiable extends AbstractInvokable { + /** Test task class. */ + public abstract static class InvokableNonInstantiable extends AbstractInvokable { public InvokableNonInstantiable(Environment environment) { super(environment); } } + /** Test task class. */ public static final class InvokableBlockingInInvoke extends AbstractInvokable { public InvokableBlockingInInvoke(Environment environment) { @@ -1231,7 +1232,7 @@ public InvokableBlockingInInvoke(Environment environment) { @Override public void invoke() throws Exception { awaitLatch.trigger(); - + // block forever synchronized (this) { wait(); @@ -1239,6 +1240,7 @@ public void invoke() throws Exception { } } + /** Test task class. */ public static final class InvokableWithCancelTaskExceptionInInvoke extends AbstractInvokable { public InvokableWithCancelTaskExceptionInInvoke(Environment environment) { @@ -1253,11 +1255,12 @@ public void invoke() throws Exception { triggerLatch.await(); } catch (Throwable ignored) {} - + throw new CancelTaskException(); } } + /** Test task class. */ public static final class InvokableInterruptableSharedLockInInvokeAndCancel extends AbstractInvokable { private final Object lock = new Object(); @@ -1282,6 +1285,7 @@ public void cancel() throws Exception { } } + /** Test task class. */ public static final class InvokableBlockingInCancel extends AbstractInvokable { public InvokableBlockingInCancel(Environment environment) { @@ -1313,6 +1317,7 @@ public void cancel() throws Exception { } } + /** Test task class. */ public static final class InvokableUninterruptibleBlockingInvoke extends AbstractInvokable { public InvokableUninterruptibleBlockingInvoke(Environment environment) { @@ -1337,6 +1342,7 @@ public void cancel() throws Exception { } } + /** Test task class. */ public static final class FailingInvokableWithChainedException extends AbstractInvokable { public FailingInvokableWithChainedException(Environment environment) { From 1e1237628a6bf05cdcdde6f9f3a236d961f05b5d Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 6 Mar 2018 15:18:33 +0100 Subject: [PATCH 0129/2294] [FLINK-8856] [TaskManager] Move all cancellation interrupt calls to TaskCanceller thread This cleans up the code and guards against a JVM bug where 'interrupt()' calls block/deadlock if the thread is engaged in certain I/O operations. In addition, this makes sure that the process really goes away when the cancellation timeout expires, rather than relying on the TaskManager to be able to properly handle the fatal error notification. --- .../flink/runtime/taskmanager/Task.java | 280 +++++++++++------- 1 file changed, 177 insertions(+), 103 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java index 00fbdfefca68b4..ccb850e85bc127 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java @@ -67,6 +67,7 @@ import org.apache.flink.runtime.query.TaskKvStateRegistry; import org.apache.flink.runtime.state.CheckpointListener; import org.apache.flink.runtime.state.TaskStateManager; +import org.apache.flink.runtime.util.FatalExitExceptionHandler; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.Preconditions; @@ -77,7 +78,6 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.IOException; import java.lang.reflect.Constructor; @@ -94,11 +94,11 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; import static org.apache.flink.util.Preconditions.checkState; @@ -1048,15 +1048,51 @@ else if (current == ExecutionState.RUNNING) { invokable, executingThread, taskNameWithSubtask, - taskCancellationInterval, - taskCancellationTimeout, - taskManagerActions, producedPartitions, inputGates); - Thread cancelThread = new Thread(executingThread.getThreadGroup(), canceler, + + Thread cancelThread = new Thread( + executingThread.getThreadGroup(), + canceler, String.format("Canceler for %s (%s).", taskNameWithSubtask, executionId)); cancelThread.setDaemon(true); + cancelThread.setUncaughtExceptionHandler(FatalExitExceptionHandler.INSTANCE); cancelThread.start(); + + // the periodic interrupting thread - a different thread than the canceller, in case + // the application code does blocking stuff in its cancellation paths. + Runnable interrupter = new TaskInterrupter( + LOG, + executingThread, + taskNameWithSubtask, + taskCancellationInterval); + + Thread interruptingThread = new Thread( + executingThread.getThreadGroup(), + interrupter, + String.format("Canceler/Interrupts for %s (%s).", taskNameWithSubtask, executionId)); + interruptingThread.setDaemon(true); + interruptingThread.setUncaughtExceptionHandler(FatalExitExceptionHandler.INSTANCE); + interruptingThread.start(); + + // if a cancellation timeout is set, the watchdog thread kills the process + // if graceful cancellation does not succeed + if (taskCancellationTimeout > 0) { + Runnable cancelWatchdog = new TaskCancelerWatchDog( + executingThread, + taskManagerActions, + taskCancellationTimeout, + LOG); + + Thread watchDogThread = new Thread( + executingThread.getThreadGroup(), + cancelWatchdog, + String.format("Cancellation Watchdog for %s (%s).", + taskNameWithSubtask, executionId)); + watchDogThread.setDaemon(true); + watchDogThread.setUncaughtExceptionHandler(FatalExitExceptionHandler.INSTANCE); + watchDogThread.start(); + } } return; } @@ -1408,12 +1444,28 @@ private static AbstractInvokable loadAndInstantiateInvokable( } // ------------------------------------------------------------------------ - // TaskCanceler + // Task cancellation + // + // The task cancellation uses in total three threads, as a safety net + // against various forms of user- and JVM bugs. + // + // - The first thread calls 'cancel()' on the invokable and closes + // the input and output connections, for fast thread termination + // - The second thread periodically interrupts the invokable in order + // to pull the thread out of blocking wait and I/O operations + // - The third thread (watchdog thread) waits until the cancellation + // timeout and then performs a hard cancel (kill process, or let + // the TaskManager know) + // + // Previously, thread two and three were in one thread, but we needed + // to separate this to make sure the watchdog thread does not call + // 'interrupt()'. This is a workaround for the following JVM bug + // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8138622 // ------------------------------------------------------------------------ /** - * This runner calls cancel() on the invokable and periodically interrupts the - * thread until it has terminated. + * This runner calls cancel() on the invokable, closes input-/output resources, + * and initially interrupts the task thread. */ private static class TaskCanceler implements Runnable { @@ -1424,27 +1476,11 @@ private static class TaskCanceler implements Runnable { private final ResultPartition[] producedPartitions; private final SingleInputGate[] inputGates; - /** Interrupt interval. */ - private final long interruptInterval; - - /** Timeout after which a fatal error notification happens. */ - private final long interruptTimeout; - - /** TaskManager to notify about a timeout */ - private final TaskManagerActions taskManager; - - /** Watch Dog thread */ - @Nullable - private final Thread watchDogThread; - public TaskCanceler( Logger logger, AbstractInvokable invokable, Thread executer, String taskName, - long cancellationInterval, - long cancellationTimeout, - TaskManagerActions taskManager, ResultPartition[] producedPartitions, SingleInputGate[] inputGates) { @@ -1452,39 +1488,19 @@ public TaskCanceler( this.invokable = invokable; this.executer = executer; this.taskName = taskName; - this.interruptInterval = cancellationInterval; - this.interruptTimeout = cancellationTimeout; - this.taskManager = taskManager; this.producedPartitions = producedPartitions; this.inputGates = inputGates; - - if (cancellationTimeout > 0) { - // The watch dog repeatedly interrupts the executor until - // the cancellation timeout kicks in (at which point the - // task manager is notified about a fatal error) or the - // executor has terminated. - this.watchDogThread = new Thread( - executer.getThreadGroup(), - new TaskCancelerWatchDog(), - "WatchDog for " + taskName + " cancellation"); - this.watchDogThread.setDaemon(true); - } else { - this.watchDogThread = null; - } } @Override public void run() { try { - if (watchDogThread != null) { - watchDogThread.start(); - } - // the user-defined cancel method may throw errors. // we need do continue despite that try { invokable.cancel(); } catch (Throwable t) { + ExceptionUtils.rethrowIfFatalError(t); logger.error("Error while canceling the task {}.", taskName, t); } @@ -1499,6 +1515,7 @@ public void run() { try { partition.destroyBufferPool(); } catch (Throwable t) { + ExceptionUtils.rethrowIfFatalError(t); LOG.error("Failed to release result partition buffer pool for task {}.", taskName, t); } } @@ -1507,89 +1524,146 @@ public void run() { try { inputGate.releaseAllResources(); } catch (Throwable t) { + ExceptionUtils.rethrowIfFatalError(t); LOG.error("Failed to release input gate for task {}.", taskName, t); } } - // interrupt the running thread initially + // send the initial interruption signal executer.interrupt(); - try { - executer.join(interruptInterval); - } - catch (InterruptedException e) { - // we can ignore this - } - - if (watchDogThread != null) { - watchDogThread.interrupt(); - watchDogThread.join(); - } - } catch (Throwable t) { + } + catch (Throwable t) { + ExceptionUtils.rethrowIfFatalError(t); logger.error("Error in the task canceler for task {}.", taskName, t); } } + } + + /** + * This thread sends the delayed, periodic interrupt calls to the executing thread. + */ + private static final class TaskInterrupter implements Runnable { - /** - * Watchdog for the cancellation. If the task is stuck in cancellation, - * we notify the task manager about a fatal error. - */ - private class TaskCancelerWatchDog implements Runnable { + /** The logger to report on the fatal condition. */ + private final Logger log; - @Override - public void run() { - long intervalNanos = TimeUnit.NANOSECONDS.convert(interruptInterval, TimeUnit.MILLISECONDS); - long timeoutNanos = TimeUnit.NANOSECONDS.convert(interruptTimeout, TimeUnit.MILLISECONDS); - long deadline = System.nanoTime() + timeoutNanos; + /** The executing task thread that we wait for to terminate. */ + private final Thread executerThread; - try { - // Initial wait before interrupting periodically - Thread.sleep(interruptInterval); - } catch (InterruptedException ignored) { - } + /** The name of the task, for logging purposes. */ + private final String taskName; - // It is possible that the user code does not react to the task canceller. - // for that reason, we spawn this separate thread that repeatedly interrupts - // the user code until it exits. If the user code does not exit within - // the timeout, we notify the job manager about a fatal error. - while (executer.isAlive()) { - long now = System.nanoTime(); + /** The interval in which we interrupt. */ + private final long interruptIntervalMillis; + + TaskInterrupter( + Logger log, + Thread executerThread, + String taskName, + long interruptIntervalMillis) { + + this.log = log; + this.executerThread = executerThread; + this.taskName = taskName; + this.interruptIntervalMillis = interruptIntervalMillis; + } + @Override + public void run() { + try { + // we initially wait for one interval + // in most cases, the threads go away immediately (by the cancellation thread) + // and we need not actually do anything + executerThread.join(interruptIntervalMillis); + + // log stack trace where the executing thread is stuck and + // interrupt the running thread periodically while it is still alive + while (executerThread.isAlive()) { // build the stack trace of where the thread is stuck, for the log + StackTraceElement[] stack = executerThread.getStackTrace(); StringBuilder bld = new StringBuilder(); - StackTraceElement[] stack = executer.getStackTrace(); for (StackTraceElement e : stack) { bld.append(e).append('\n'); } - if (now >= deadline) { - long duration = TimeUnit.SECONDS.convert(interruptInterval, TimeUnit.MILLISECONDS); - String msg = String.format("Task '%s' did not react to cancelling signal in " + - "the last %d seconds, but is stuck in method:\n %s", - taskName, - duration, - bld.toString()); + log.warn("Task '{}' did not react to cancelling signal for {} seconds, but is stuck in method:\n {}", + taskName, (interruptIntervalMillis / 1000), bld); - logger.info("Notifying TaskManager about fatal error. {}.", msg); + executerThread.interrupt(); + try { + executerThread.join(interruptIntervalMillis); + } + catch (InterruptedException e) { + // we ignore this and fall through the loop + } + } + } catch (Throwable t) { + ExceptionUtils.rethrowIfFatalError(t); + log.error("Error in the task canceler for task {}.", taskName, t); + } + } + } - taskManager.notifyFatalError(msg, null); + /** + * Watchdog for the cancellation. + * If the task thread does not go away gracefully within a certain time, we + * trigger a hard cancel action (notify TaskManager of fatal error, which in + * turn kills the process). + */ + private static class TaskCancelerWatchDog implements Runnable { - return; // done, don't forget to leave the loop - } else { - logger.warn("Task '{}' did not react to cancelling signal, but is stuck in method:\n {}", - taskName, bld.toString()); + /** The logger to report on the fatal condition. */ + private final Logger log; - executer.interrupt(); - try { - long timeLeftNanos = Math.min(intervalNanos, deadline - now); - long timeLeftMillis = TimeUnit.MILLISECONDS.convert(timeLeftNanos, TimeUnit.NANOSECONDS); + /** The executing task thread that we wait for to terminate. */ + private final Thread executerThread; - if (timeLeftMillis > 0) { - executer.join(timeLeftMillis); - } - } catch (InterruptedException ignored) { - } + /** The TaskManager to notify if cancellation does not happen in time. */ + private final TaskManagerActions taskManager; + + /** The timeout for cancellation. */ + private final long timeoutMillis; + + TaskCancelerWatchDog( + Thread executerThread, + TaskManagerActions taskManager, + long timeoutMillis, + Logger log) { + + checkArgument(timeoutMillis > 0); + + this.log = log; + this.executerThread = executerThread; + this.taskManager = taskManager; + this.timeoutMillis = timeoutMillis; + } + + @Override + public void run() { + try { + final long hardKillDeadline = System.nanoTime() + timeoutMillis * 1_000_000; + + long millisLeft; + while (executerThread.isAlive() + && (millisLeft = (hardKillDeadline - System.nanoTime()) / 1_000_000) > 0) { + + try { + executerThread.join(millisLeft); + } + catch (InterruptedException ignored) { + // we don't react to interrupted exceptions, simply fall through the loop } } + + if (executerThread.isAlive()) { + String msg = "Task did not exit gracefully within " + (timeoutMillis / 1000) + " + seconds."; + log.error(msg); + taskManager.notifyFatalError(msg, null); + } + } + catch (Throwable t) { + ExceptionUtils.rethrowIfFatalError(t); + log.error("Error in Task Cancellation Watch Dog", t); } } } From 1d77b66833e75359dd4088f01083ecd9b68898ff Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 6 Mar 2018 16:36:13 +0100 Subject: [PATCH 0130/2294] [FLINK-8883] [core] Make ThreadDeath a fatal error in ExceptionUtils --- .../src/main/java/org/apache/flink/util/ExceptionUtils.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java index b9a21ae3495287..6af16fcfa4f6c1 100644 --- a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java +++ b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java @@ -81,12 +81,15 @@ public static String stringifyException(final Throwable e) { *

    Currently considered fatal exceptions are Virtual Machine errors indicating * that the JVM is corrupted, like {@link InternalError}, {@link UnknownError}, * and {@link java.util.zip.ZipError} (a special case of InternalError). + * The {@link ThreadDeath} exception is also treated as a fatal error, because when + * a thread is forcefully stopped, there is a high chance that parts of the system + * are in an inconsistent state. * * @param t The exception to check. * @return True, if the exception is considered fatal to the JVM, false otherwise. */ public static boolean isJvmFatalError(Throwable t) { - return (t instanceof InternalError) || (t instanceof UnknownError); + return (t instanceof InternalError) || (t instanceof UnknownError) || (t instanceof ThreadDeath); } /** From dd82f8720fa5aeb3a29c858cd7c032ff25f935ae Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 6 Mar 2018 17:14:54 +0100 Subject: [PATCH 0131/2294] [FLINK-8885] [TaskManager] DispatcherThreadFactory registers a fatal error exception handler In case dispatcher threads let an exception bubble out (do not handle the exception), the exception handler terminates the process, to ensure we don't leave broken processes. --- .../runtime/taskmanager/DispatcherThreadFactory.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/DispatcherThreadFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/DispatcherThreadFactory.java index 543b15929e1dea..9cec52402b06cf 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/DispatcherThreadFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/DispatcherThreadFactory.java @@ -18,6 +18,8 @@ package org.apache.flink.runtime.taskmanager; +import org.apache.flink.runtime.util.FatalExitExceptionHandler; + import javax.annotation.Nullable; import java.util.concurrent.ThreadFactory; @@ -27,13 +29,14 @@ * thread group, and set them to daemon mode. */ public class DispatcherThreadFactory implements ThreadFactory { - + private final ThreadGroup group; - + private final String threadName; + @Nullable private final ClassLoader classLoader; - + /** * Creates a new thread factory. * @@ -67,6 +70,7 @@ public Thread newThread(Runnable r) { t.setContextClassLoader(classLoader); } t.setDaemon(true); + t.setUncaughtExceptionHandler(FatalExitExceptionHandler.INSTANCE); return t; } } From 4dcc492bdbd9eb986e0b367c27f53fda7238eeb7 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 6 Mar 2018 17:18:38 +0100 Subject: [PATCH 0132/2294] [hotfix] [runtime] Harden FatalExitExceptionHandler In case the logging framework throws an exception when handling the exception, we still kill the process, as intended. --- .../flink/runtime/util/FatalExitExceptionHandler.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/FatalExitExceptionHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/FatalExitExceptionHandler.java index c57b75a452fa40..e162a580cbe12d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/FatalExitExceptionHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/FatalExitExceptionHandler.java @@ -34,9 +34,14 @@ public final class FatalExitExceptionHandler implements Thread.UncaughtException public static final FatalExitExceptionHandler INSTANCE = new FatalExitExceptionHandler(); @Override + @SuppressWarnings("finally") public void uncaughtException(Thread t, Throwable e) { - LOG.error("FATAL: Thread '" + t.getName() + - "' produced an uncaught exception. Stopping the process...", e); - System.exit(-17); + try { + LOG.error("FATAL: Thread '" + t.getName() + + "' produced an uncaught exception. Stopping the process...", e); + } + finally { + System.exit(-17); + } } } From 94e959f4fc747f86b84055b333c0b60f45fb5c40 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 7 Mar 2018 15:02:05 +0100 Subject: [PATCH 0133/2294] [FLINK-8887][tests] Add single retry in MiniClusterClient This closes #5657. --- .../client/program/MiniClusterClient.java | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 67e49fa28e31f3..7475071e23e6e3 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -25,11 +25,17 @@ import org.apache.flink.runtime.client.JobExecutionException; import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse; +import org.apache.flink.runtime.concurrent.FutureUtils; +import org.apache.flink.runtime.concurrent.ScheduledExecutor; +import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.minicluster.MiniCluster; +import org.apache.flink.runtime.rpc.akka.exceptions.AkkaRpcException; +import org.apache.flink.runtime.rpc.exceptions.FencingTokenException; +import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.runtime.util.LeaderRetrievalUtils; import org.apache.flink.util.FlinkException; @@ -43,6 +49,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Supplier; /** * Client to interact with a {@link MiniCluster}. @@ -50,6 +59,8 @@ public class MiniClusterClient extends ClusterClient { private final MiniCluster miniCluster; + private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(4, new ExecutorThreadFactory("Flink-MiniClusterClient")); + private final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(scheduledExecutorService); public MiniClusterClient(@Nonnull Configuration configuration, @Nonnull MiniCluster miniCluster) throws Exception { super(configuration, miniCluster.getHighAvailabilityServices(), true); @@ -57,6 +68,12 @@ public MiniClusterClient(@Nonnull Configuration configuration, @Nonnull MiniClus this.miniCluster = miniCluster; } + @Override + public void shutdown() throws Exception { + super.shutdown(); + scheduledExecutorService.shutdown(); + } + @Override public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { if (isDetached()) { @@ -82,12 +99,12 @@ public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) @Override public void cancel(JobID jobId) throws Exception { - miniCluster.cancelJob(jobId); + guardWithSingleRetry(() -> miniCluster.cancelJob(jobId), scheduledExecutor); } @Override public String cancelWithSavepoint(JobID jobId, @Nullable String savepointDirectory) throws Exception { - return miniCluster.triggerSavepoint(jobId, savepointDirectory, true).get(); + return guardWithSingleRetry(() -> miniCluster.triggerSavepoint(jobId, savepointDirectory, true), scheduledExecutor).get(); } @Override @@ -122,7 +139,7 @@ public Map getAccumulators(JobID jobID, ClassLoader loader) thro @Override public CompletableFuture getJobStatus(JobID jobId) { - return miniCluster.getJobStatus(jobId); + return guardWithSingleRetry(() -> miniCluster.getJobStatus(jobId), scheduledExecutor); } @Override @@ -174,4 +191,13 @@ public boolean hasUserJarsInClassPath(List userJarFiles) { enum MiniClusterId { INSTANCE } + + private static CompletableFuture guardWithSingleRetry(Supplier> operation, ScheduledExecutor executor) { + return FutureUtils.retryWithDelay( + operation, + 1, + Time.milliseconds(500), + throwable -> throwable instanceof FencingTokenException || throwable instanceof AkkaRpcException, + executor); + } } From 4e0bc008e162bfbdec2151fbb10f815b81bc24db Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 5 Mar 2018 13:45:33 +0100 Subject: [PATCH 0134/2294] [hotfix][tests] Do not use singleActorSystem in LocalFlinkMiniCluster Using a singleActorSystem rendered the returned client unusable. This closes #5652. --- .../flink/test/util/MiniClusterResource.java | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java index 954b06f65a9ca4..66cbb9f9b77006 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java @@ -62,6 +62,8 @@ public class MiniClusterResource extends ExternalResource { private JobExecutorService jobExecutorService; + private final boolean enableClusterClient; + private ClusterClient clusterClient; private int numberSlots = -1; @@ -69,16 +71,25 @@ public class MiniClusterResource extends ExternalResource { private TestEnvironment executionEnvironment; public MiniClusterResource(final MiniClusterResourceConfiguration miniClusterResourceConfiguration) { + this(miniClusterResourceConfiguration, false); + } + + public MiniClusterResource( + final MiniClusterResourceConfiguration miniClusterResourceConfiguration, + final boolean enableClusterClient) { this( miniClusterResourceConfiguration, - Objects.equals(FLIP6_CODEBASE, System.getProperty(CODEBASE_KEY)) ? MiniClusterType.FLIP6 : MiniClusterType.OLD); + Objects.equals(FLIP6_CODEBASE, System.getProperty(CODEBASE_KEY)) ? MiniClusterType.FLIP6 : MiniClusterType.OLD, + enableClusterClient); } - public MiniClusterResource( + private MiniClusterResource( final MiniClusterResourceConfiguration miniClusterResourceConfiguration, - final MiniClusterType miniClusterType) { + final MiniClusterType miniClusterType, + final boolean enableClusterClient) { this.miniClusterResourceConfiguration = Preconditions.checkNotNull(miniClusterResourceConfiguration); this.miniClusterType = Preconditions.checkNotNull(miniClusterType); + this.enableClusterClient = enableClusterClient; } public int getNumberSlots() { @@ -86,6 +97,12 @@ public int getNumberSlots() { } public ClusterClient getClusterClient() { + if (!enableClusterClient) { + // this check is technically only necessary for legacy clusters + // we still fail here for flip6 to keep the behaviors in sync + throw new IllegalStateException("To use the client you must enable it with the constructor."); + } + return clusterClient; } @@ -113,10 +130,12 @@ public void after() { Exception exception = null; - try { - clusterClient.shutdown(); - } catch (Exception e) { - exception = e; + if (clusterClient != null) { + try { + clusterClient.shutdown(); + } catch (Exception e) { + exception = e; + } } clusterClient = null; @@ -158,10 +177,12 @@ private void startOldMiniCluster() throws Exception { final LocalFlinkMiniCluster flinkMiniCluster = TestBaseUtils.startCluster( configuration, - true); + !enableClusterClient); // the cluster client only works if separate actor systems are used jobExecutorService = flinkMiniCluster; - clusterClient = new StandaloneClusterClient(configuration, flinkMiniCluster.highAvailabilityServices(), true); + if (enableClusterClient) { + clusterClient = new StandaloneClusterClient(configuration, flinkMiniCluster.highAvailabilityServices(), true); + } } private void startFlip6MiniCluster() throws Exception { @@ -188,7 +209,9 @@ private void startFlip6MiniCluster() throws Exception { configuration.setInteger(RestOptions.REST_PORT, miniCluster.getRestAddress().getPort()); jobExecutorService = miniCluster; - clusterClient = new MiniClusterClient(configuration, miniCluster); + if (enableClusterClient) { + clusterClient = new MiniClusterClient(configuration, miniCluster); + } } /** From f897c14c7581c1506575548681ac242ba136bbd8 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 7 Mar 2018 10:18:03 +0100 Subject: [PATCH 0135/2294] [FLINK-8889][tests] Do not override cluster config values This closes #5651. --- .../flink/test/util/MiniClusterResource.java | 4 +- .../apache/flink/test/util/TestBaseUtils.java | 43 +++++++++++++------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java index 66cbb9f9b77006..2f12bdc347e846 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java @@ -190,7 +190,9 @@ private void startFlip6MiniCluster() throws Exception { // we need to set this since a lot of test expect this because TestBaseUtils.startCluster() // enabled this by default - configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); + if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) { + configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); + } // set rest port to 0 to avoid clashes with concurrent MiniClusters configuration.setInteger(RestOptions.REST_PORT, 0); diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java index d7142f5fbfa18c..dd255fd6dc2fde 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/TestBaseUtils.java @@ -143,22 +143,41 @@ public static LocalFlinkMiniCluster startCluster( Configuration config, boolean singleActorSystem) throws Exception { - logDir = File.createTempFile("TestBaseUtils-logdir", null); - Assert.assertTrue("Unable to delete temp file", logDir.delete()); - Assert.assertTrue("Unable to create temp directory", logDir.mkdir()); - Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath()); - Files.createFile(new File(logDir, "jobmanager.out").toPath()); + if (!config.contains(WebOptions.LOG_PATH) || !config.containsKey(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY)) { + logDir = File.createTempFile("TestBaseUtils-logdir", null); + Assert.assertTrue("Unable to delete temp file", logDir.delete()); + Assert.assertTrue("Unable to create temp directory", logDir.mkdir()); + Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath()); + Files.createFile(new File(logDir, "jobmanager.out").toPath()); + + if (!config.contains(WebOptions.LOG_PATH)) { + config.setString(WebOptions.LOG_PATH, logFile.toString()); + } + + if (!config.containsKey(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY)) { + config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.toString()); + } + } + + if (!config.contains(WebOptions.PORT)) { + config.setInteger(WebOptions.PORT, 8081); + } - config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, TASK_MANAGER_MEMORY_SIZE); - config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); + if (!config.contains(AkkaOptions.ASK_TIMEOUT)) { + config.setString(AkkaOptions.ASK_TIMEOUT, DEFAULT_AKKA_ASK_TIMEOUT + "s"); + } - config.setString(AkkaOptions.ASK_TIMEOUT, DEFAULT_AKKA_ASK_TIMEOUT + "s"); - config.setString(AkkaOptions.STARTUP_TIMEOUT, DEFAULT_AKKA_STARTUP_TIMEOUT); + if (!config.contains(AkkaOptions.STARTUP_TIMEOUT)) { + config.setString(AkkaOptions.STARTUP_TIMEOUT, DEFAULT_AKKA_STARTUP_TIMEOUT); + } - config.setInteger(WebOptions.PORT, 8081); - config.setString(WebOptions.LOG_PATH, logFile.toString()); + if (!config.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) { + config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); + } - config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.toString()); + if (!config.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) { + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, TASK_MANAGER_MEMORY_SIZE); + } LocalFlinkMiniCluster cluster = new LocalFlinkMiniCluster(config, singleActorSystem); From b8dac871203e0a47adebee810678db740e3a92cc Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 5 Mar 2018 13:55:04 +0100 Subject: [PATCH 0136/2294] [FLINK-8860][flip6] stop SlotManager spamming logs for every TM heartbeat at log level 'info' This closes #5637. --- .../flink/runtime/resourcemanager/slotmanager/SlotManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java index ca3371945c8fd4..43e670eb9a42b4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java @@ -384,7 +384,7 @@ public boolean unregisterTaskManager(InstanceID instanceId) { public boolean reportSlotStatus(InstanceID instanceId, SlotReport slotReport) { checkInit(); - LOG.info("Received slot report from instance {}.", instanceId); + LOG.trace("Received slot report from instance {}.", instanceId); TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(instanceId); From b8eb6af99dd310eeec3497b44f252a0402068d6b Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 21 Feb 2018 15:30:16 +0100 Subject: [PATCH 0137/2294] [FLINK-8729][streaming] Refactor JSONGenerator to use jackson This closes #5554. --- flink-streaming-java/pom.xml | 6 --- .../streaming/api/graph/JSONGenerator.java | 54 +++++++++---------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/flink-streaming-java/pom.xml b/flink-streaming-java/pom.xml index 300a2926c96d7b..f8f2348ad65473 100644 --- a/flink-streaming-java/pom.xml +++ b/flink-streaming-java/pom.xml @@ -67,12 +67,6 @@ under the License. 3.5 - - org.apache.sling - org.apache.sling.commons.json - 2.0.6 - - diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java index a9bb0b696f1e71..263e0aabdf197e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/JSONGenerator.java @@ -20,9 +20,9 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.streaming.api.operators.StreamOperator; -import org.apache.sling.commons.json.JSONArray; -import org.apache.sling.commons.json.JSONException; -import org.apache.sling.commons.json.JSONObject; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.Collections; @@ -48,14 +48,15 @@ public class JSONGenerator { public static final String PARALLELISM = "parallelism"; private StreamGraph streamGraph; + private final ObjectMapper mapper = new ObjectMapper(); public JSONGenerator(StreamGraph streamGraph) { this.streamGraph = streamGraph; } - public String getJSON() throws JSONException { - JSONObject json = new JSONObject(); - JSONArray nodes = new JSONArray(); + public String getJSON() { + ObjectNode json = mapper.createObjectNode(); + ArrayNode nodes = mapper.createArrayNode(); json.put("nodes", nodes); List operatorIDs = new ArrayList(streamGraph.getVertexIDs()); Collections.sort(operatorIDs, new Comparator() { @@ -75,8 +76,8 @@ public int compare(Integer o1, Integer o2) { return json.toString(); } - private void visit(JSONArray jsonArray, List toVisit, - Map edgeRemapings) throws JSONException { + private void visit(ArrayNode jsonArray, List toVisit, + Map edgeRemapings) { Integer vertexID = toVisit.get(0); StreamNode vertex = streamGraph.getStreamNode(vertexID); @@ -84,11 +85,11 @@ private void visit(JSONArray jsonArray, List toVisit, if (streamGraph.getSourceIDs().contains(vertexID) || Collections.disjoint(vertex.getInEdges(), toVisit)) { - JSONObject node = new JSONObject(); + ObjectNode node = mapper.createObjectNode(); decorateNode(vertexID, node); if (!streamGraph.getSourceIDs().contains(vertexID)) { - JSONArray inputs = new JSONArray(); + ArrayNode inputs = mapper.createArrayNode(); node.put(PREDECESSORS, inputs); for (StreamEdge inEdge : vertex.getInEdges()) { @@ -99,7 +100,7 @@ private void visit(JSONArray jsonArray, List toVisit, decorateEdge(inputs, inEdge, mappedID); } } - jsonArray.put(node); + jsonArray.add(node); toVisit.remove(vertexID); } else { Integer iterationHead = -1; @@ -111,18 +112,18 @@ private void visit(JSONArray jsonArray, List toVisit, } } - JSONObject obj = new JSONObject(); - JSONArray iterationSteps = new JSONArray(); + ObjectNode obj = mapper.createObjectNode(); + ArrayNode iterationSteps = mapper.createArrayNode(); obj.put(STEPS, iterationSteps); obj.put(ID, iterationHead); obj.put(PACT, "IterativeDataStream"); obj.put(PARALLELISM, streamGraph.getStreamNode(iterationHead).getParallelism()); obj.put(CONTENTS, "Stream Iteration"); - JSONArray iterationInputs = new JSONArray(); + ArrayNode iterationInputs = mapper.createArrayNode(); obj.put(PREDECESSORS, iterationInputs); toVisit.remove(iterationHead); visitIteration(iterationSteps, toVisit, iterationHead, edgeRemapings, iterationInputs); - jsonArray.put(obj); + jsonArray.add(obj); } if (!toVisit.isEmpty()) { @@ -130,8 +131,8 @@ private void visit(JSONArray jsonArray, List toVisit, } } - private void visitIteration(JSONArray jsonArray, List toVisit, int headId, - Map edgeRemapings, JSONArray iterationInEdges) throws JSONException { + private void visitIteration(ArrayNode jsonArray, List toVisit, int headId, + Map edgeRemapings, ArrayNode iterationInEdges) { Integer vertexID = toVisit.get(0); StreamNode vertex = streamGraph.getStreamNode(vertexID); @@ -139,10 +140,10 @@ private void visitIteration(JSONArray jsonArray, List toVisit, int head // Ignoring head and tail to avoid redundancy if (!streamGraph.vertexIDtoLoopTimeout.containsKey(vertexID)) { - JSONObject obj = new JSONObject(); - jsonArray.put(obj); + ObjectNode obj = mapper.createObjectNode(); + jsonArray.add(obj); decorateNode(vertexID, obj); - JSONArray inEdges = new JSONArray(); + ArrayNode inEdges = mapper.createArrayNode(); obj.put(PREDECESSORS, inEdges); for (StreamEdge inEdge : vertex.getInEdges()) { @@ -161,16 +162,15 @@ private void visitIteration(JSONArray jsonArray, List toVisit, int head } - private void decorateEdge(JSONArray inputArray, StreamEdge inEdge, int mappedInputID) - throws JSONException { - JSONObject input = new JSONObject(); - inputArray.put(input); + private void decorateEdge(ArrayNode inputArray, StreamEdge inEdge, int mappedInputID) { + ObjectNode input = mapper.createObjectNode(); + inputArray.add(input); input.put(ID, mappedInputID); - input.put(SHIP_STRATEGY, inEdge.getPartitioner()); - input.put(SIDE, (inputArray.length() == 0) ? "first" : "second"); + input.put(SHIP_STRATEGY, inEdge.getPartitioner().toString()); + input.put(SIDE, (inputArray.size() == 0) ? "first" : "second"); } - private void decorateNode(Integer vertexID, JSONObject node) throws JSONException { + private void decorateNode(Integer vertexID, ObjectNode node) { StreamNode vertex = streamGraph.getStreamNode(vertexID); From 82dec053aece27e4c658218dee0cbb1e83c936b5 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Fri, 9 Mar 2018 12:03:41 +0100 Subject: [PATCH 0138/2294] [FLINK-8860] Change slot-report message to DEBUG --- .../flink/runtime/resourcemanager/slotmanager/SlotManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java index 43e670eb9a42b4..120e1aa1a5ac3e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java @@ -384,7 +384,7 @@ public boolean unregisterTaskManager(InstanceID instanceId) { public boolean reportSlotStatus(InstanceID instanceId, SlotReport slotReport) { checkInit(); - LOG.trace("Received slot report from instance {}.", instanceId); + LOG.debug("Received slot report from instance {}.", instanceId); TaskManagerRegistration taskManagerRegistration = taskManagerRegistrations.get(instanceId); From 5eae8bd423256fb372a57151e482c501c955c008 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Thu, 8 Mar 2018 12:22:32 +0100 Subject: [PATCH 0139/2294] [FLINK-8896] [kafka08] remove all cancel MARKERs before trying to find partition leaders This guards us against #cancel() being called multiple times and then trying to look up an invalid topic/partition pair. This closes #5661 --- .../streaming/connectors/kafka/internals/Kafka08Fetcher.java | 3 ++- .../streaming/connectors/kafka/FlinkKafkaConsumerBase.java | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java index a2edb72d2dac79..96540412f0cec9 100644 --- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java +++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java @@ -189,7 +189,8 @@ public void runFetchLoop() throws Exception { // special marker into the queue List> partitionsToAssign = unassignedPartitionsQueue.getBatchBlocking(5000); - partitionsToAssign.remove(MARKER); + // note: if there are more markers, remove them all + partitionsToAssign.removeIf(MARKER::equals); if (!partitionsToAssign.isEmpty()) { LOG.info("Assigning {} partitions to broker threads", partitionsToAssign.size()); diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java index e19772a1f801c9..82ac2c37b8faa6 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java @@ -709,7 +709,10 @@ public void run() { discoveryLoopErrorRef.set(e); } finally { // calling cancel will also let the fetcher loop escape - cancel(); + // (if not running, cancel() was already called) + if (running) { + cancel(); + } } } }); From 69b3299623e1dec8c67d384e57d67343e2f4c215 Mon Sep 17 00:00:00 2001 From: Andreas Fink Date: Mon, 5 Mar 2018 18:26:57 +0100 Subject: [PATCH 0140/2294] [FLINK-8091] [scripts] Support running historyserver in foreground This closes #5642 --- docs/monitoring/historyserver.md | 2 +- flink-dist/src/main/flink-bin/bin/flink-console.sh | 6 +++++- flink-dist/src/main/flink-bin/bin/historyserver.sh | 10 +++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/monitoring/historyserver.md b/docs/monitoring/historyserver.md index dfbbc8edbc1d4c..9c68e6525083d0 100644 --- a/docs/monitoring/historyserver.md +++ b/docs/monitoring/historyserver.md @@ -37,7 +37,7 @@ After you have configured the HistoryServer *and* JobManager, you start and stop ```sh # Start or stop the HistoryServer -bin/historyserver.sh (start|stop) +bin/historyserver.sh (start|start-foreground|stop) ``` By default, this server binds to `localhost` and listens at port `8082`. diff --git a/flink-dist/src/main/flink-bin/bin/flink-console.sh b/flink-dist/src/main/flink-bin/bin/flink-console.sh index 574376c6ede6d8..3ccbbd0b99d5b8 100644 --- a/flink-dist/src/main/flink-bin/bin/flink-console.sh +++ b/flink-dist/src/main/flink-bin/bin/flink-console.sh @@ -19,7 +19,7 @@ # Start a Flink service as a console application. Must be stopped with Ctrl-C # or with SIGTERM by kill or the controlling process. -USAGE="Usage: flink-console.sh (jobmanager|taskmanager|zookeeper) [args]" +USAGE="Usage: flink-console.sh (jobmanager|taskmanager|historyserver|zookeeper) [args]" SERVICE=$1 ARGS=("${@:2}") # get remaining arguments as array @@ -42,6 +42,10 @@ case $SERVICE in CLASS_TO_RUN=org.apache.flink.runtime.taskexecutor.TaskManagerRunner ;; + (historyserver) + CLASS_TO_RUN=org.apache.flink.runtime.webmonitor.history.HistoryServer + ;; + (zookeeper) CLASS_TO_RUN=org.apache.flink.runtime.zookeeper.FlinkZooKeeperQuorumPeer ;; diff --git a/flink-dist/src/main/flink-bin/bin/historyserver.sh b/flink-dist/src/main/flink-bin/bin/historyserver.sh index adc966004cc35b..150ab9469aeedf 100644 --- a/flink-dist/src/main/flink-bin/bin/historyserver.sh +++ b/flink-dist/src/main/flink-bin/bin/historyserver.sh @@ -18,7 +18,7 @@ ################################################################################ # Start/stop a Flink HistoryServer -USAGE="Usage: historyserver.sh (start|stop)" +USAGE="Usage: historyserver.sh (start|start-foreground|stop)" STARTSTOP=$1 @@ -27,8 +27,12 @@ bin=`cd "$bin"; pwd` . "$bin"/config.sh -if [[ $STARTSTOP == "start" ]]; then +if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then args=("--configDir" "${FLINK_CONF_DIR}") fi -"${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP historyserver "${args[@]}" +if [[ $STARTSTOP == "start-foreground" ]]; then + exec "${FLINK_BIN_DIR}"/flink-console.sh historyserver "${args[@]}" +else + "${FLINK_BIN_DIR}"/flink-daemon.sh $STARTSTOP historyserver "${args[@]}" +fi From c531486288caf5241cdf7f0f00f087f3ce82239f Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Thu, 8 Mar 2018 11:51:38 +0100 Subject: [PATCH 0141/2294] [FLINK-8854] [table] Fix schema mapping with time attributes This closes #5662. --- .../KafkaJsonTableSourceFactoryTestBase.java | 20 +++++-- .../table/descriptors/SchemaValidator.scala | 19 +++++- .../table/sources/definedTimeAttributes.scala | 19 +++++- .../sources/tsextractors/ExistingField.scala | 10 +++- .../BoundedOutOfOrderTimestamps.scala | 10 ++++ .../descriptors/SchemaValidatorTest.scala | 58 ++++++++++++++++++- 6 files changed, 120 insertions(+), 16 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java index 2b081a9f9157d7..583b71dd4e88ad 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java @@ -25,10 +25,13 @@ import org.apache.flink.table.descriptors.FormatDescriptor; import org.apache.flink.table.descriptors.Json; import org.apache.flink.table.descriptors.Kafka; +import org.apache.flink.table.descriptors.Rowtime; import org.apache.flink.table.descriptors.Schema; import org.apache.flink.table.descriptors.TestTableSourceDescriptor; import org.apache.flink.table.sources.TableSource; import org.apache.flink.table.sources.TableSourceFactoryService; +import org.apache.flink.table.sources.tsextractors.ExistingField; +import org.apache.flink.table.sources.wmstrategies.PreserveWatermarks; import org.junit.Test; @@ -55,9 +58,11 @@ public abstract class KafkaJsonTableSourceFactoryTestBase { " 'type': 'integer'" + " }," + " 'time': {" + - " 'description': 'Age in years'," + - " 'type': 'number'" + - " }" + " }," + + " 'description': 'row time'," + + " 'type': 'string'," + + " 'format': 'date-time'" + + " }" + + " }," + " 'required': ['name', 'count', 'time']" + "}"; @@ -89,9 +94,10 @@ private void testTableSource(FormatDescriptor format) { // construct table source using a builder final Map tableJsonMapping = new HashMap<>(); + tableJsonMapping.put("name", "name"); tableJsonMapping.put("fruit-name", "name"); tableJsonMapping.put("count", "count"); - tableJsonMapping.put("event-time", "time"); + tableJsonMapping.put("time", "time"); final Properties props = new Properties(); props.put("group.id", "test-group"); @@ -112,10 +118,11 @@ private void testTableSource(FormatDescriptor format) { TableSchema.builder() .field("fruit-name", Types.STRING) .field("count", Types.BIG_INT) - .field("event-time", Types.BIG_DEC) + .field("event-time", Types.SQL_TIMESTAMP) .field("proc-time", Types.SQL_TIMESTAMP) .build()) .withProctimeAttribute("proc-time") + .withRowtimeAttribute("event-time", new ExistingField("time"), PreserveWatermarks.INSTANCE()) .build(); // construct table source using descriptors and table source factory @@ -135,7 +142,8 @@ private void testTableSource(FormatDescriptor format) { new Schema() .field("fruit-name", Types.STRING).from("name") .field("count", Types.BIG_INT) // no from so it must match with the input - .field("event-time", Types.BIG_DEC).from("time") + .field("event-time", Types.SQL_TIMESTAMP).rowtime( + new Rowtime().timestampsFromField("time").watermarksFromSource()) .field("proc-time", Types.SQL_TIMESTAMP).proctime()); final TableSource factorySource = TableSourceFactoryService.findAndCreateTableSource(testDesc); diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala index 0a2391175bf019..9cb3258d68abaa 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala @@ -23,7 +23,7 @@ import java.util.Optional import org.apache.flink.table.api.{TableSchema, ValidationException} import org.apache.flink.table.descriptors.DescriptorProperties.{toJava, toScala} -import org.apache.flink.table.descriptors.RowtimeValidator.{ROWTIME, ROWTIME_TIMESTAMPS_TYPE} +import org.apache.flink.table.descriptors.RowtimeValidator.{ROWTIME, ROWTIME_TIMESTAMPS_FROM, ROWTIME_TIMESTAMPS_TYPE, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD} import org.apache.flink.table.descriptors.SchemaValidator._ import org.apache.flink.table.sources.RowtimeAttributeDescriptor @@ -148,6 +148,13 @@ object SchemaValidator { val schema = properties.getTableSchema(SCHEMA) + // add all source fields first because rowtime might reference one of them + toScala(sourceSchema).map(_.getColumnNames).foreach { names => + names.foreach { name => + mapping.put(name, name) + } + } + // add all schema fields first for implicit mappings schema.getColumnNames.foreach { name => mapping.put(name, name) @@ -198,14 +205,20 @@ object SchemaValidator { val isProctime = properties .getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME") .orElse(false) - val isRowtime = properties - .containsKey(s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_TYPE") + val tsType = s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_TYPE" + val isRowtime = properties.containsKey(tsType) if (!isProctime && !isRowtime) { // check for a aliasing val fieldName = properties.getOptionalString(s"$SCHEMA.$i.$SCHEMA_FROM") .orElse(n) builder.field(fieldName, t) } + // only use the rowtime attribute if it references a field + else if (isRowtime && + properties.getString(tsType) == ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD) { + val field = properties.getString(s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_FROM") + builder.field(field, t) + } } builder.build() diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/definedTimeAttributes.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/definedTimeAttributes.scala index f09baa35b73f15..73b76a58584eb4 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/definedTimeAttributes.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/definedTimeAttributes.scala @@ -19,6 +19,7 @@ package org.apache.flink.table.sources import java.util +import java.util.Objects import org.apache.flink.table.api.TableSchema import org.apache.flink.table.api.Types @@ -65,9 +66,9 @@ trait DefinedRowtimeAttributes { * @param watermarkStrategy The watermark strategy associated with the attribute. */ class RowtimeAttributeDescriptor( - attributeName: String, - timestampExtractor: TimestampExtractor, - watermarkStrategy: WatermarkStrategy) { + val attributeName: String, + val timestampExtractor: TimestampExtractor, + val watermarkStrategy: WatermarkStrategy) { /** Returns the name of the rowtime attribute. */ def getAttributeName: String = attributeName @@ -77,4 +78,16 @@ class RowtimeAttributeDescriptor( /** Returns the [[WatermarkStrategy]] for the attribute. */ def getWatermarkStrategy: WatermarkStrategy = watermarkStrategy + + override def equals(other: Any): Boolean = other match { + case that: RowtimeAttributeDescriptor => + Objects.equals(attributeName, that.attributeName) && + Objects.equals(timestampExtractor, that.timestampExtractor) && + Objects.equals(watermarkStrategy, that.watermarkStrategy) + case _ => false + } + + override def hashCode(): Int = { + Objects.hash(attributeName, timestampExtractor, watermarkStrategy) + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala index 12cd564395cea1..866029bcd3859c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala @@ -27,7 +27,7 @@ import org.apache.flink.table.expressions.{Cast, Expression, ResolvedFieldRefere * * @param field The field to convert into a rowtime attribute. */ -class ExistingField(field: String) extends TimestampExtractor { +class ExistingField(val field: String) extends TimestampExtractor { override def getArgumentFields: Array[String] = Array(field) @@ -65,4 +65,12 @@ class ExistingField(field: String) extends TimestampExtractor { } } + override def equals(other: Any): Boolean = other match { + case that: ExistingField => field == that.field + case _ => false + } + + override def hashCode(): Int = { + field.hashCode + } } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/wmstrategies/BoundedOutOfOrderTimestamps.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/wmstrategies/BoundedOutOfOrderTimestamps.scala index 8f7c23560187bd..4718bad57d4550 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/wmstrategies/BoundedOutOfOrderTimestamps.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/wmstrategies/BoundedOutOfOrderTimestamps.scala @@ -38,4 +38,14 @@ final class BoundedOutOfOrderTimestamps(val delay: Long) extends PeriodicWaterma } override def getWatermark: Watermark = new Watermark(maxTimestamp - delay) + + override def equals(other: Any): Boolean = other match { + case that: BoundedOutOfOrderTimestamps => + delay == that.delay + case _ => false + } + + override def hashCode(): Int = { + delay.hashCode() + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala index ba05dfff2074db..bf7b84b8e769d3 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala @@ -21,7 +21,7 @@ package org.apache.flink.table.descriptors import java.util.Optional import org.apache.flink.table.api.{TableSchema, Types} -import org.apache.flink.table.sources.tsextractors.StreamRecordTimestamp +import org.apache.flink.table.sources.tsextractors.{ExistingField, StreamRecordTimestamp} import org.apache.flink.table.sources.wmstrategies.PreserveWatermarks import org.junit.Assert.{assertEquals, assertTrue} import org.junit.Test @@ -34,7 +34,7 @@ import scala.collection.JavaConverters._ class SchemaValidatorTest { @Test - def testSchema(): Unit = { + def testSchemaWithRowtimeFromSource(): Unit = { val desc1 = Schema() .field("otherField", Types.STRING).from("csvField") .field("abcField", Types.STRING) @@ -60,7 +60,11 @@ class SchemaValidatorTest { assertTrue(rowtime.getWatermarkStrategy.isInstanceOf[PreserveWatermarks]) // test field mapping - val expectedMapping = Map("otherField" -> "csvField", "abcField" -> "abcField").asJava + val expectedMapping = Map( + "otherField" -> "csvField", + "csvField" -> "csvField", + "abcField" -> "abcField", + "myField" -> "myField").asJava assertEquals( expectedMapping, SchemaValidator.deriveFieldMapping(props, Optional.of(inputSchema))) @@ -73,4 +77,52 @@ class SchemaValidatorTest { .build() assertEquals(expectedFormatSchema, formatSchema) } + + @Test + def testSchemaWithRowtimeFromField(): Unit = { + val desc1 = Schema() + .field("otherField", Types.STRING).from("csvField") + .field("abcField", Types.STRING) + .field("p", Types.SQL_TIMESTAMP).proctime() + .field("r", Types.SQL_TIMESTAMP).rowtime( + Rowtime().timestampsFromField("myTime").watermarksFromSource()) + val props = new DescriptorProperties() + desc1.addProperties(props) + + val inputSchema = TableSchema.builder() + .field("csvField", Types.STRING) + .field("abcField", Types.STRING) + .field("myField", Types.BOOLEAN) + .field("myTime", Types.SQL_TIMESTAMP) + .build() + + // test proctime + assertEquals(Optional.of("p"), SchemaValidator.deriveProctimeAttribute(props)) + + // test rowtime + val rowtime = SchemaValidator.deriveRowtimeAttributes(props).get(0) + assertEquals("r", rowtime.getAttributeName) + assertTrue(rowtime.getTimestampExtractor.isInstanceOf[ExistingField]) + assertTrue(rowtime.getWatermarkStrategy.isInstanceOf[PreserveWatermarks]) + + // test field mapping + val expectedMapping = Map( + "otherField" -> "csvField", + "csvField" -> "csvField", + "abcField" -> "abcField", + "myField" -> "myField", + "myTime" -> "myTime").asJava + assertEquals( + expectedMapping, + SchemaValidator.deriveFieldMapping(props, Optional.of(inputSchema))) + + // test field format + val formatSchema = SchemaValidator.deriveFormatFields(props) + val expectedFormatSchema = TableSchema.builder() + .field("csvField", Types.STRING) // aliased + .field("abcField", Types.STRING) + .field("myTime", Types.SQL_TIMESTAMP) + .build() + assertEquals(expectedFormatSchema, formatSchema) + } } From 6abc8a9333ffe72f65593d4859f1f10662068a1f Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Thu, 22 Feb 2018 14:11:13 +0100 Subject: [PATCH 0142/2294] [hotfix] [network] Rename RecordWriter#closeBufferConsumer() to closeBufferBuilder() --- .../flink/runtime/io/network/api/writer/RecordWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java index 4ec28631e07f4c..c35c7f3e8aa90a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java @@ -175,7 +175,7 @@ public void flushAll() { public void clearBuffers() { for (int targetChannel = 0; targetChannel < numChannels; targetChannel++) { RecordSerializer serializer = serializers[targetChannel]; - closeBufferConsumer(targetChannel); + closeBufferBuilder(targetChannel); serializer.clear(); } } @@ -213,7 +213,7 @@ private BufferBuilder requestNewBufferBuilder(int targetChannel) throws IOExcept return bufferBuilder; } - private void closeBufferConsumer(int targetChannel) { + private void closeBufferBuilder(int targetChannel) { if (bufferBuilders[targetChannel].isPresent()) { bufferBuilders[targetChannel].get().finish(); bufferBuilders[targetChannel] = Optional.empty(); From 30eb8cd026a2e00397cea645814b384f5774366d Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Thu, 22 Feb 2018 14:17:06 +0100 Subject: [PATCH 0143/2294] [hotfix] [network] Various minor improvements --- .../netty/CreditBasedSequenceNumberingViewReader.java | 3 ++- .../runtime/io/network/netty/PartitionRequestQueue.java | 5 ++++- .../io/network/netty/SequenceNumberingViewReader.java | 1 + .../runtime/io/network/partition/SubpartitionTestBase.java | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java index 9acbbacf2735ba..8fc7ef4842b489 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java @@ -114,7 +114,7 @@ public boolean isRegisteredAsAvailable() { */ @Override public boolean isAvailable() { - // BEWARE: this must be in sync with #isAvailable()! + // BEWARE: this must be in sync with #isAvailable(BufferAndBacklog)! return hasBuffersAvailable() && (numCreditsAvailable > 0 || subpartitionView.nextBufferIsEvent()); } @@ -130,6 +130,7 @@ public boolean isAvailable() { * current buffer and backlog including information about the next buffer */ private boolean isAvailable(BufferAndBacklog bufferAndBacklog) { + // BEWARE: this must be in sync with #isAvailable()! return bufferAndBacklog.isMoreAvailable() && (numCreditsAvailable > 0 || bufferAndBacklog.nextBufferIsEvent()); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java index d63a88e718276e..8c05b8208f90f3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueue.java @@ -38,6 +38,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.ArrayDeque; import java.util.Set; @@ -52,7 +54,7 @@ */ class PartitionRequestQueue extends ChannelInboundHandlerAdapter { - private final Logger LOG = LoggerFactory.getLogger(PartitionRequestQueue.class); + private static final Logger LOG = LoggerFactory.getLogger(PartitionRequestQueue.class); private final ChannelFutureListener writeListener = new WriteAndFlushNextMessageIfPossibleListener(); @@ -278,6 +280,7 @@ private void registerAvailableReader(NetworkSequenceViewReader reader) { reader.setRegisteredAsAvailable(true); } + @Nullable private NetworkSequenceViewReader pollAvailableReader() { NetworkSequenceViewReader reader = availableReaders.poll(); if (reader != null) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java index 6a83af13837827..054046f086a75b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/SequenceNumberingViewReader.java @@ -148,6 +148,7 @@ public String toString() { "requestLock=" + requestLock + ", receiverId=" + receiverId + ", sequenceNumber=" + sequenceNumber + + ", isRegisteredAsAvailable=" + isRegisteredAvailable + '}'; } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java index 215726b3b5ad1f..a3f18f6c18cead 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java @@ -134,7 +134,7 @@ private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) assertTrue(view.isReleased()); } - protected void assertNextBuffer( + static void assertNextBuffer( ResultSubpartitionView readView, int expectedReadableBufferSize, boolean expectedIsMoreAvailable, From 496239806e68535c9c8291e320f8886d3a1b8709 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 23 Feb 2018 10:35:41 +0100 Subject: [PATCH 0144/2294] [hotfix] [network] [tests] Make AwaitableBufferAvailablityListener thread-safe This is called asynchronously by the spill writer and thus may need synchronization on incrementing the counter but definately had visibility issues with the counter. Using an AtomicLong fixes that. --- .../AwaitableBufferAvailablityListener.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java index 2b6b834c0b5604..6cf9d64f1b3bcb 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/AwaitableBufferAvailablityListener.java @@ -18,29 +18,31 @@ package org.apache.flink.runtime.io.network.partition; +import java.util.concurrent.atomic.AtomicLong; + /** * Test implementation of {@link BufferAvailabilityListener}. */ class AwaitableBufferAvailablityListener implements BufferAvailabilityListener { - private long numNotifications; + private final AtomicLong numNotifications = new AtomicLong(); @Override public void notifyDataAvailable() { - ++numNotifications; + numNotifications.getAndIncrement(); } public long getNumNotifications() { - return numNotifications; + return numNotifications.get(); } public void resetNotificationCounters() { - numNotifications = 0; + numNotifications.set(0L); } void awaitNotifications(long awaitedNumNotifications, long timeoutMillis) throws InterruptedException { long deadline = System.currentTimeMillis() + timeoutMillis; - while (numNotifications < awaitedNumNotifications && System.currentTimeMillis() < deadline) { + while (numNotifications.get() < awaitedNumNotifications && System.currentTimeMillis() < deadline) { Thread.sleep(1); } } From 18b75e32bb8f4f155f729574b2d377459104471e Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 23 Feb 2018 10:19:58 +0100 Subject: [PATCH 0145/2294] [FLINK-8755] [network] Fix SpilledSubpartitionView relying on the backlog for determining whether more data is available Fix SpilledSubpartitionView#getNextBuffer() to not only rely on the backlog: instead it is sufficient to also return true if the next buffer is an event since either there is a real buffer enqueued (reflected by the backlog) or at least one event. --- .../runtime/io/network/partition/SpilledSubpartitionView.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java index 378b0867d6feec..2a6a71f05d667a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpilledSubpartitionView.java @@ -148,7 +148,7 @@ public BufferAndBacklog getNextBuffer() throws IOException, InterruptedException } int newBacklog = parent.decreaseBuffersInBacklog(current); - return new BufferAndBacklog(current, newBacklog > 0, newBacklog, nextBufferIsEvent); + return new BufferAndBacklog(current, newBacklog > 0 || nextBufferIsEvent, newBacklog, nextBufferIsEvent); } @Nullable From 112c54fb07e2ffffa3f33322ba99a9d59c1a8dbc Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Mon, 26 Feb 2018 16:27:44 +0100 Subject: [PATCH 0146/2294] [FLINK-8786] [network] Fix SpillableSubpartitionView#getNextBuffer returning wrong isMoreAvailable when processing last in-memory buffer When processing the last in-memory buffer in SpillableSubpartitionView#getNextBuffer while the rest of the buffers are spilled, need to rely on the spilled view's isAvailable instead of always setting the isMoreAvailable flag of the returned BufferAndBacklog to false. --- .../runtime/io/network/partition/SpillableSubpartitionView.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java index 3c73e43d8cb9ff..0f51bc8b03e37a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java @@ -177,7 +177,7 @@ public BufferAndBacklog getNextBuffer() throws IOException, InterruptedException SpilledSubpartitionView spilled = spilledView; if (spilled != null) { if (current != null) { - return new BufferAndBacklog(current, isMoreAvailable, newBacklog, spilled.nextBufferIsEvent()); + return new BufferAndBacklog(current, spilled.isAvailable(), newBacklog, spilled.nextBufferIsEvent()); } else { return spilled.getNextBuffer(); } From c19df9ff670c06aeb381339c244bbd22fe13fd4d Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 23 Feb 2018 12:13:20 +0100 Subject: [PATCH 0147/2294] [FLINK-8755] [FLINK-8786] [network] Add and improve subpartition tests + also improve the subpartition tests in general to reduce some duplication This closes #5581 --- .../partition/SpillableSubpartitionView.java | 2 +- .../partition/PipelinedSubpartitionTest.java | 11 +- .../partition/SpillableSubpartitionTest.java | 130 ++++++------------ .../partition/SubpartitionTestBase.java | 78 ++++++++++- 4 files changed, 121 insertions(+), 100 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java index 0f51bc8b03e37a..65790d79df28d9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionView.java @@ -167,7 +167,7 @@ public BufferAndBacklog getNextBuffer() throws IOException, InterruptedException parent.updateStatistics(current); // if we are spilled (but still process a non-spilled nextBuffer), we don't know the - // state of nextBufferIsEvent... + // state of nextBufferIsEvent or whether more buffers are available if (spilledView == null) { return new BufferAndBacklog(current, isMoreAvailable, newBacklog, nextBufferIsEvent); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java index ee678abc4ccc4c..bc66c9d292dc7e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java @@ -135,7 +135,8 @@ public void testAddNonEmptyNotFinishedBuffer() throws Exception { bufferBuilder.appendAndCommit(ByteBuffer.allocate(1024)); subpartition.add(bufferBuilder.createBufferConsumer()); - assertNextBuffer(readView, 1024, false, 1); + // note that since the buffer builder is not finished, there is still a retained instance! + assertNextBuffer(readView, 1024, false, 1, false, false); assertEquals(1, subpartition.getBuffersInBacklog()); } finally { readView.releaseAllResources(); @@ -157,7 +158,7 @@ public void testUnfinishedBufferBehindFinished() throws Exception { subpartition.add(createFilledBufferConsumer(1025)); // finished subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished - assertNextBuffer(readView, 1025, false, 1); + assertNextBuffer(readView, 1025, false, 1, false, true); } finally { subpartition.release(); } @@ -178,8 +179,8 @@ public void testFlushWithUnfinishedBufferBehindFinished() throws Exception { subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished subpartition.flush(); - assertNextBuffer(readView, 1025, true, 1); - assertNextBuffer(readView, 1024, false, 1); + assertNextBuffer(readView, 1025, true, 1, false, true); + assertNextBuffer(readView, 1024, false, 1, false, false); } finally { subpartition.release(); } @@ -208,7 +209,7 @@ public void testMultipleEmptyBuffers() throws Exception { subpartition.add(createFilledBufferConsumer(1024)); assertEquals(2, availablityListener.getNumNotifications()); - assertNextBuffer(readView, 1024, false, 0); + assertNextBuffer(readView, 1024, false, 0, false, true); } finally { readView.releaseAllResources(); subpartition.release(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java index e41a85c5207b44..840669e7c3fccc 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java @@ -24,13 +24,13 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager; import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync; import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsyncWithNoOpBufferFileWriter; +import org.apache.flink.runtime.io.network.api.CancelCheckpointMarker; import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent; import org.apache.flink.runtime.io.network.api.serialization.EventSerializer; +import org.apache.flink.runtime.io.network.buffer.Buffer; import org.apache.flink.runtime.io.network.buffer.BufferBuilder; -import org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils; import org.apache.flink.runtime.io.network.buffer.BufferConsumer; import org.apache.flink.runtime.io.network.buffer.BufferProvider; -import org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog; import org.junit.AfterClass; import org.junit.Assert; @@ -52,7 +52,6 @@ import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -190,10 +189,13 @@ public void testConsumeSpilledPartition() throws Exception { SpillableSubpartition partition = createSubpartition(); BufferConsumer bufferConsumer = createFilledBufferConsumer(BUFFER_DATA_SIZE, BUFFER_DATA_SIZE); + BufferConsumer eventBufferConsumer = + EventSerializer.toBufferConsumer(new CancelCheckpointMarker(1)); + final int eventSize = eventBufferConsumer.getWrittenBytes(); partition.add(bufferConsumer.copy()); partition.add(bufferConsumer.copy()); - partition.add(BufferBuilderTestUtils.createEventBufferConsumer(BUFFER_DATA_SIZE)); + partition.add(eventBufferConsumer); partition.add(bufferConsumer); assertEquals(4, partition.getTotalNumberOfBuffers()); @@ -207,13 +209,13 @@ public void testConsumeSpilledPartition() throws Exception { // still same statistics assertEquals(4, partition.getTotalNumberOfBuffers()); assertEquals(3, partition.getBuffersInBacklog()); - assertEquals(BUFFER_DATA_SIZE * 4, partition.getTotalNumberOfBytes()); + assertEquals(BUFFER_DATA_SIZE * 3 + eventSize, partition.getTotalNumberOfBytes()); partition.finish(); // + one EndOfPartitionEvent assertEquals(5, partition.getTotalNumberOfBuffers()); assertEquals(3, partition.getBuffersInBacklog()); - assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); + assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); AwaitableBufferAvailablityListener listener = new AwaitableBufferAvailablityListener(); SpilledSubpartitionView reader = (SpilledSubpartitionView) partition.createReadView(listener); @@ -221,59 +223,24 @@ public void testConsumeSpilledPartition() throws Exception { assertEquals(1, listener.getNumNotifications()); assertFalse(reader.nextBufferIsEvent()); // buffer - BufferAndBacklog read = reader.getNextBuffer(); - assertNotNull(read); - assertTrue(read.buffer().isBuffer()); + assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 2, false, true); assertEquals(2, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertFalse(read.buffer().isRecycled()); - read.buffer().recycleBuffer(); - assertTrue(read.buffer().isRecycled()); - assertFalse(read.nextBufferIsEvent()); assertFalse(reader.nextBufferIsEvent()); // buffer - read = reader.getNextBuffer(); - assertNotNull(read); - assertTrue(read.buffer().isBuffer()); + assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 1, true, true); assertEquals(1, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertFalse(read.buffer().isRecycled()); - read.buffer().recycleBuffer(); - assertTrue(read.buffer().isRecycled()); - assertTrue(read.nextBufferIsEvent()); assertTrue(reader.nextBufferIsEvent()); // event - read = reader.getNextBuffer(); - assertNotNull(read); - assertFalse(read.buffer().isBuffer()); + assertNextEvent(reader, eventSize, CancelCheckpointMarker.class, true, 1, false, true); assertEquals(1, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - read.buffer().recycleBuffer(); - assertFalse(read.nextBufferIsEvent()); assertFalse(reader.nextBufferIsEvent()); // buffer - read = reader.getNextBuffer(); - assertNotNull(read); - assertTrue(read.buffer().isBuffer()); + assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 0, true, true); assertEquals(0, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertFalse(read.buffer().isRecycled()); - read.buffer().recycleBuffer(); - assertTrue(read.buffer().isRecycled()); - assertTrue(read.nextBufferIsEvent()); assertTrue(reader.nextBufferIsEvent()); // end of partition event - read = reader.getNextBuffer(); - assertNotNull(read); - assertFalse(read.buffer().isBuffer()); + assertNextEvent(reader, 4, EndOfPartitionEvent.class, false, 0, false, true); assertEquals(0, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertEquals(EndOfPartitionEvent.class, - EventSerializer.fromBuffer(read.buffer(), ClassLoader.getSystemClassLoader()).getClass()); - assertFalse(read.buffer().isRecycled()); - read.buffer().recycleBuffer(); - assertTrue(read.buffer().isRecycled()); - assertFalse(read.nextBufferIsEvent()); // finally check that the bufferConsumer has been freed after a successful (or failed) write final long deadline = System.currentTimeMillis() + 30_000L; // 30 secs @@ -292,10 +259,13 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception SpillableSubpartition partition = createSubpartition(); BufferConsumer bufferConsumer = createFilledBufferConsumer(BUFFER_DATA_SIZE, BUFFER_DATA_SIZE); + BufferConsumer eventBufferConsumer = + EventSerializer.toBufferConsumer(new CancelCheckpointMarker(1)); + final int eventSize = eventBufferConsumer.getWrittenBytes(); partition.add(bufferConsumer.copy()); partition.add(bufferConsumer.copy()); - partition.add(BufferBuilderTestUtils.createEventBufferConsumer(BUFFER_DATA_SIZE)); + partition.add(eventBufferConsumer); partition.add(bufferConsumer); partition.finish(); @@ -311,17 +281,12 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception assertFalse(bufferConsumer.isRecycled()); assertFalse(reader.nextBufferIsEvent()); - BufferAndBacklog read = reader.getNextBuffer(); // first buffer (non-spilled) - assertNotNull(read); - assertTrue(read.buffer().isBuffer()); + // first buffer (non-spilled) + assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 2, false, false); assertEquals(BUFFER_DATA_SIZE, partition.getTotalNumberOfBytes()); // only updated when getting/spilling the buffers assertEquals(2, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - read.buffer().recycleBuffer(); - assertTrue(read.isMoreAvailable()); assertEquals(1, listener.getNumNotifications()); // since isMoreAvailable is set to true, no need for notification assertFalse(bufferConsumer.isRecycled()); - assertFalse(read.nextBufferIsEvent()); // Spill now assertEquals(3, partition.releaseMemory()); @@ -330,59 +295,44 @@ public void testConsumeSpillablePartitionSpilledDuringConsume() throws Exception assertEquals(5, partition.getTotalNumberOfBuffers()); assertEquals(2, partition.getBuffersInBacklog()); // only updated when getting/spilling the buffers but without the nextBuffer (kept in memory) - assertEquals(BUFFER_DATA_SIZE * 3 + 4, partition.getTotalNumberOfBytes()); + assertEquals(BUFFER_DATA_SIZE * 2 + eventSize + 4, partition.getTotalNumberOfBytes()); + // wait for successfully spilling all buffers (before that we may not access any spilled buffer and cannot rely on isMoreAvailable!) listener.awaitNotifications(2, 30_000); // Spiller finished assertEquals(2, listener.getNumNotifications()); + // after consuming and releasing the next buffer, the bufferConsumer may be freed, + // depending on the timing of the last write operation + // -> retain once so that we can check below + Buffer buffer = bufferConsumer.build(); + buffer.retainBuffer(); + assertFalse(reader.nextBufferIsEvent()); // second buffer (retained in SpillableSubpartition#nextBuffer) - read = reader.getNextBuffer(); - assertNotNull(read); - assertTrue(read.buffer().isBuffer()); - assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // finally integrates the nextBuffer statistics + assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 1, true, false); + assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // finally integrates the nextBuffer statistics assertEquals(1, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - read.buffer().recycleBuffer(); - // now the bufferConsumer may be freed, depending on the timing of the write operation - // -> let's do this check at the end of the test (to save some time) - assertTrue(read.nextBufferIsEvent()); + + bufferConsumer.close(); // recycle the retained buffer from above (should be the last reference!) assertTrue(reader.nextBufferIsEvent()); // the event (spilled) - read = reader.getNextBuffer(); - assertNotNull(read); - assertFalse(read.buffer().isBuffer()); - assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling + assertNextEvent(reader, eventSize, CancelCheckpointMarker.class, true, 1, false, true); + assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // already updated during spilling assertEquals(1, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - read.buffer().recycleBuffer(); - assertFalse(read.nextBufferIsEvent()); assertFalse(reader.nextBufferIsEvent()); // last buffer (spilled) - read = reader.getNextBuffer(); - assertNotNull(read); - assertTrue(read.buffer().isBuffer()); - assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling + assertNextBuffer(reader, BUFFER_DATA_SIZE, true, 0, true, true); + assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // already updated during spilling assertEquals(0, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertFalse(read.buffer().isRecycled()); - read.buffer().recycleBuffer(); - assertTrue(read.buffer().isRecycled()); - assertTrue(read.nextBufferIsEvent()); + + buffer.recycleBuffer(); + assertTrue(buffer.isRecycled()); // End of partition assertTrue(reader.nextBufferIsEvent()); - read = reader.getNextBuffer(); - assertNotNull(read); - assertEquals(BUFFER_DATA_SIZE * 4 + 4, partition.getTotalNumberOfBytes()); // already updated during spilling + assertNextEvent(reader, 4, EndOfPartitionEvent.class, false, 0, false, true); + assertEquals(BUFFER_DATA_SIZE * 3 + eventSize + 4, partition.getTotalNumberOfBytes()); // already updated during spilling assertEquals(0, partition.getBuffersInBacklog()); - assertEquals(partition.getBuffersInBacklog(), read.buffersInBacklog()); - assertEquals(EndOfPartitionEvent.class, - EventSerializer.fromBuffer(read.buffer(), ClassLoader.getSystemClassLoader()).getClass()); - assertFalse(read.buffer().isRecycled()); - read.buffer().recycleBuffer(); - assertTrue(read.buffer().isRecycled()); - assertFalse(read.nextBufferIsEvent()); // finally check that the bufferConsumer has been freed after a successful (or failed) write final long deadline = System.currentTimeMillis() + 30_000L; // 30 secs diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java index a3f18f6c18cead..8c902157da977e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java @@ -18,19 +18,26 @@ package org.apache.flink.runtime.io.network.partition; +import org.apache.flink.runtime.event.AbstractEvent; +import org.apache.flink.runtime.io.network.api.serialization.EventSerializer; import org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils; import org.apache.flink.runtime.io.network.buffer.BufferConsumer; import org.apache.flink.util.TestLogger; import org.junit.Test; +import javax.annotation.Nullable; + import java.io.IOException; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.hamcrest.core.IsInstanceOf.instanceOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -138,11 +145,74 @@ static void assertNextBuffer( ResultSubpartitionView readView, int expectedReadableBufferSize, boolean expectedIsMoreAvailable, - int expectedBuffersInBacklog) throws IOException, InterruptedException { + int expectedBuffersInBacklog, + boolean expectedNextBufferIsEvent, + boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException { + assertNextBufferOrEvent( + readView, + expectedReadableBufferSize, + true, + null, + expectedIsMoreAvailable, + expectedBuffersInBacklog, + expectedNextBufferIsEvent, + expectedRecycledAfterRecycle); + } + + static void assertNextEvent( + ResultSubpartitionView readView, + int expectedReadableBufferSize, + Class expectedEventClass, + boolean expectedIsMoreAvailable, + int expectedBuffersInBacklog, + boolean expectedNextBufferIsEvent, + boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException { + assertNextBufferOrEvent( + readView, + expectedReadableBufferSize, + false, + expectedEventClass, + expectedIsMoreAvailable, + expectedBuffersInBacklog, + expectedNextBufferIsEvent, + expectedRecycledAfterRecycle); + } + + private static void assertNextBufferOrEvent( + ResultSubpartitionView readView, + int expectedReadableBufferSize, + boolean expectedIsBuffer, + @Nullable Class expectedEventClass, + boolean expectedIsMoreAvailable, + int expectedBuffersInBacklog, + boolean expectedNextBufferIsEvent, + boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException { + checkArgument(expectedEventClass == null || !expectedIsBuffer); + ResultSubpartition.BufferAndBacklog bufferAndBacklog = readView.getNextBuffer(); - assertEquals(expectedReadableBufferSize, bufferAndBacklog.buffer().readableBytes()); - assertEquals(expectedIsMoreAvailable, bufferAndBacklog.isMoreAvailable()); - assertEquals(expectedBuffersInBacklog, bufferAndBacklog.buffersInBacklog()); + assertNotNull(bufferAndBacklog); + try { + assertEquals("buffer size", expectedReadableBufferSize, + bufferAndBacklog.buffer().readableBytes()); + assertEquals("buffer or event", expectedIsBuffer, + bufferAndBacklog.buffer().isBuffer()); + if (expectedEventClass != null) { + assertThat(EventSerializer + .fromBuffer(bufferAndBacklog.buffer(), ClassLoader.getSystemClassLoader()), + instanceOf(expectedEventClass)); + } + assertEquals("more available", expectedIsMoreAvailable, + bufferAndBacklog.isMoreAvailable()); + assertEquals("more available", expectedIsMoreAvailable, readView.isAvailable()); + assertEquals("backlog", expectedBuffersInBacklog, bufferAndBacklog.buffersInBacklog()); + assertEquals("next is event", expectedNextBufferIsEvent, + bufferAndBacklog.nextBufferIsEvent()); + + assertFalse("not recycled", bufferAndBacklog.buffer().isRecycled()); + } finally { + bufferAndBacklog.buffer().recycleBuffer(); + } + assertEquals("recycled", expectedRecycledAfterRecycle, bufferAndBacklog.buffer().isRecycled()); } protected void assertNoNextBuffer(ResultSubpartitionView readView) throws IOException, InterruptedException { From 92133b76cb3de55a98a0e14e3a57188f102985c3 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 28 Feb 2018 09:42:38 +0100 Subject: [PATCH 0148/2294] [FLINK-8800][REST] Reduce logging of all requests to TRACE This closes #5594. --- .../java/org/apache/flink/runtime/rest/AbstractHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java index a259801d7a8fb1..cb50a4f0d2ede3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/AbstractHandler.java @@ -84,8 +84,8 @@ protected AbstractHandler( @Override protected void respondAsLeader(ChannelHandlerContext ctx, Routed routed, T gateway) throws Exception { - if (log.isDebugEnabled()) { - log.debug("Received request " + routed.request().getUri() + '.'); + if (log.isTraceEnabled()) { + log.trace("Received request " + routed.request().getUri() + '.'); } final HttpRequest httpRequest = routed.request(); From 870ff31a944da5eba2c228f4529a26b6ff1d6846 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 6 Mar 2018 10:45:25 +0100 Subject: [PATCH 0149/2294] [FLINK-8847][build] Always generate .class files for package-info.java This closes #5644. --- pom.xml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 40c0e2447d93d0..4e2340b9820d37 100644 --- a/pom.xml +++ b/pom.xml @@ -1094,9 +1094,13 @@ under the License. ${java.version} ${java.version} - - -Xlint:all + + + -Xlint:all + + -Xpkginfo:always + From 8e85e1aa9886a328f257db780a1800a1ef8759a5 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 6 Mar 2018 11:04:42 +0100 Subject: [PATCH 0150/2294] [hotfix][build] Enable incremental compilation --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 4e2340b9820d37..f8649186c130f8 100644 --- a/pom.xml +++ b/pom.xml @@ -1094,6 +1094,8 @@ under the License. ${java.version} ${java.version} + + false From 3913382bfeccdf44fadde33c9adeb7be01978b35 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 28 Feb 2018 15:06:59 +0100 Subject: [PATCH 0151/2294] Add our own Deadline implementation --- .../flink/api/common/time/Deadline.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/time/Deadline.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/time/Deadline.java b/flink-core/src/main/java/org/apache/flink/api/common/time/Deadline.java new file mode 100644 index 00000000000000..2db6579b10c622 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/api/common/time/Deadline.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.api.common.time; + +import org.apache.flink.annotation.Internal; + +import java.time.Duration; + +/** + * This class stores a deadline, as obtained via {@link #now()} or from {@link #plus(Duration)}. + */ +@Internal +public class Deadline { + private final long timeNanos; + + private Deadline(Duration time) { + this.timeNanos = time.toNanos(); + } + + public Deadline plus(Duration other) { + return new Deadline(Duration.ofNanos(timeNanos).plus(other)); + } + + /** + * Returns the time left between the deadline and now. The result is negative if the deadline + * has passed. + */ + public Duration timeLeft() { + return Duration.ofNanos(timeNanos).minus(Duration.ofNanos(System.nanoTime())); + } + + /** + * Returns whether there is any time left between the deadline and now. + */ + public boolean hasTimeLeft() { + return !isOverdue(); + } + + /** + * Determines whether the deadline is in the past, i.e. whether the time left is negative. + */ + public boolean isOverdue() { + return timeNanos - System.nanoTime() < 0; + } + + /** + * Constructs a {@link Deadline} that has now as the deadline. Use this and then extend via + * {@link #plus(Duration)} to specify a deadline in the future. + */ + public static Deadline now() { + return new Deadline(Duration.ofNanos(System.nanoTime())); + } +} From ccb78b0abadc30871ada17b9b9173cc806f78d43 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Tue, 27 Feb 2018 13:40:51 +0100 Subject: [PATCH 0152/2294] [FLINK-8758] Add FutureUtils.retrySuccessfulWithDelay() This retries getting a result until it matches a given predicate or until we run out of retries. --- .../flink/runtime/concurrent/FutureUtils.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java index da77bdc8e047f4..a2d0710e87977d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.concurrent; +import org.apache.flink.api.common.time.Deadline; import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.apache.flink.util.ExceptionUtils; @@ -223,6 +224,81 @@ private static void retryOperationWithDelay( } } + /** + * Retry the given operation with the given delay in between successful completions where the + * result does not match a given predicate. + * + * @param operation to retry + * @param retryDelay delay between retries + * @param deadline A deadline that specifies at what point we should stop retrying + * @param acceptancePredicate Predicate to test whether the result is acceptable + * @param scheduledExecutor executor to be used for the retry operation + * @param type of the result + * @return Future which retries the given operation a given amount of times and delays the retry + * in case the predicate isn't matched + */ + public static CompletableFuture retrySuccesfulWithDelay( + final Supplier> operation, + final Time retryDelay, + final Deadline deadline, + final Predicate acceptancePredicate, + final ScheduledExecutor scheduledExecutor) { + + final CompletableFuture resultFuture = new CompletableFuture<>(); + + retrySuccessfulOperationWithDelay( + resultFuture, + operation, + retryDelay, + deadline, + acceptancePredicate, + scheduledExecutor); + + return resultFuture; + } + + private static void retrySuccessfulOperationWithDelay( + final CompletableFuture resultFuture, + final Supplier> operation, + final Time retryDelay, + final Deadline deadline, + final Predicate acceptancePredicate, + final ScheduledExecutor scheduledExecutor) { + + if (!resultFuture.isDone()) { + final CompletableFuture operationResultFuture = operation.get(); + + operationResultFuture.whenComplete( + (t, throwable) -> { + if (throwable != null) { + if (throwable instanceof CancellationException) { + resultFuture.completeExceptionally(new RetryException("Operation future was cancelled.", throwable)); + } else { + resultFuture.completeExceptionally(throwable); + } + } else { + if (acceptancePredicate.test(t)) { + resultFuture.complete(t); + } else if (deadline.hasTimeLeft()) { + final ScheduledFuture scheduledFuture = scheduledExecutor.schedule( + () -> retrySuccessfulOperationWithDelay(resultFuture, operation, retryDelay, deadline, acceptancePredicate, scheduledExecutor), + retryDelay.toMilliseconds(), + TimeUnit.MILLISECONDS); + + resultFuture.whenComplete( + (innerT, innerThrowable) -> scheduledFuture.cancel(false)); + } else { + resultFuture.completeExceptionally( + new RetryException("Could not satisfy the predicate within the allowed time.")); + } + } + }); + + resultFuture.whenComplete( + (t, throwable) -> operationResultFuture.cancel(false)); + } + } + /** * Exception with which the returned future is completed if the {@link #retry(Supplier, int, Executor)} * operation fails. From 6732669a684de0b230046b8f4291e367e35d9477 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Tue, 27 Feb 2018 13:42:09 +0100 Subject: [PATCH 0153/2294] [FLINK-8797] Port AbstractOperatorRestoreTestBase to MiniClusterResource --- .../AbstractOperatorRestoreTestBase.java | 248 ++++++------------ 1 file changed, 81 insertions(+), 167 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java index 9710c2080c0778..72f700a82bcc4f 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/AbstractOperatorRestoreTestBase.java @@ -19,55 +19,40 @@ package org.apache.flink.test.state.operator.restore; import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.akka.ListeningBehaviour; import org.apache.flink.runtime.checkpoint.savepoint.SavepointSerializers; -import org.apache.flink.runtime.clusterframework.types.ResourceID; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; -import org.apache.flink.runtime.instance.ActorGateway; -import org.apache.flink.runtime.instance.AkkaActorGateway; +import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; -import org.apache.flink.runtime.jobmanager.JobManager; -import org.apache.flink.runtime.messages.JobManagerMessages; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.state.memory.MemoryStateBackend; -import org.apache.flink.runtime.taskmanager.TaskManager; -import org.apache.flink.runtime.testingUtils.TestingJobManager; -import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages; -import org.apache.flink.runtime.testingUtils.TestingMemoryArchivist; -import org.apache.flink.runtime.testingUtils.TestingTaskManager; -import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages; import org.apache.flink.runtime.testingUtils.TestingUtils; -import org.apache.flink.runtime.util.LeaderRetrievalUtils; import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator; +import org.apache.flink.test.util.MiniClusterResource; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.actor.PoisonPill; -import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.net.URL; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import scala.Option; -import scala.Tuple2; -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.duration.FiniteDuration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; /** * Abstract class to verify that it is possible to migrate a savepoint across upgraded Flink versions and that the @@ -79,16 +64,21 @@ */ public abstract class AbstractOperatorRestoreTestBase extends TestLogger { + private static final int NUM_TMS = 1; + private static final int NUM_SLOTS_PER_TM = 4; + private static final Duration TEST_TIMEOUT = Duration.ofSeconds(10000L); + @Rule public final TemporaryFolder tmpFolder = new TemporaryFolder(); - private static ActorSystem actorSystem = null; - private static HighAvailabilityServices highAvailabilityServices = null; - private static ActorGateway jobManager = null; - private static ActorGateway archiver = null; - private static ActorGateway taskManager = null; + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + new Configuration(), + NUM_TMS, + NUM_SLOTS_PER_TM), + true); - private static final FiniteDuration timeout = new FiniteDuration(30L, TimeUnit.SECONDS); private final boolean allowNonRestoredState; protected AbstractOperatorRestoreTestBase() { @@ -104,91 +94,21 @@ public static void beforeClass() { SavepointSerializers.setFailWhenLegacyStateDetected(false); } - @BeforeClass - public static void setupCluster() throws Exception { - final Configuration configuration = new Configuration(); - - FiniteDuration timeout = new FiniteDuration(30L, TimeUnit.SECONDS); - - actorSystem = AkkaUtils.createLocalActorSystem(new Configuration()); - - highAvailabilityServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices( - configuration, - TestingUtils.defaultExecutor()); - - Tuple2 master = JobManager.startJobManagerActors( - configuration, - actorSystem, - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - highAvailabilityServices, - NoOpMetricRegistry.INSTANCE, - Option.empty(), - Option.apply("jm"), - Option.apply("arch"), - TestingJobManager.class, - TestingMemoryArchivist.class); - - jobManager = LeaderRetrievalUtils.retrieveLeaderGateway( - highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID), - actorSystem, - timeout); - - archiver = new AkkaActorGateway(master._2(), jobManager.leaderSessionID()); - - Configuration tmConfig = new Configuration(); - tmConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 4); - - ActorRef taskManagerRef = TaskManager.startTaskManagerComponentsAndActor( - tmConfig, - ResourceID.generate(), - actorSystem, - highAvailabilityServices, - NoOpMetricRegistry.INSTANCE, - "localhost", - Option.apply("tm"), - true, - TestingTaskManager.class); - - taskManager = new AkkaActorGateway(taskManagerRef, jobManager.leaderSessionID()); - - // Wait until connected - Object msg = new TestingTaskManagerMessages.NotifyWhenRegisteredAtJobManager(jobManager.actor()); - Await.ready(taskManager.ask(msg, timeout), timeout); - } - - @AfterClass - public static void tearDownCluster() throws Exception { - if (highAvailabilityServices != null) { - highAvailabilityServices.closeAndCleanupAllData(); - } - - if (actorSystem != null) { - actorSystem.shutdown(); - } - - if (archiver != null) { - archiver.actor().tell(PoisonPill.getInstance(), ActorRef.noSender()); - } - - if (jobManager != null) { - jobManager.actor().tell(PoisonPill.getInstance(), ActorRef.noSender()); - } - - if (taskManager != null) { - taskManager.actor().tell(PoisonPill.getInstance(), ActorRef.noSender()); - } - } - @Test public void testMigrationAndRestore() throws Throwable { + ClassLoader classLoader = this.getClass().getClassLoader(); + ClusterClient clusterClient = MINI_CLUSTER_RESOURCE.getClusterClient(); + clusterClient.setDetached(true); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); + // submit job with old version savepoint and create a migrated savepoint in the new version - String savepointPath = migrateJob(); + String savepointPath = migrateJob(classLoader, clusterClient, deadline); // restore from migrated new version savepoint - restoreJob(savepointPath); + restoreJob(classLoader, clusterClient, deadline, savepointPath); } - private String migrateJob() throws Throwable { + private String migrateJob(ClassLoader classLoader, ClusterClient clusterClient, Deadline deadline) throws Throwable { + URL savepointResource = AbstractOperatorRestoreTestBase.class.getClassLoader().getResource("operatorstate/" + getMigrationSavepointName()); if (savepointResource == null) { throw new IllegalArgumentException("Savepoint file does not exist."); @@ -196,86 +116,80 @@ private String migrateJob() throws Throwable { JobGraph jobToMigrate = createJobGraph(ExecutionMode.MIGRATE); jobToMigrate.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointResource.getFile())); - Object msg; - Object result; - - // Submit job graph - msg = new JobManagerMessages.SubmitJob(jobToMigrate, ListeningBehaviour.DETACHED); - result = Await.result(jobManager.ask(msg, timeout), timeout); + assertNotNull(jobToMigrate.getJobID()); - if (result instanceof JobManagerMessages.JobResultFailure) { - JobManagerMessages.JobResultFailure failure = (JobManagerMessages.JobResultFailure) result; - throw new Exception(failure.cause()); - } - Assert.assertSame(JobManagerMessages.JobSubmitSuccess.class, result.getClass()); + clusterClient.submitJob(jobToMigrate, classLoader); - // Wait for all tasks to be running - msg = new TestingJobManagerMessages.WaitForAllVerticesToBeRunning(jobToMigrate.getJobID()); - Await.result(jobManager.ask(msg, timeout), timeout); + CompletableFuture jobRunningFuture = FutureUtils.retrySuccesfulWithDelay( + () -> clusterClient.getJobStatus(jobToMigrate.getJobID()), + Time.milliseconds(50), + deadline, + (jobStatus) -> jobStatus == JobStatus.RUNNING, + TestingUtils.defaultScheduledExecutor()); + assertEquals( + JobStatus.RUNNING, + jobRunningFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); // Trigger savepoint File targetDirectory = tmpFolder.newFolder(); - msg = new JobManagerMessages.CancelJobWithSavepoint(jobToMigrate.getJobID(), targetDirectory.getAbsolutePath()); + String savepointPath = null; // FLINK-6918: Retry cancel with savepoint message in case that StreamTasks were not running // TODO: The retry logic should be removed once the StreamTask lifecycle has been fixed (see FLINK-4714) - boolean retry = true; - for (int i = 0; retry && i < 10; i++) { - Future future = jobManager.ask(msg, timeout); - result = Await.result(future, timeout); - - if (result instanceof JobManagerMessages.CancellationFailure) { - Thread.sleep(50L); - } else { - retry = false; + while (deadline.hasTimeLeft() && savepointPath == null) { + try { + savepointPath = clusterClient.cancelWithSavepoint( + jobToMigrate.getJobID(), + targetDirectory.getAbsolutePath()); + } catch (Exception e) { + String exceptionString = ExceptionUtils.stringifyException(e); + if (!(exceptionString.matches("(.*\n)*.*savepoint for the job .* failed(.*\n)*") // legacy + || exceptionString.matches("(.*\n)*.*Not all required tasks are currently running(.*\n)*") // flip6 + || exceptionString.matches("(.*\n)*.*Checkpoint was declined \\(tasks not ready\\)(.*\n)*"))) { // flip6 + throw e; + } } } - if (result instanceof JobManagerMessages.CancellationFailure) { - JobManagerMessages.CancellationFailure failure = (JobManagerMessages.CancellationFailure) result; - throw new Exception(failure.cause()); - } - - String savepointPath = ((JobManagerMessages.CancellationSuccess) result).savepointPath(); + assertNotNull("Could not take savepoint.", savepointPath); - // Wait until canceled - msg = new TestingJobManagerMessages.NotifyWhenJobStatus(jobToMigrate.getJobID(), JobStatus.CANCELED); - Await.ready(jobManager.ask(msg, timeout), timeout); + CompletableFuture jobCanceledFuture = FutureUtils.retrySuccesfulWithDelay( + () -> clusterClient.getJobStatus(jobToMigrate.getJobID()), + Time.milliseconds(50), + deadline, + (jobStatus) -> jobStatus == JobStatus.CANCELED, + TestingUtils.defaultScheduledExecutor()); + assertEquals( + JobStatus.CANCELED, + jobCanceledFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); return savepointPath; } - private void restoreJob(String savepointPath) throws Exception { + private void restoreJob(ClassLoader classLoader, ClusterClient clusterClient, Deadline deadline, String savepointPath) throws Exception { JobGraph jobToRestore = createJobGraph(ExecutionMode.RESTORE); jobToRestore.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState)); - Object msg; - Object result; - - // Submit job graph - msg = new JobManagerMessages.SubmitJob(jobToRestore, ListeningBehaviour.DETACHED); - result = Await.result(jobManager.ask(msg, timeout), timeout); + assertNotNull("Job doesn't have a JobID.", jobToRestore.getJobID()); - if (result instanceof JobManagerMessages.JobResultFailure) { - JobManagerMessages.JobResultFailure failure = (JobManagerMessages.JobResultFailure) result; - throw new Exception(failure.cause()); - } - Assert.assertSame(JobManagerMessages.JobSubmitSuccess.class, result.getClass()); - - msg = new JobManagerMessages.RequestJobStatus(jobToRestore.getJobID()); - JobStatus status = ((JobManagerMessages.CurrentJobStatus) Await.result(jobManager.ask(msg, timeout), timeout)).status(); - while (!status.isTerminalState()) { - status = ((JobManagerMessages.CurrentJobStatus) Await.result(jobManager.ask(msg, timeout), timeout)).status(); - } + clusterClient.submitJob(jobToRestore, classLoader); - Assert.assertEquals(JobStatus.FINISHED, status); + CompletableFuture jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + () -> clusterClient.getJobStatus(jobToRestore.getJobID()), + Time.milliseconds(50), + deadline, + (jobStatus) -> jobStatus == JobStatus.FINISHED, + TestingUtils.defaultScheduledExecutor()); + assertEquals( + JobStatus.FINISHED, + jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); } private JobGraph createJobGraph(ExecutionMode mode) { - StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setRestartStrategy(RestartStrategies.noRestart()); - env.setStateBackend(new MemoryStateBackend()); + env.setStateBackend((StateBackend) new MemoryStateBackend()); switch (mode) { case MIGRATE: From 8365c90b8d8fe637e8a54fb21d56e001258db5f2 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Mon, 26 Feb 2018 11:55:14 +0100 Subject: [PATCH 0154/2294] [FLINK-8778] Port queryable state ITCases to use MiniClusterResource --- .../AbstractQueryableStateTestBase.java | 240 ++++++++---------- .../HAAbstractQueryableStateTestBase.java | 93 ------- .../HAQueryableStateFsBackendITCase.java | 90 ++++++- .../HAQueryableStateRocksDBBackendITCase.java | 91 ++++++- .../NonHAAbstractQueryableStateTestBase.java | 75 ------ .../NonHAQueryableStateFsBackendITCase.java | 60 ++++- ...nHAQueryableStateRocksDBBackendITCase.java | 61 ++++- 7 files changed, 375 insertions(+), 335 deletions(-) delete mode 100644 flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAAbstractQueryableStateTestBase.java delete mode 100644 flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAAbstractQueryableStateTestBase.java diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java index 623e42b43e0100..e99a28b36a212b 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/AbstractQueryableStateTestBase.java @@ -37,12 +37,15 @@ import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.client.program.ProgramInvocationException; import org.apache.flink.configuration.Configuration; import org.apache.flink.queryablestate.client.QueryableStateClient; import org.apache.flink.queryablestate.client.VoidNamespace; @@ -53,12 +56,9 @@ import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; -import org.apache.flink.runtime.messages.JobManagerMessages; -import org.apache.flink.runtime.messages.JobManagerMessages.CancellationSuccess; -import org.apache.flink.runtime.minicluster.FlinkMiniCluster; import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.CheckpointListener; -import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages; +import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.QueryableStateStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -68,6 +68,7 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.Collector; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.util.TestLogger; @@ -76,6 +77,7 @@ import org.junit.Ignore; import org.junit.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -93,11 +95,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLongArray; -import scala.concurrent.duration.Deadline; -import scala.concurrent.duration.FiniteDuration; -import scala.reflect.ClassTag$; - +import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -106,7 +106,7 @@ */ public abstract class AbstractQueryableStateTestBase extends TestLogger { - private static final FiniteDuration TEST_TIMEOUT = new FiniteDuration(10000L, TimeUnit.SECONDS); + private static final Duration TEST_TIMEOUT = Duration.ofSeconds(10000L); public static final long RETRY_TIMEOUT = 50L; private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); @@ -117,28 +117,23 @@ public abstract class AbstractQueryableStateTestBase extends TestLogger { */ protected AbstractStateBackend stateBackend; - /** - * Shared between all the test. Make sure to have at least NUM_SLOTS - * available after your test finishes, e.g. cancel the job you submitted. - */ - protected static FlinkMiniCluster cluster; - /** * Client shared between all the test. */ protected static QueryableStateClient client; + protected static ClusterClient clusterClient; + protected static int maxParallelism; @Before public void setUp() throws Exception { - // NOTE: do not use a shared instance for all tests as the tests may brake + // NOTE: do not use a shared instance for all tests as the tests may break this.stateBackend = createStateBackend(); - Assert.assertNotNull(cluster); + Assert.assertNotNull(clusterClient); - maxParallelism = cluster.configuration().getInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1) * - cluster.configuration().getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); + maxParallelism = 4; } /** @@ -160,8 +155,7 @@ public void setUp() throws Exception { @Test @SuppressWarnings("unchecked") public void testQueryableState() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final int numKeys = 256; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -188,12 +182,13 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState(queryName, reducingState); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); final AtomicLongArray counts = new AtomicLongArray(numKeys); @@ -257,9 +252,8 @@ public Integer getKey(Tuple2 value) { /** * Tests that duplicate query registrations fail the job at the JobManager. */ - @Test + @Test(timeout = 60_000) public void testDuplicateRegistrationFailsJob() throws Exception { - final Deadline deadline = TEST_TIMEOUT.fromNow(); final int numKeys = 256; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -302,54 +296,19 @@ public Integer getKey(Tuple2 value) { // Submit the job graph final JobGraph jobGraph = env.getStreamGraph().getJobGraph(); - final JobID jobId = jobGraph.getJobID(); - - final CompletableFuture failedFuture = - notifyWhenJobStatusIs(jobId, JobStatus.FAILED, deadline); - final CompletableFuture cancellationFuture = - notifyWhenJobStatusIs(jobId, JobStatus.CANCELED, deadline); - - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(false); + boolean caughtException = false; try { - final TestingJobManagerMessages.JobStatusIs jobStatus = - failedFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - - assertEquals(JobStatus.FAILED, jobStatus.state()); - } catch (Exception e) { - - // if the assertion fails, it means that the job was (falsely) not cancelled. - // in this case, and given that the mini-cluster is shared with other tests, - // we cancel the job and wait for the cancellation so that the resources are freed. - - if (jobId != null) { - cluster.getLeaderGateway(deadline.timeLeft()) - .ask(new JobManagerMessages.CancelJob(jobId), deadline.timeLeft()) - .mapTo(ClassTag$.MODULE$.apply(CancellationSuccess.class)); - - cancellationFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - } - - // and we re-throw the exception. - throw e; + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); + } catch (ProgramInvocationException e) { + String failureCause = ExceptionUtils.stringifyException(e); + assertThat(failureCause, containsString("KvState with name '" + queryName + "' has already been registered by another operator")); + caughtException = true; } - // Get the job and check the cause - JobManagerMessages.JobFound jobFound = FutureUtils.toJava( - cluster.getLeaderGateway(deadline.timeLeft()) - .ask(new JobManagerMessages.RequestJob(jobId), deadline.timeLeft()) - .mapTo(ClassTag$.MODULE$.apply(JobManagerMessages.JobFound.class))) - .get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - - String failureCause = jobFound.executionGraph().getFailureInfo().getExceptionAsString(); - - assertEquals(JobStatus.FAILED, jobFound.executionGraph().getState()); - assertTrue("Not instance of SuppressRestartsException", failureCause.startsWith("org.apache.flink.runtime.execution.SuppressRestartsException")); - int causedByIndex = failureCause.indexOf("Caused by: "); - String subFailureCause = failureCause.substring(causedByIndex + "Caused by: ".length()); - assertTrue("Not caused by IllegalStateException", subFailureCause.startsWith("java.lang.IllegalStateException")); - assertTrue("Exception does not contain registration name", subFailureCause.contains(queryName)); + assertTrue(caughtException); } /** @@ -360,8 +319,7 @@ public Integer getKey(Tuple2 value) { */ @Test public void testValueState() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final long numElements = 1024L; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -386,12 +344,13 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState("hakuna", valueState); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); executeValueQuery(deadline, client, jobId, "hakuna", valueState, numElements); } @@ -404,8 +363,7 @@ public Integer getKey(Tuple2 value) { @Test @Ignore public void testWrongJobIdAndWrongQueryableStateName() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final long numElements = 1024L; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -425,18 +383,22 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState("hakuna", valueState); - try (AutoCancellableJob closableJobGraph = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob closableJobGraph = new AutoCancellableJob(deadline, clusterClient, env)) { - // register to be notified when the job is running. - CompletableFuture runningFuture = - notifyWhenJobStatusIs(closableJobGraph.getJobId(), JobStatus.RUNNING, deadline); + clusterClient.setDetached(true); + clusterClient.submitJob( + closableJobGraph.getJobGraph(), AbstractQueryableStateTestBase.class.getClassLoader()); - cluster.submitJobDetached(closableJobGraph.getJobGraph()); + CompletableFuture jobStatusFuture = + clusterClient.getJobStatus(closableJobGraph.getJobId()); + + while (deadline.hasTimeLeft() && !jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).equals(JobStatus.RUNNING)) { + Thread.sleep(50); + jobStatusFuture = + clusterClient.getJobStatus(closableJobGraph.getJobId()); + } - // expect for the job to be running - TestingJobManagerMessages.JobStatusIs jobStatus = - runningFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - assertEquals(JobStatus.RUNNING, jobStatus.state()); + assertEquals(JobStatus.RUNNING, jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); final JobID wrongJobId = new JobID(); @@ -484,14 +446,13 @@ public Integer getKey(Tuple2 value) { */ @Test public void testQueryNonStartedJobState() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final long numElements = 1024L; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStateBackend(stateBackend); env.setParallelism(maxParallelism); - // Very important, because cluster is shared between tests and we + // Very important, because clusterClient is shared between tests and we // don't explicitly check that all slots are available before // submitting. env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L)); @@ -512,7 +473,7 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState("hakuna", valueState); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); @@ -527,7 +488,8 @@ public Integer getKey(Tuple2 value) { BasicTypeInfo.INT_TYPE_INFO, valueState); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); executeValueQuery(deadline, client, jobId, "hakuna", valueState, expected); } @@ -543,8 +505,7 @@ public Integer getKey(Tuple2 value) { */ @Test(expected = UnknownKeyOrNamespaceException.class) public void testValueStateDefault() throws Throwable { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final long numElements = 1024L; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -571,12 +532,13 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState("hakuna", valueState); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); // Now query int key = 0; @@ -611,8 +573,7 @@ public Integer getKey(Tuple2 value) { */ @Test public void testValueStateShortcut() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final long numElements = 1024L; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -639,12 +600,14 @@ public Integer getKey(Tuple2 value) { final ValueStateDescriptor> stateDesc = (ValueStateDescriptor>) queryableState.getStateDescriptor(); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); + executeValueQuery(deadline, client, jobId, "matata", stateDesc, numElements); } } @@ -658,8 +621,7 @@ public Integer getKey(Tuple2 value) { */ @Test public void testFoldingState() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final int numElements = 1024; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -684,12 +646,13 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState("pumba", foldingState); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); final String expected = Integer.toString(numElements * (numElements + 1) / 2); @@ -731,8 +694,7 @@ public Integer getKey(Tuple2 value) { */ @Test public void testReducingState() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final long numElements = 1024L; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -757,12 +719,13 @@ public Integer getKey(Tuple2 value) { } }).asQueryableState("jungle", reducingState); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); final long expected = numElements * (numElements + 1L) / 2L; @@ -804,8 +767,7 @@ public Integer getKey(Tuple2 value) { */ @Test public void testMapState() throws Exception { - - final Deadline deadline = TEST_TIMEOUT.fromNow(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); final long numElements = 1024L; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); @@ -850,12 +812,13 @@ public void processElement(Tuple2 value, Context ctx, Collector value, Context ctx, Collector value, Context ctx, Collector> results = new HashMap<>(); @@ -994,8 +957,7 @@ public void processElement(Tuple2 value, Context ctx, Collector value) { new AggregatingTestOperator(aggrStateDescriptor) ); - try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(cluster, env, deadline)) { + try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) { final JobID jobId = autoCancellableJob.getJobId(); final JobGraph jobGraph = autoCancellableJob.getJobGraph(); - cluster.submitJobDetached(jobGraph); + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader()); for (int key = 0; key < maxParallelism; key++) { boolean success = false; @@ -1277,22 +1240,22 @@ public Tuple2 reduce(Tuple2 value1, Tuple2 clusterClient; private final JobGraph jobGraph; private final JobID jobId; - private final CompletableFuture cancellationFuture; - AutoCancellableJob(final FlinkMiniCluster cluster, final StreamExecutionEnvironment env, final Deadline deadline) { + private final Deadline deadline; + + AutoCancellableJob(Deadline deadline, final ClusterClient clusterClient, final StreamExecutionEnvironment env) { Preconditions.checkNotNull(env); - this.cluster = Preconditions.checkNotNull(cluster); + this.clusterClient = Preconditions.checkNotNull(clusterClient); this.jobGraph = env.getStreamGraph().getJobGraph(); - this.deadline = Preconditions.checkNotNull(deadline); - this.jobId = jobGraph.getJobID(); - this.cancellationFuture = notifyWhenJobStatusIs(jobId, JobStatus.CANCELED, deadline); + this.jobId = Preconditions.checkNotNull(jobGraph.getJobID()); + + this.deadline = deadline; } JobGraph getJobGraph() { @@ -1306,25 +1269,20 @@ JobID getJobId() { @Override public void close() throws Exception { // Free cluster resources - if (jobId != null) { - cluster.getLeaderGateway(deadline.timeLeft()) - .ask(new JobManagerMessages.CancelJob(jobId), deadline.timeLeft()) - .mapTo(ClassTag$.MODULE$.apply(CancellationSuccess.class)); - - cancellationFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - } + clusterClient.cancel(jobId); + // cancel() is non-blocking so do this to make sure the job finished + CompletableFuture jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + () -> clusterClient.getJobStatus(jobId), + Time.milliseconds(50), + deadline, + (jobStatus) -> jobStatus.equals(JobStatus.CANCELED), + TestingUtils.defaultScheduledExecutor()); + assertEquals( + JobStatus.CANCELED, + jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); } } - private static CompletableFuture notifyWhenJobStatusIs( - final JobID jobId, final JobStatus status, final Deadline deadline) { - - return FutureUtils.toJava( - cluster.getLeaderGateway(deadline.timeLeft()) - .ask(new TestingJobManagerMessages.NotifyWhenJobStatus(jobId, status), deadline.timeLeft()) - .mapTo(ClassTag$.MODULE$.apply(TestingJobManagerMessages.JobStatusIs.class))); - } - private static CompletableFuture getKvState( final Deadline deadline, final QueryableStateClient client, diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAAbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAAbstractQueryableStateTestBase.java deleted file mode 100644 index 8767b5214e98d3..00000000000000 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAAbstractQueryableStateTestBase.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.queryablestate.itcases; - -import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.HighAvailabilityOptions; -import org.apache.flink.configuration.QueryableStateOptions; -import org.apache.flink.queryablestate.client.QueryableStateClient; -import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; -import org.apache.flink.runtime.testingUtils.TestingCluster; - -import org.apache.curator.test.TestingServer; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; - -import static org.junit.Assert.fail; - -/** - * Base class with the cluster configuration for the tests on the NON-HA mode. - */ -public abstract class HAAbstractQueryableStateTestBase extends AbstractQueryableStateTestBase { - - private static final int NUM_JMS = 2; - private static final int NUM_TMS = 2; - private static final int NUM_SLOTS_PER_TM = 4; - - private static TestingServer zkServer; - private static TemporaryFolder temporaryFolder; - - public static void setup(int proxyPortRangeStart, int serverPortRangeStart) { - try { - zkServer = new TestingServer(); - temporaryFolder = new TemporaryFolder(); - temporaryFolder.create(); - - Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); - config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2); - config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2); - config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2); - config.setString(QueryableStateOptions.PROXY_PORT_RANGE, proxyPortRangeStart + "-" + (proxyPortRangeStart + NUM_TMS)); - config.setString(QueryableStateOptions.SERVER_PORT_RANGE, serverPortRangeStart + "-" + (serverPortRangeStart + NUM_TMS)); - config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString()); - config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString()); - config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); - - cluster = new TestingCluster(config, false); - cluster.start(true); - - client = new QueryableStateClient("localhost", proxyPortRangeStart); - - // verify that we are in HA mode - Assert.assertTrue(cluster.haMode() == HighAvailabilityMode.ZOOKEEPER); - - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - @AfterClass - public static void tearDown() throws IOException { - client.shutdownAndWait(); - - cluster.stop(); - cluster.awaitTermination(); - - zkServer.stop(); - zkServer.close(); - } -} diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java index 6f31e76b5aa1fc..a47045f35a8c56 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java @@ -18,28 +18,102 @@ package org.apache.flink.queryablestate.itcases; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.HighAvailabilityOptions; +import org.apache.flink.configuration.QueryableStateOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.configuration.WebOptions; +import org.apache.flink.queryablestate.client.QueryableStateClient; import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.test.util.MiniClusterResource; +import org.apache.curator.test.TestingServer; +import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.rules.TemporaryFolder; /** * Several integration tests for queryable state using the {@link FsStateBackend}. */ -public class HAQueryableStateFsBackendITCase extends HAAbstractQueryableStateTestBase { +public class HAQueryableStateFsBackendITCase extends AbstractQueryableStateTestBase { - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private static final int NUM_JMS = 2; + // NUM_TMS * NUM_SLOTS_PER_TM must match the parallelism of the pipelines so that + // we always use all TaskManagers so that the JM oracle is always properly re-registered + private static final int NUM_TMS = 2; + private static final int NUM_SLOTS_PER_TM = 2; - @BeforeClass - public static void setup() { - setup(9064, 9069); - } + private static final int QS_PROXY_PORT_RANGE_START = 9064; + private static final int QS_SERVER_PORT_RANGE_START = 9069; + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static TestingServer zkServer; + + private static MiniClusterResource miniClusterResource; @Override protected AbstractStateBackend createStateBackend() throws Exception { return new FsStateBackend(temporaryFolder.newFolder().toURI().toString()); } + + @BeforeClass + public static void setup() throws Exception { + zkServer = new TestingServer(); + + // we have to manage this manually because we have to create the ZooKeeper server + // ahead of this + miniClusterResource = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfig(), + NUM_TMS, + NUM_SLOTS_PER_TM), + true); + + miniClusterResource.before(); + + client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); + + clusterClient = miniClusterResource.getClusterClient(); + } + + @AfterClass + public static void tearDown() throws Exception { + miniClusterResource.after(); + + client.shutdownAndWait(); + + zkServer.stop(); + zkServer.close(); + } + + private static Configuration getConfig() throws Exception { + + Configuration config = new Configuration(); + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); + config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); + config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2); + config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2); + config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2); + config.setString( + QueryableStateOptions.PROXY_PORT_RANGE, + QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS)); + config.setString( + QueryableStateOptions.SERVER_PORT_RANGE, + QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS)); + config.setBoolean(WebOptions.SUBMIT_ENABLE, false); + + config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString()); + + config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString()); + config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); + + return config; + } } diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java index cae02e2ba69c8d..b1092c14167025 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java @@ -18,28 +18,103 @@ package org.apache.flink.queryablestate.itcases; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.HighAvailabilityOptions; +import org.apache.flink.configuration.QueryableStateOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.configuration.WebOptions; import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; +import org.apache.flink.queryablestate.client.QueryableStateClient; import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.test.util.MiniClusterResource; +import org.apache.curator.test.TestingServer; +import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.rules.TemporaryFolder; /** * Several integration tests for queryable state using the {@link RocksDBStateBackend}. */ -public class HAQueryableStateRocksDBBackendITCase extends HAAbstractQueryableStateTestBase { +public class HAQueryableStateRocksDBBackendITCase extends AbstractQueryableStateTestBase { - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private static final int NUM_JMS = 2; + // NUM_TMS * NUM_SLOTS_PER_TM must match the parallelism of the pipelines so that + // we always use all TaskManagers so that the JM oracle is always properly re-registered + private static final int NUM_TMS = 2; + private static final int NUM_SLOTS_PER_TM = 2; - @BeforeClass - public static void setup() { - setup(9074, 9079); - } + private static final int QS_PROXY_PORT_RANGE_START = 9074; + private static final int QS_SERVER_PORT_RANGE_START = 9079; + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static TestingServer zkServer; + + private static MiniClusterResource miniClusterResource; @Override protected AbstractStateBackend createStateBackend() throws Exception { return new RocksDBStateBackend(temporaryFolder.newFolder().toURI().toString()); } + + @BeforeClass + public static void setup() throws Exception { + zkServer = new TestingServer(); + + // we have to manage this manually because we have to create the ZooKeeper server + // ahead of this + miniClusterResource = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfig(), + NUM_TMS, + NUM_SLOTS_PER_TM), + true); + + miniClusterResource.before(); + + client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); + + clusterClient = miniClusterResource.getClusterClient(); + } + + @AfterClass + public static void tearDown() throws Exception { + miniClusterResource.after(); + + client.shutdownAndWait(); + + zkServer.stop(); + zkServer.close(); + } + + private static Configuration getConfig() throws Exception { + + Configuration config = new Configuration(); + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); + config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); + config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2); + config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2); + config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2); + config.setString( + QueryableStateOptions.PROXY_PORT_RANGE, + QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS)); + config.setString( + QueryableStateOptions.SERVER_PORT_RANGE, + QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS)); + config.setBoolean(WebOptions.SUBMIT_ENABLE, false); + + config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString()); + + config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString()); + config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); + + return config; + } + } diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAAbstractQueryableStateTestBase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAAbstractQueryableStateTestBase.java deleted file mode 100644 index 2686a2981f3196..00000000000000 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAAbstractQueryableStateTestBase.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.queryablestate.itcases; - -import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.QueryableStateOptions; -import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.queryablestate.client.QueryableStateClient; -import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; -import org.apache.flink.runtime.testingUtils.TestingCluster; - -import org.junit.AfterClass; -import org.junit.Assert; - -import static org.junit.Assert.fail; - -/** - * Base class with the cluster configuration for the tests on the HA mode. - */ -public abstract class NonHAAbstractQueryableStateTestBase extends AbstractQueryableStateTestBase { - - private static final int NUM_TMS = 2; - private static final int NUM_SLOTS_PER_TM = 4; - - public static void setup(int proxyPortRangeStart, int serverPortRangeStart) { - try { - Configuration config = new Configuration(); - config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); - config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1); - config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1); - config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1); - config.setString(QueryableStateOptions.PROXY_PORT_RANGE, proxyPortRangeStart + "-" + (proxyPortRangeStart + NUM_TMS)); - config.setString(QueryableStateOptions.SERVER_PORT_RANGE, serverPortRangeStart + "-" + (serverPortRangeStart + NUM_TMS)); - - cluster = new TestingCluster(config, false); - cluster.start(true); - - client = new QueryableStateClient("localhost", proxyPortRangeStart); - - // verify that we are not in HA mode - Assert.assertTrue(cluster.haMode() == HighAvailabilityMode.NONE); - - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - @AfterClass - public static void tearDown() { - client.shutdownAndWait(); - - cluster.stop(); - cluster.awaitTermination(); - } -} diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java index 9457e0f0471273..eb300c12e4cab3 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java @@ -18,28 +18,78 @@ package org.apache.flink.queryablestate.itcases; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.QueryableStateOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.configuration.WebOptions; +import org.apache.flink.queryablestate.client.QueryableStateClient; import org.apache.flink.runtime.state.AbstractStateBackend; import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.test.util.MiniClusterResource; +import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.rules.TemporaryFolder; /** * Several integration tests for queryable state using the {@link FsStateBackend}. */ -public class NonHAQueryableStateFsBackendITCase extends NonHAAbstractQueryableStateTestBase { +public class NonHAQueryableStateFsBackendITCase extends AbstractQueryableStateTestBase { + + // NUM_TMS * NUM_SLOTS_PER_TM must match the parallelism of the pipelines so that + // we always use all TaskManagers so that the JM oracle is always properly re-registered + private static final int NUM_TMS = 2; + private static final int NUM_SLOTS_PER_TM = 2; + + private static final int QS_PROXY_PORT_RANGE_START = 9084; + private static final int QS_SERVER_PORT_RANGE_START = 9089; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @BeforeClass - public static void setup() { - setup(9084, 9089); - } + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfig(), + NUM_TMS, + NUM_SLOTS_PER_TM), + true); @Override protected AbstractStateBackend createStateBackend() throws Exception { return new FsStateBackend(temporaryFolder.newFolder().toURI().toString()); } + + @BeforeClass + public static void setup() throws Exception { + client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); + + clusterClient = MINI_CLUSTER_RESOURCE.getClusterClient(); + } + + @AfterClass + public static void tearDown() { + client.shutdownAndWait(); + } + + private static Configuration getConfig() { + Configuration config = new Configuration(); + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); + config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1); + config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1); + config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1); + config.setString( + QueryableStateOptions.PROXY_PORT_RANGE, + QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS)); + config.setString( + QueryableStateOptions.SERVER_PORT_RANGE, + QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS)); + config.setBoolean(WebOptions.SUBMIT_ENABLE, false); + return config; + } } diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java index 7778a9446bd9d5..3d6a3e3fcfdc06 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java @@ -18,28 +18,79 @@ package org.apache.flink.queryablestate.itcases; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.QueryableStateOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.configuration.WebOptions; import org.apache.flink.contrib.streaming.state.RocksDBStateBackend; +import org.apache.flink.queryablestate.client.QueryableStateClient; import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.test.util.MiniClusterResource; +import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.rules.TemporaryFolder; /** * Several integration tests for queryable state using the {@link RocksDBStateBackend}. */ -public class NonHAQueryableStateRocksDBBackendITCase extends NonHAAbstractQueryableStateTestBase { +public class NonHAQueryableStateRocksDBBackendITCase extends AbstractQueryableStateTestBase { + + // NUM_TMS * NUM_SLOTS_PER_TM must match the parallelism of the pipelines so that + // we always use all TaskManagers so that the JM oracle is always properly re-registered + private static final int NUM_TMS = 2; + private static final int NUM_SLOTS_PER_TM = 2; + + private static final int QS_PROXY_PORT_RANGE_START = 9094; + private static final int QS_SERVER_PORT_RANGE_START = 9099; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @BeforeClass - public static void setup() { - setup(9094, 9099); - } + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfig(), + NUM_TMS, + NUM_SLOTS_PER_TM), + true); @Override protected AbstractStateBackend createStateBackend() throws Exception { return new RocksDBStateBackend(temporaryFolder.newFolder().toURI().toString()); } + + @BeforeClass + public static void setup() throws Exception { + client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START); + + clusterClient = MINI_CLUSTER_RESOURCE.getClusterClient(); + } + + @AfterClass + public static void tearDown() { + client.shutdownAndWait(); + } + + private static Configuration getConfig() { + Configuration config = new Configuration(); + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); + config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1); + config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1); + config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1); + config.setString( + QueryableStateOptions.PROXY_PORT_RANGE, + QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS)); + config.setString( + QueryableStateOptions.SERVER_PORT_RANGE, + QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS)); + config.setBoolean(WebOptions.SUBMIT_ENABLE, false); + return config; + } + } From 0f3b3f9404743e2920e50099645bb659a418bd00 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Fri, 9 Mar 2018 14:11:04 +0100 Subject: [PATCH 0155/2294] [FLINK-8911] Add separate script for nightly end-to-end tests --- flink-end-to-end-tests/README.md | 18 +++++++- flink-end-to-end-tests/run-nightly-tests.sh | 51 +++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100755 flink-end-to-end-tests/run-nightly-tests.sh diff --git a/flink-end-to-end-tests/README.md b/flink-end-to-end-tests/README.md index 1c8aadcc6177b8..82d7e419ad8f9d 100644 --- a/flink-end-to-end-tests/README.md +++ b/flink-end-to-end-tests/README.md @@ -1,14 +1,28 @@ # Flink End-to-End Tests -This module contains tests that verify end-to-end behaviour of Flink. +This module contains tests that verify end-to-end behaviour of Flink. We +categorize end-to-end tests as either pre-commit tests or nightly tests. The +former should be run on every commit, that is every Travis run, while the second +category should be run by a nightly job or when manually verifying a release or +making sure that the tests pass. + +Tests in the pre-commit category should be more lightweight while tests in the +nightly category can be quite heavyweight because we don't run them for every +commit. ## Running Tests -You can run all tests by executing +You can run all pre-commit tests by executing ``` $ FLINK_DIR= flink-end-to-end-tests/run-pre-commit-tests.sh ``` +and all nightly tests via + +``` +$ FLINK_DIR= flink-end-to-end-tests/run-nightly-tests.sh +``` + where is a Flink distribution directory. You can also run tests individually via diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh new file mode 100755 index 00000000000000..8ee526bc23ed0e --- /dev/null +++ b/flink-end-to-end-tests/run-nightly-tests.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +END_TO_END_DIR="`dirname \"$0\"`" # relative +END_TO_END_DIR="`( cd \"$END_TO_END_DIR\" && pwd )`" # absolutized and normalized +if [ -z "$END_TO_END_DIR" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +if [ -z "$FLINK_DIR" ] ; then + echo "You have to export the Flink distribution directory as FLINK_DIR" + exit 1 +fi + +FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized + +echo "flink-end-to-end-test directory: $END_TO_END_DIR" +echo "Flink distribution directory: $FLINK_DIR" + +EXIT_CODE=0 + +# Template for adding a test: + +# if [ $EXIT_CODE == 0 ]; then +# printf "\n==============================================================================\n" +# printf "Running my fancy nightly end-to-end test\n" +# printf "==============================================================================\n" +# $END_TO_END_DIR/test-scripts/test_something_very_fancy.sh +# EXIT_CODE=$? +# fi + +# Exit code for Travis build success/failure +exit $EXIT_CODE From 4c85b74016a1e6587038053f576b8293aa72126e Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Sat, 3 Mar 2018 09:34:56 +0100 Subject: [PATCH 0156/2294] [FLINK-8487] Verify ZooKeeper checkpoint store behaviour with ITCase --- .../ZooKeeperCompletedCheckpointStore.java | 2 +- .../ZooKeeperHighAvailabilityITCase.java | 326 ++++++++++++++++++ 2 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java index 0cbd4fb6c9e9f0..f22127041d31bd 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/ZooKeeperCompletedCheckpointStore.java @@ -209,7 +209,7 @@ public void recover() throws Exception { if (completedCheckpoints.isEmpty() && numberOfInitialCheckpoints > 0) { throw new FlinkException( - "Could not read any of the " + numberOfInitialCheckpoints + " from storage."); + "Could not read any of the " + numberOfInitialCheckpoints + " checkpoints from storage."); } else if (completedCheckpoints.size() != numberOfInitialCheckpoints) { LOG.warn( "Could only fetch {} of {} checkpoints from storage.", diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java new file mode 100644 index 00000000000000..156d4486c627bf --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/ZooKeeperHighAvailabilityITCase.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.checkpointing; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.functions.RichMapFunction; +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.HighAvailabilityOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.core.testutils.OneShotLatch; +import org.apache.flink.runtime.concurrent.FutureUtils; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.state.FunctionInitializationContext; +import org.apache.flink.runtime.state.FunctionSnapshotContext; +import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.filesystem.FsStateBackend; +import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.test.util.MiniClusterResource; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.TestLogger; + +import org.apache.curator.test.TestingServer; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * Integration tests for {@link org.apache.flink.runtime.checkpoint.ZooKeeperCompletedCheckpointStore}. + */ +public class ZooKeeperHighAvailabilityITCase extends TestLogger { + + private static final Duration TEST_TIMEOUT = Duration.ofSeconds(10000L); + + private static final int NUM_JMS = 1; + private static final int NUM_TMS = 1; + private static final int NUM_SLOTS_PER_TM = 1; + + @ClassRule + public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder(); + + private static File haStorageDir; + + private static TestingServer zkServer; + + private static MiniClusterResource miniClusterResource; + + private static OneShotLatch waitForCheckpointLatch = new OneShotLatch(); + private static OneShotLatch failInCheckpointLatch = new OneShotLatch(); + + @BeforeClass + public static void setup() throws Exception { + zkServer = new TestingServer(); + + Configuration config = new Configuration(); + config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); + + haStorageDir = TEMPORARY_FOLDER.newFolder(); + + config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haStorageDir.toString()); + config.setString(HighAvailabilityOptions.HA_CLUSTER_ID, UUID.randomUUID().toString()); + config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString()); + config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper"); + + // we have to manage this manually because we have to create the ZooKeeper server + // ahead of this + miniClusterResource = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + config, + NUM_TMS, + NUM_SLOTS_PER_TM), + true); + + miniClusterResource.before(); + } + + @AfterClass + public static void tearDown() throws Exception { + miniClusterResource.after(); + + zkServer.stop(); + zkServer.close(); + } + + /** + * Verify that we don't start a job from scratch if we cannot restore any of the + * CompletedCheckpoints. + * + *

    Synchronization for the different steps and things we want to observe happens via + * latches in the test method and the methods of {@link CheckpointBlockingFunction}. + * + *

    The test follows these steps: + *

      + *
    1. Start job and block on a latch until we have done some checkpoints + *
    2. Block in the special function + *
    3. Move away the contents of the ZooKeeper HA directory to make restoring from + * checkpoints impossible + *
    4. Unblock the special function, which now induces a failure + *
    5. Make sure that the job does not recover successfully + *
    6. Move back the HA directory + *
    7. Make sure that the job recovers, we use a latch to ensure that the operator + * restored successfully + *
    + */ + @Test(timeout = 120_000L) + public void testRestoreBehaviourWithFaultyStateHandles() throws Exception { + CheckpointBlockingFunction.allowedInitializeCallsWithoutRestore.set(1); + CheckpointBlockingFunction.successfulRestores.set(0); + CheckpointBlockingFunction.illegalRestores.set(0); + CheckpointBlockingFunction.afterMessWithZooKeeper.set(false); + CheckpointBlockingFunction.failedAlready.set(false); + + waitForCheckpointLatch = new OneShotLatch(); + failInCheckpointLatch = new OneShotLatch(); + + ClusterClient clusterClient = miniClusterResource.getClusterClient(); + final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0)); + env.enableCheckpointing(10); // Flink doesn't allow lower than 10 ms + + File checkpointLocation = TEMPORARY_FOLDER.newFolder(); + env.setStateBackend((StateBackend) new FsStateBackend(checkpointLocation.toURI())); + + DataStreamSource source = env.addSource(new UnboundedSource()); + + source + .keyBy((str) -> str) + .map(new CheckpointBlockingFunction()); + + JobGraph jobGraph = env.getStreamGraph().getJobGraph(); + JobID jobID = Preconditions.checkNotNull(jobGraph.getJobID()); + + clusterClient.setDetached(true); + clusterClient.submitJob(jobGraph, ZooKeeperHighAvailabilityITCase.class.getClassLoader()); + + // wait until we did some checkpoints + waitForCheckpointLatch.await(); + + // mess with the HA directory so that the job cannot restore + File movedCheckpointLocation = TEMPORARY_FOLDER.newFolder(); + int numCheckpoints = 0; + File[] files = haStorageDir.listFiles(); + assertNotNull(files); + for (File file : files) { + if (file.getName().startsWith("completedCheckpoint")) { + assertTrue(file.renameTo(new File(movedCheckpointLocation, file.getName()))); + numCheckpoints++; + } + } + // Note to future developers: This will break when we change Flink to not put the + // checkpoint metadata into the HA directory but instead rely on the fact that the + // actual checkpoint directory on DFS contains the checkpoint metadata. In this case, + // ZooKeeper will only contain a "handle" (read: String) that points to the metadata + // in DFS. The likely solution will be that we have to go directly to ZooKeeper, find + // out where the checkpoint is stored and mess with that. + assertTrue(numCheckpoints > 0); + + failInCheckpointLatch.trigger(); + + // Ensure that we see at least one cycle where the job tries to restart and fails. + CompletableFuture jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + () -> clusterClient.getJobStatus(jobID), + Time.milliseconds(1), + deadline, + (jobStatus) -> jobStatus == JobStatus.RESTARTING, + TestingUtils.defaultScheduledExecutor()); + assertEquals(JobStatus.RESTARTING, jobStatusFuture.get()); + + jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + () -> clusterClient.getJobStatus(jobID), + Time.milliseconds(1), + deadline, + (jobStatus) -> jobStatus == JobStatus.FAILING, + TestingUtils.defaultScheduledExecutor()); + assertEquals(JobStatus.FAILING, jobStatusFuture.get()); + + // move back the HA directory so that the job can restore + CheckpointBlockingFunction.afterMessWithZooKeeper.set(true); + + files = movedCheckpointLocation.listFiles(); + assertNotNull(files); + for (File file : files) { + if (file.getName().startsWith("completedCheckpoint")) { + assertTrue(file.renameTo(new File(haStorageDir, file.getName()))); + } + } + + // now the job should be able to go to RUNNING again and then eventually to FINISHED, + // which it only does if it could successfully restore + jobStatusFuture = FutureUtils.retrySuccesfulWithDelay( + () -> clusterClient.getJobStatus(jobID), + Time.milliseconds(50), + deadline, + (jobStatus) -> jobStatus == JobStatus.FINISHED, + TestingUtils.defaultScheduledExecutor()); + assertEquals(JobStatus.FINISHED, jobStatusFuture.get()); + + assertThat("We saw illegal restores.", CheckpointBlockingFunction.illegalRestores.get(), is(0)); + } + + private static class UnboundedSource implements SourceFunction { + private volatile boolean running = true; + + @Override + public void run(SourceContext ctx) throws Exception { + while (running && !CheckpointBlockingFunction.afterMessWithZooKeeper.get()) { + ctx.collect("hello"); + // don't overdo it ... ;-) + Thread.sleep(50); + } + } + + @Override + public void cancel() { + running = false; + } + } + + private static class CheckpointBlockingFunction + extends RichMapFunction + implements CheckpointedFunction { + + // verify that we only call initializeState() + // once with isRestored() == false. All other invocations must have isRestored() == true. This + // verifies that we don't restart a job from scratch in case the CompletedCheckpoints can't + // be read. + static AtomicInteger allowedInitializeCallsWithoutRestore = new AtomicInteger(1); + + // we count when we see restores that are not allowed. We only + // allow restores once we messed with the HA directory and moved it back again + static AtomicInteger illegalRestores = new AtomicInteger(0); + static AtomicInteger successfulRestores = new AtomicInteger(0); + + // whether we are after the phase where we messed with the ZooKeeper HA directory, i.e. + // whether it's now ok for a restore to happen + static AtomicBoolean afterMessWithZooKeeper = new AtomicBoolean(false); + + static AtomicBoolean failedAlready = new AtomicBoolean(false); + + // also have some state to write to the checkpoint + private final ValueStateDescriptor stateDescriptor = + new ValueStateDescriptor<>("state", StringSerializer.INSTANCE); + + @Override + public String map(String value) throws Exception { + getRuntimeContext().getState(stateDescriptor).update("42"); + return value; + } + + @Override + public void snapshotState(FunctionSnapshotContext context) throws Exception { + if (context.getCheckpointId() > 5) { + waitForCheckpointLatch.trigger(); + failInCheckpointLatch.await(); + if (!failedAlready.getAndSet(true)) { + throw new RuntimeException("Failing on purpose."); + } + } + } + + @Override + public void initializeState(FunctionInitializationContext context) { + if (!context.isRestored()) { + int updatedValue = allowedInitializeCallsWithoutRestore.decrementAndGet(); + if (updatedValue < 0) { + illegalRestores.getAndIncrement(); + throw new RuntimeException("We are not allowed any more restores."); + } + } else { + if (!afterMessWithZooKeeper.get()) { + illegalRestores.getAndIncrement(); + } else if (successfulRestores.getAndIncrement() > 0) { + // already saw the one allowed successful restore + illegalRestores.getAndIncrement(); + } + } + } + } +} From 1f55f730a65c077573ac85e67ecebc6e25c252f7 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Thu, 1 Mar 2018 16:26:21 +0100 Subject: [PATCH 0157/2294] [FLINK-8274] [table] Split generated methods for preventing compiler exceptions This closes #5613. This closes #5174. --- .../apache/flink/table/api/TableConfig.scala | 33 +++- .../flink/table/codegen/CodeGenerator.scala | 143 ++++++++++++++---- .../codegen/CollectorCodeGenerator.scala | 66 +++++--- .../table/codegen/FunctionCodeGenerator.scala | 96 +++++++----- .../codegen/InputFormatCodeGenerator.scala | 6 +- .../runtime/batch/table/CorrelateITCase.scala | 15 ++ .../table/runtime/stream/sql/SqlITCase.scala | 54 ++++++- .../table/runtime/utils/StreamTestData.scala | 6 + .../utils/StreamingWithStateTestBase.scala | 3 +- 9 files changed, 320 insertions(+), 102 deletions(-) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala index c78a022bec44a1..51c9a37f155347 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableConfig.scala @@ -40,13 +40,19 @@ class TableConfig { /** * Defines the configuration of Calcite for Table API and SQL queries. */ - private var calciteConfig = CalciteConfig.DEFAULT + private var calciteConfig: CalciteConfig = CalciteConfig.DEFAULT /** * Defines the default context for decimal division calculation. * We use Scala's default MathContext.DECIMAL128. */ - private var decimalContext = MathContext.DECIMAL128 + private var decimalContext: MathContext = MathContext.DECIMAL128 + + /** + * Specifies a threshold where generated code will be split into sub-function calls. Java has a + * maximum method length of 64 KB. This setting allows for finer granularity if necessary. + */ + private var maxGeneratedCodeLength: Int = 64000 // just an estimate /** * Sets the timezone for date/time/timestamp conversions. @@ -59,12 +65,12 @@ class TableConfig { /** * Returns the timezone for date/time/timestamp conversions. */ - def getTimeZone = timeZone + def getTimeZone: TimeZone = timeZone /** * Returns the NULL check. If enabled, all fields need to be checked for NULL first. */ - def getNullCheck = nullCheck + def getNullCheck: Boolean = nullCheck /** * Sets the NULL check. If enabled, all fields need to be checked for NULL first. @@ -99,6 +105,25 @@ class TableConfig { def setDecimalContext(mathContext: MathContext): Unit = { this.decimalContext = mathContext } + + /** + * Returns the current threshold where generated code will be split into sub-function calls. + * Java has a maximum method length of 64 KB. This setting allows for finer granularity if + * necessary. Default is 64000. + */ + def getMaxGeneratedCodeLength: Int = maxGeneratedCodeLength + + /** + * Returns the current threshold where generated code will be split into sub-function calls. + * Java has a maximum method length of 64 KB. This setting allows for finer granularity if + * necessary. Default is 64000. + */ + def setMaxGeneratedCodeLength(maxGeneratedCodeLength: Int): Unit = { + if (maxGeneratedCodeLength <= 0) { + throw new IllegalArgumentException("Length must be greater than 0.") + } + this.maxGeneratedCodeLength = maxGeneratedCodeLength + } } object TableConfig { diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala index e4064d6f1acd80..44885e3d72e2c2 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala @@ -109,31 +109,45 @@ abstract class CodeGenerator( // set of member statements that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableMemberStatements = mutable.LinkedHashSet[String]() + protected val reusableMemberStatements: mutable.LinkedHashSet[String] = + mutable.LinkedHashSet[String]() // set of constructor statements that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableInitStatements = mutable.LinkedHashSet[String]() + protected val reusableInitStatements: mutable.LinkedHashSet[String] = + mutable.LinkedHashSet[String]() // set of open statements for RichFunction that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableOpenStatements = mutable.LinkedHashSet[String]() + protected val reusableOpenStatements: mutable.LinkedHashSet[String] = + mutable.LinkedHashSet[String]() // set of close statements for RichFunction that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableCloseStatements = mutable.LinkedHashSet[String]() + protected val reusableCloseStatements: mutable.LinkedHashSet[String] = + mutable.LinkedHashSet[String]() - // set of statements that will be added only once per record + // set of statements that will be added only once per record; + // code should only update member variables because local variables are not accessible if + // the code needs to be split; // we use a LinkedHashSet to keep the insertion order - protected val reusablePerRecordStatements = mutable.LinkedHashSet[String]() + protected val reusablePerRecordStatements: mutable.LinkedHashSet[String] = + mutable.LinkedHashSet[String]() // map of initial input unboxing expressions that will be added only once // (inputTerm, index) -> expr - protected val reusableInputUnboxingExprs = mutable.Map[(String, Int), GeneratedExpression]() + protected val reusableInputUnboxingExprs: mutable.Map[(String, Int), GeneratedExpression] = + mutable.Map[(String, Int), GeneratedExpression]() // set of constructor statements that will be added only once // we use a LinkedHashSet to keep the insertion order - protected val reusableConstructorStatements = mutable.LinkedHashSet[(String, String)]() + protected val reusableConstructorStatements: mutable.LinkedHashSet[(String, String)] = + mutable.LinkedHashSet[(String, String)]() + + /** + * Flag that indicates that the generated code needed to be split into several methods. + */ + protected var hasCodeSplits: Boolean = false /** * @return code block of statements that need to be placed in the member area of the Function @@ -384,7 +398,7 @@ abstract class CodeGenerator( returnType match { case ri: RowTypeInfo => addReusableOutRecord(ri) - val resultSetters: String = boxedFieldExprs.zipWithIndex map { + val resultSetters = boxedFieldExprs.zipWithIndex map { case (fieldExpr, i) => if (nullCheck) { s""" @@ -403,13 +417,15 @@ abstract class CodeGenerator( |$outRecordTerm.setField($i, ${fieldExpr.resultTerm}); |""".stripMargin } - } mkString "\n" + } + + val code = generateCodeSplits(resultSetters) - GeneratedExpression(outRecordTerm, "false", resultSetters, returnType) + GeneratedExpression(outRecordTerm, NEVER_NULL, code, returnType) case pt: PojoTypeInfo[_] => addReusableOutRecord(pt) - val resultSetters: String = boxedFieldExprs.zip(resultFieldNames) map { + val resultSetters = boxedFieldExprs.zip(resultFieldNames) map { case (fieldExpr, fieldName) => val accessor = getFieldAccessor(pt.getTypeClass, fieldName) @@ -474,13 +490,15 @@ abstract class CodeGenerator( |""".stripMargin } } - } mkString "\n" + } + + val code = generateCodeSplits(resultSetters) - GeneratedExpression(outRecordTerm, "false", resultSetters, returnType) + GeneratedExpression(outRecordTerm, NEVER_NULL, code, returnType) case tup: TupleTypeInfo[_] => addReusableOutRecord(tup) - val resultSetters: String = boxedFieldExprs.zipWithIndex map { + val resultSetters = boxedFieldExprs.zipWithIndex map { case (fieldExpr, i) => val fieldName = "f" + i if (nullCheck) { @@ -500,11 +518,13 @@ abstract class CodeGenerator( |$outRecordTerm.$fieldName = ${fieldExpr.resultTerm}; |""".stripMargin } - } mkString "\n" + } + + val code = generateCodeSplits(resultSetters) - GeneratedExpression(outRecordTerm, "false", resultSetters, returnType) + GeneratedExpression(outRecordTerm, NEVER_NULL, code, returnType) - case cc: CaseClassTypeInfo[_] => + case _: CaseClassTypeInfo[_] => val fieldCodes: String = boxedFieldExprs.map(_.code).mkString("\n") val constructorParams: String = boxedFieldExprs.map(_.resultTerm).mkString(", ") val resultTerm = newName(outRecordTerm) @@ -528,9 +548,10 @@ abstract class CodeGenerator( |$returnTypeTerm $resultTerm = new $returnTypeTerm($constructorParams); |""".stripMargin - GeneratedExpression(resultTerm, "false", resultCode, returnType) + // case classes are not splittable + GeneratedExpression(resultTerm, NEVER_NULL, resultCode, returnType) - case t: TypeInformation[_] => + case _: TypeInformation[_] => val fieldExpr = boxedFieldExprs.head val nullCheckCode = if (nullCheck) { s""" @@ -547,7 +568,8 @@ abstract class CodeGenerator( |$nullCheckCode |""".stripMargin - GeneratedExpression(fieldExpr.resultTerm, "false", resultCode, returnType) + // other types are not splittable + GeneratedExpression(fieldExpr.resultTerm, fieldExpr.nullTerm, resultCode, returnType) case _ => throw new CodeGenException(s"Unsupported result type: $returnType") @@ -1024,6 +1046,55 @@ abstract class CodeGenerator( // generator helping methods // ---------------------------------------------------------------------------------------------- + private def generateCodeSplits(splits: Seq[String]): String = { + val totalLen = splits.map(_.length + 1).sum // 1 for a line break + + // split + if (totalLen > config.getMaxGeneratedCodeLength) { + + hasCodeSplits = true + + // add input unboxing to member area such that all split functions can access it + reusableInputUnboxingExprs.foreach { case (_, expr) => + + // declaration + val resultTypeTerm = primitiveTypeTermForTypeInfo(expr.resultType) + if (nullCheck) { + reusableMemberStatements.add(s"private boolean ${expr.nullTerm};") + } + reusableMemberStatements.add(s"private $resultTypeTerm ${expr.resultTerm};") + + // assignment + if (nullCheck) { + reusablePerRecordStatements.add(s"this.${expr.nullTerm} = ${expr.nullTerm};") + } + reusablePerRecordStatements.add(s"this.${expr.resultTerm} = ${expr.resultTerm};") + } + + // add split methods to the member area and return the code necessary to call those methods + val methodCalls = splits.map { split => + val methodName = newName(s"split") + + val method = + s""" + |private final void $methodName() throws Exception { + | $split + |} + |""".stripMargin + reusableMemberStatements.add(method) + + // create method call + s"$methodName();" + } + + methodCalls.mkString("\n") + } + // don't split + else { + splits.mkString("\n") + } + } + private def generateFieldAccess(refExpr: GeneratedExpression, index: Int): GeneratedExpression = { val fieldAccessExpr = generateFieldAccess( @@ -1644,9 +1715,13 @@ abstract class CodeGenerator( def addReusableTimestamp(): String = { val fieldTerm = s"timestamp" + // declaration + reusableMemberStatements.add(s"private long $fieldTerm;") + + // assignment val field = s""" - |final long $fieldTerm = java.lang.System.currentTimeMillis(); + |$fieldTerm = java.lang.System.currentTimeMillis(); |""".stripMargin reusablePerRecordStatements.add(field) fieldTerm @@ -1660,9 +1735,13 @@ abstract class CodeGenerator( val timestamp = addReusableTimestamp() + // declaration + reusableMemberStatements.add(s"private long $fieldTerm;") + + // assignment val field = s""" - |final long $fieldTerm = $timestamp + java.util.TimeZone.getDefault().getOffset(timestamp); + |$fieldTerm = $timestamp + java.util.TimeZone.getDefault().getOffset($timestamp); |""".stripMargin reusablePerRecordStatements.add(field) fieldTerm @@ -1676,10 +1755,14 @@ abstract class CodeGenerator( val timestamp = addReusableTimestamp() + // declaration + reusableMemberStatements.add(s"private int $fieldTerm;") + + // assignment // adopted from org.apache.calcite.runtime.SqlFunctions.currentTime() val field = s""" - |final int $fieldTerm = (int) ($timestamp % ${DateTimeUtils.MILLIS_PER_DAY}); + |$fieldTerm = (int) ($timestamp % ${DateTimeUtils.MILLIS_PER_DAY}); |if (time < 0) { | time += ${DateTimeUtils.MILLIS_PER_DAY}; |} @@ -1696,10 +1779,14 @@ abstract class CodeGenerator( val localtimestamp = addReusableLocalTimestamp() + // declaration + reusableMemberStatements.add(s"private int $fieldTerm;") + + // assignment // adopted from org.apache.calcite.runtime.SqlFunctions.localTime() val field = s""" - |final int $fieldTerm = (int) ($localtimestamp % ${DateTimeUtils.MILLIS_PER_DAY}); + |$fieldTerm = (int) ($localtimestamp % ${DateTimeUtils.MILLIS_PER_DAY}); |""".stripMargin reusablePerRecordStatements.add(field) fieldTerm @@ -1715,10 +1802,14 @@ abstract class CodeGenerator( val timestamp = addReusableTimestamp() val time = addReusableTime() + // declaration + reusableMemberStatements.add(s"private int $fieldTerm;") + + // assignment // adopted from org.apache.calcite.runtime.SqlFunctions.currentDate() val field = s""" - |final int $fieldTerm = (int) ($timestamp / ${DateTimeUtils.MILLIS_PER_DAY}); + |$fieldTerm = (int) ($timestamp / ${DateTimeUtils.MILLIS_PER_DAY}); |if ($time < 0) { | $fieldTerm -= 1; |} diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala index 70f6638998a449..9fc76e329833f1 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala @@ -61,38 +61,54 @@ class CollectorCodeGenerator( * @return instance of GeneratedCollector */ def generateTableFunctionCollector( - name: String, - bodyCode: String, - collectedType: TypeInformation[Any]) - : GeneratedCollector = { + name: String, + bodyCode: String, + collectedType: TypeInformation[Any]) + : GeneratedCollector = { val className = newName(name) val input1TypeClass = boxedTypeTermForTypeInfo(input1) val input2TypeClass = boxedTypeTermForTypeInfo(collectedType) - val funcCode = j""" - public class $className extends ${classOf[TableFunctionCollector[_]].getCanonicalName} { - - ${reuseMemberCode()} + // declaration in case of code splits + val recordMember = if (hasCodeSplits) { + s"private $input2TypeClass $input2Term;" + } else { + "" + } - public $className() throws Exception { - ${reuseInitCode()} - } + // assignment in case of code splits + val recordAssignment = if (hasCodeSplits) { + s"$input2Term" // use member + } else { + s"$input2TypeClass $input2Term" // local variable + } - @Override - public void collect(Object record) throws Exception { - super.collect(record); - $input1TypeClass $input1Term = ($input1TypeClass) getInput(); - $input2TypeClass $input2Term = ($input2TypeClass) record; - ${reuseInputUnboxingCode()} - $bodyCode - } - - @Override - public void close() { - } - } - """.stripMargin + val funcCode = j""" + |public class $className extends ${classOf[TableFunctionCollector[_]].getCanonicalName} { + | + | $recordMember + | ${reuseMemberCode()} + | + | public $className() throws Exception { + | ${reuseInitCode()} + | } + | + | @Override + | public void collect(Object record) throws Exception { + | super.collect(record); + | $input1TypeClass $input1Term = ($input1TypeClass) getInput(); + | $recordAssignment = ($input2TypeClass) record; + | ${reuseInputUnboxingCode()} + | ${reusePerRecordCode()} + | $bodyCode + | } + | + | @Override + | public void close() { + | } + |} + |""".stripMargin GeneratedCollector(className, funcCode) } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/FunctionCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/FunctionCodeGenerator.scala index 2bd2fe7ef7ef93..8ac18cdda46e1a 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/FunctionCodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/FunctionCodeGenerator.scala @@ -24,6 +24,7 @@ import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.table.api.TableConfig import org.apache.flink.table.codegen.CodeGenUtils.{boxedTypeTermForTypeInfo, newName} import org.apache.flink.table.codegen.Indenter.toISC +import org.apache.flink.util.Collector /** * A code generator for generating Flink [[org.apache.flink.api.common.functions.Function]]s. @@ -85,22 +86,23 @@ class FunctionCodeGenerator( * @return instance of GeneratedFunction */ def generateFunction[F <: Function, T <: Any]( - name: String, - clazz: Class[F], - bodyCode: String, - returnType: TypeInformation[T]) - : GeneratedFunction[F, T] = { + name: String, + clazz: Class[F], + bodyCode: String, + returnType: TypeInformation[T]) + : GeneratedFunction[F, T] = { val funcName = newName(name) + val collectorTypeTerm = classOf[Collector[Any]].getCanonicalName // Janino does not support generics, that's why we need // manual casting here - val samHeader = + val (functionClass, signature, inputStatements) = // FlatMapFunction if (clazz == classOf[FlatMapFunction[_, _]]) { val baseClass = classOf[RichFlatMapFunction[_, _]] val inputTypeTerm = boxedTypeTermForTypeInfo(input1) (baseClass, - s"void flatMap(Object _in1, org.apache.flink.util.Collector $collectorTerm)", + s"void flatMap(Object _in1, $collectorTypeTerm $collectorTerm)", List(s"$inputTypeTerm $input1Term = ($inputTypeTerm) _in1;")) } @@ -120,7 +122,7 @@ class FunctionCodeGenerator( val inputTypeTerm2 = boxedTypeTermForTypeInfo(input2.getOrElse( throw new CodeGenException("Input 2 for FlatJoinFunction should not be null"))) (baseClass, - s"void join(Object _in1, Object _in2, org.apache.flink.util.Collector $collectorTerm)", + s"void join(Object _in1, Object _in2, $collectorTypeTerm $collectorTerm)", List(s"$inputTypeTerm1 $input1Term = ($inputTypeTerm1) _in1;", s"$inputTypeTerm2 $input2Term = ($inputTypeTerm2) _in2;")) } @@ -141,11 +143,22 @@ class FunctionCodeGenerator( else if (clazz == classOf[ProcessFunction[_, _]]) { val baseClass = classOf[ProcessFunction[_, _]] val inputTypeTerm = boxedTypeTermForTypeInfo(input1) + val contextTypeTerm = classOf[ProcessFunction[Any, Any]#Context].getCanonicalName + + // make context accessible also for split code + val globalContext = if (hasCodeSplits) { + // declaration + reusableMemberStatements.add(s"private $contextTypeTerm $contextTerm;") + // assignment + List(s"this.$contextTerm = $contextTerm;") + } else { + Nil + } + (baseClass, - s"void processElement(Object _in1, " + - s"org.apache.flink.streaming.api.functions.ProcessFunction.Context $contextTerm," + - s"org.apache.flink.util.Collector $collectorTerm)", - List(s"$inputTypeTerm $input1Term = ($inputTypeTerm) _in1;")) + s"void processElement(Object _in1, $contextTypeTerm $contextTerm, " + + s"$collectorTypeTerm $collectorTerm)", + List(s"$inputTypeTerm $input1Term = ($inputTypeTerm) _in1;") ++ globalContext) } else { // TODO more functions @@ -153,36 +166,35 @@ class FunctionCodeGenerator( } val funcCode = j""" - public class $funcName - extends ${samHeader._1.getCanonicalName} { - - ${reuseMemberCode()} - - public $funcName() throws Exception { - ${reuseInitCode()} - } - - ${reuseConstructorCode(funcName)} - - @Override - public void open(${classOf[Configuration].getCanonicalName} parameters) throws Exception { - ${reuseOpenCode()} - } - - @Override - public ${samHeader._2} throws Exception { - ${samHeader._3.mkString("\n")} - ${reusePerRecordCode()} - ${reuseInputUnboxingCode()} - $bodyCode - } - - @Override - public void close() throws Exception { - ${reuseCloseCode()} - } - } - """.stripMargin + |public class $funcName extends ${functionClass.getCanonicalName} { + | + | ${reuseMemberCode()} + | + | public $funcName() throws Exception { + | ${reuseInitCode()} + | } + | + | ${reuseConstructorCode(funcName)} + | + | @Override + | public void open(${classOf[Configuration].getCanonicalName} parameters) throws Exception { + | ${reuseOpenCode()} + | } + | + | @Override + | public $signature throws Exception { + | ${inputStatements.mkString("\n")} + | ${reuseInputUnboxingCode()} + | ${reusePerRecordCode()} + | $bodyCode + | } + | + | @Override + | public void close() throws Exception { + | ${reuseCloseCode()} + | } + |} + |""".stripMargin GeneratedFunction(funcName, returnType, funcCode) } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/InputFormatCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/InputFormatCodeGenerator.scala index 6d6e1b675b063c..30d33005b2ea33 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/InputFormatCodeGenerator.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/InputFormatCodeGenerator.scala @@ -71,12 +71,16 @@ class InputFormatCodeGenerator( } @Override - public Object nextRecord(Object reuse) { + public Object nextRecord(Object reuse) throws java.io.IOException { switch (nextIdx) { ${records.zipWithIndex.map { case (r, i) => s""" |case $i: + |try { | $r + |} catch (Exception e) { + | throw new java.io.IOException(e); + |} |break; """.stripMargin }.mkString("\n")} diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala index 828a9e2654a625..b385015102ce3c 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/CorrelateITCase.scala @@ -81,6 +81,21 @@ class CorrelateITCase( TestBaseUtils.compareResultAsText(results.asJava, expected) } + @Test + def testLeftOuterJoinWithSplit(): Unit = { + val env = ExecutionEnvironment.getExecutionEnvironment + val tableEnv = TableEnvironment.getTableEnvironment(env, config) + tableEnv.getConfig.setMaxGeneratedCodeLength(1) // split every field + val in = testData(env).toTable(tableEnv).as('a, 'b, 'c) + + val func2 = new TableFunc2 + val result = in.leftOuterJoin(func2('c) as ('s, 'l)).select('c, 's, 'l).toDataSet[Row] + val results = result.collect() + val expected = "Jack#22,Jack,4\n" + "Jack#22,22,2\n" + "John#19,John,4\n" + + "John#19,19,2\n" + "Anna#44,Anna,4\n" + "Anna#44,44,2\n" + "nosharp,null,null" + TestBaseUtils.compareResultAsText(results.asJava, expected) + } + /** * Common join predicates are temporarily forbidden (see FLINK-7865). */ diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala index 1e2cf9c602183d..b7950b7d9929e1 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala @@ -173,7 +173,7 @@ class SqlITCase extends StreamingWithStateTestBase { val t = StreamTestData.get3TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c) tEnv.registerTable("MyTable", t) - val result = tEnv.sql(sqlQuery).toRetractStream[Row] + val result = tEnv.sqlQuery(sqlQuery).toRetractStream[Row] result.addSink(new StreamITCase.RetractingSink).setParallelism(1) env.execute() @@ -208,7 +208,7 @@ class SqlITCase extends StreamingWithStateTestBase { tEnv.registerTable("MyTable", env.fromCollection(data).toTable(tEnv).as('a, 'b, 'c)) - val result = tEnv.sql(sqlQuery).toRetractStream[Row] + val result = tEnv.sqlQuery(sqlQuery).toRetractStream[Row] result.addSink(new StreamITCase.RetractingSink).setParallelism(1) env.execute() @@ -261,6 +261,27 @@ class SqlITCase extends StreamingWithStateTestBase { assertEquals(expected.sorted, StreamITCase.testResults.sorted) } + @Test + def testSelectExpressionWithSplitFromTable(): Unit = { + + val env = StreamExecutionEnvironment.getExecutionEnvironment + val tEnv = TableEnvironment.getTableEnvironment(env) + tEnv.getConfig.setMaxGeneratedCodeLength(1) // split every field + StreamITCase.clear + + val sqlQuery = "SELECT a * 2, b - 1 FROM MyTable" + + val t = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv).as('a, 'b, 'c) + tEnv.registerTable("MyTable", t) + + val result = tEnv.sqlQuery(sqlQuery).toAppendStream[Row] + result.addSink(new StreamITCase.StringSink[Row]) + env.execute() + + val expected = List("2,0", "4,1", "6,1") + assertEquals(expected.sorted, StreamITCase.testResults.sorted) + } + /** test filtering with registered table **/ @Test def testSimpleFilter(): Unit = { @@ -580,7 +601,7 @@ class SqlITCase extends StreamingWithStateTestBase { tEnv.registerTable("T1", t1) - val result = tEnv.sql(sqlQuery).toAppendStream[Row] + val result = tEnv.sqlQuery(sqlQuery).toAppendStream[Row] result.addSink(new StreamITCase.StringSink[Row]) env.execute() @@ -638,6 +659,33 @@ class SqlITCase extends StreamingWithStateTestBase { "3,3300") assertEquals(expected.sorted, StreamITCase.testResults.sorted) } + + @Test + def testVeryBigQuery(): Unit = { + + val env = StreamExecutionEnvironment.getExecutionEnvironment + val tEnv = TableEnvironment.getTableEnvironment(env) + StreamITCase.clear + + val t = StreamTestData.getSingletonDataStream(env).toTable(tEnv).as('a, 'b, 'c) + tEnv.registerTable("MyTable", t) + + val sqlQuery = new StringBuilder + sqlQuery.append("SELECT ") + val expected = new StringBuilder + for (i <- 0 until 500) { + sqlQuery.append(s"a + b + $i, ") + expected.append((1 + 42L + i).toString + ",") + } + sqlQuery.append("c FROM MyTable") + expected.append("Hi") + + val result = tEnv.sqlQuery(sqlQuery.toString()).toAppendStream[Row] + result.addSink(new StreamITCase.StringSink[Row]) + env.execute() + + assertEquals(List(expected.toString()), StreamITCase.testResults.sorted) + } } object SqlITCase { diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala index 58d3c635a66bd5..ef98791716afa4 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala @@ -25,6 +25,12 @@ import scala.collection.mutable object StreamTestData { + def getSingletonDataStream(env: StreamExecutionEnvironment): DataStream[(Int, Long, String)] = { + val data = new mutable.MutableList[(Int, Long, String)] + data.+=((1, 42L, "Hi")) + env.fromCollection(data) + } + def getSmall3TupleDataStream(env: StreamExecutionEnvironment): DataStream[(Int, Long, String)] = { val data = new mutable.MutableList[(Int, Long, String)] data.+=((1, 1L, "Hi")) diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala index 5cfab4aabac6d0..b3eeb59bce5f2c 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala @@ -18,6 +18,7 @@ package org.apache.flink.table.runtime.utils import org.apache.flink.contrib.streaming.state.RocksDBStateBackend +import org.apache.flink.runtime.state.StateBackend import org.apache.flink.test.util.AbstractTestBase import org.junit.Rule import org.junit.rules.TemporaryFolder @@ -29,7 +30,7 @@ class StreamingWithStateTestBase extends AbstractTestBase { @Rule def tempFolder: TemporaryFolder = _tempFolder - def getStateBackend: RocksDBStateBackend = { + def getStateBackend: StateBackend = { val dbPath = tempFolder.newFolder().getAbsolutePath val checkpointPath = tempFolder.newFolder().toURI.toString val backend = new RocksDBStateBackend(checkpointPath) From a389b4358125b68c83beb01966cec0cfc19e0964 Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Mon, 12 Mar 2018 11:23:40 +0100 Subject: [PATCH 0158/2294] [FLINK-8922] Revert "[FLINK-8859][checkpointing] RocksDB backend should pass WriteOption to Rocks.put() when restoring" We need to revert FLINK-8859 because it causes problems with RocksDB that make our automated tests fail on Travis. The change looks actually good and it is currently unclear why this can introduce such a problem. This might also be a Rocks in RocksDB. Nevertheless, for the sake of a proper release testing, we should revert the change for now. --- .../contrib/streaming/state/RocksDBKeyedStateBackend.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 5444dee443851c..8f95b1812d844b 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -680,7 +680,7 @@ private void restoreKVStateData() throws IOException, RocksDBException { if (RocksDBFullSnapshotOperation.hasMetaDataFollowsFlag(key)) { //clear the signal bit in the key to make it ready for insertion again RocksDBFullSnapshotOperation.clearMetaDataFollowsFlag(key); - rocksDBKeyedStateBackend.db.put(handle, rocksDBKeyedStateBackend.writeOptions, key, value); + rocksDBKeyedStateBackend.db.put(handle, key, value); //TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible kvStateId = RocksDBFullSnapshotOperation.END_OF_KEY_GROUP_MARK & compressedKgInputView.readShort(); @@ -690,7 +690,7 @@ private void restoreKVStateData() throws IOException, RocksDBException { handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); } } else { - rocksDBKeyedStateBackend.db.put(handle, rocksDBKeyedStateBackend.writeOptions, key, value); + rocksDBKeyedStateBackend.db.put(handle, key, value); } } } @@ -1091,7 +1091,6 @@ private void restoreKeyGroupsShardWithTemporaryHelperInstance( if (stateBackend.keyGroupRange.contains(keyGroup)) { stateBackend.db.put(targetColumnFamilyHandle, - stateBackend.writeOptions, iterator.key(), iterator.value()); } From 7cea4252e7ec40675db0302ef5c27f226944fd0e Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 12 Mar 2018 12:45:37 +0100 Subject: [PATCH 0159/2294] [hotfix][RAT] Add serializer snapshot to exclusions --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index f8649186c130f8..d83c81b2684a98 100644 --- a/pom.xml +++ b/pom.xml @@ -1024,6 +1024,7 @@ under the License. **/src/test/resources/*-savepoint flink-core/src/test/resources/serialized-kryo-serializer-1.3 flink-core/src/test/resources/type-without-avro-serialized-using-kryo + flink-formats/flink-avro/src/test/resources/flink-1.4-serializer-java-serialized flink-formats/flink-avro/src/test/resources/testdata.avro flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/generated/*.java From d5fc25e6773625ba5e92cc6f4dd604f0c089139c Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 12 Mar 2018 15:08:36 +0100 Subject: [PATCH 0160/2294] [hotfix][tests] Strip CompletionExceptions in MiniClusterClient#guardWithSingleRetry --- .../org/apache/flink/client/program/MiniClusterClient.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 7475071e23e6e3..f0a7631023ea46 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -38,6 +38,7 @@ import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.apache.flink.runtime.util.LeaderConnectionInfo; import org.apache.flink.runtime.util.LeaderRetrievalUtils; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; import javax.annotation.Nonnull; @@ -197,7 +198,10 @@ private static CompletableFuture guardWithSingleRetry(Supplier throwable instanceof FencingTokenException || throwable instanceof AkkaRpcException, + throwable -> { + Throwable actualException = ExceptionUtils.stripCompletionException(throwable); + return actualException instanceof FencingTokenException || actualException instanceof AkkaRpcException; + }, executor); } } From 961df0d6c1d6495560f2e27f771f71491bd608e5 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 12 Mar 2018 16:28:14 +0100 Subject: [PATCH 0161/2294] [hotfix] Don't mark Table API & SQL as 'beta' anymore --- docs/dev/table/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/table/index.md b/docs/dev/table/index.md index 5845c95099cbb3..34e94684ecf3cf 100644 --- a/docs/dev/table/index.md +++ b/docs/dev/table/index.md @@ -2,7 +2,7 @@ title: "Table API & SQL" nav-id: tableapi nav-parent_id: dev -is_beta: true +is_beta: false nav-show_overview: true nav-pos: 35 --- From 7d837a3e884eba9937fb4b14fd9c76e8895d5703 Mon Sep 17 00:00:00 2001 From: mingleiZhang Date: Wed, 7 Mar 2018 10:36:52 +0800 Subject: [PATCH 0162/2294] [FLINK-8687] [sql-client] Make MaterializedCollectStreamResult#retrievePage to have resultLock This closes #5647. --- .../gateway/local/MaterializedCollectStreamResult.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java index bd7f08ee0513f7..7935da63e0bf8f 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java @@ -86,11 +86,13 @@ else if (!isRetrieving()) { @Override public List retrievePage(int page) { - if (page <= 0 || page > pageCount) { - throw new SqlExecutionException("Invalid page '" + page + "'."); - } + synchronized (resultLock) { + if (page <= 0 || page > pageCount) { + throw new SqlExecutionException("Invalid page '" + page + "'."); + } - return snapshot.subList(pageSize * (page - 1), Math.min(snapshot.size(), pageSize * page)); + return snapshot.subList(pageSize * (page - 1), Math.min(snapshot.size(), pageSize * page)); + } } // -------------------------------------------------------------------------------------------- From f7ae309d3967e61a719fadd8261d515a19ed078e Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Mon, 12 Mar 2018 18:38:03 +0100 Subject: [PATCH 0163/2294] [hotfix] [yarn] Improve logging of container resources --- .../main/java/org/apache/flink/yarn/YarnResourceManager.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 46ef81bed17071..af789baf7e95a3 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -438,7 +438,10 @@ private void requestYarnContainer(Resource resource, Priority priority) { resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); numPendingContainerRequests++; - log.info("Requesting new TaskManager container pending requests: {}", numPendingContainerRequests); + + log.info("Requesting new TaskExecutor container with resources {}. Number pending requests {}.", + resource, + numPendingContainerRequests); } private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host) From 3debf47e5d042b73ece9a1ba204a30e997805919 Mon Sep 17 00:00:00 2001 From: sihuazhou Date: Mon, 12 Mar 2018 22:04:57 +0800 Subject: [PATCH 0164/2294] [FLINK-8927][checkpointing] Eagerly release the checkpoint object in RocksDB incremental snapshots This closes #5682. --- .../contrib/streaming/state/RocksDBKeyedStateBackend.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 8f95b1812d844b..6a661210ec703f 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -2318,8 +2318,9 @@ void takeSnapshot() throws Exception { } // create hard links of living files in the snapshot path - Checkpoint checkpoint = Checkpoint.create(stateBackend.db); - checkpoint.createCheckpoint(localBackupDirectory.getDirectory().getPath()); + try (Checkpoint checkpoint = Checkpoint.create(stateBackend.db)) { + checkpoint.createCheckpoint(localBackupDirectory.getDirectory().getPath()); + } } @Nonnull From 2d19d11007d37298dffb78f3aa43d749f9e597ce Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Mon, 12 Mar 2018 18:04:38 +0100 Subject: [PATCH 0165/2294] [FLINK-8783] [tests] Harden SlotPoolRpcTest Wait for releasing of timed out pending slot requests before checking the number of pending slots requests. This closes #5684. --- .../runtime/jobmaster/slotpool/SlotPoolRpcTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java index cc837bc0496529..4c736e8ba0284b 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolRpcTest.java @@ -195,6 +195,9 @@ public void testSlotAllocationTimeout() throws Exception { pool.start(JobMasterId.generate(), "foobar"); SlotPoolGateway slotPoolGateway = pool.getSelfGateway(SlotPoolGateway.class); + final CompletableFuture slotRequestTimeoutFuture = new CompletableFuture<>(); + pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete); + ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway(); pool.connectToResourceManager(resourceManagerGateway); @@ -213,6 +216,9 @@ public void testSlotAllocationTimeout() throws Exception { assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException); } + // wait until we have timed out the slot request + slotRequestTimeoutFuture.get(); + assertEquals(0L, (long) pool.getNumberOfPendingRequests().get()); } finally { RpcUtils.terminateRpcEndpoint(pool, timeout); @@ -243,6 +249,9 @@ public void testExtraSlotsAreKept() throws Exception { resourceManagerGateway.setRequestSlotConsumer( (SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId())); + final CompletableFuture slotRequestTimeoutFuture = new CompletableFuture<>(); + pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete); + pool.connectToResourceManager(resourceManagerGateway); SlotRequestId requestId = new SlotRequestId(); @@ -260,6 +269,9 @@ public void testExtraSlotsAreKept() throws Exception { assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException); } + // wait until we have timed out the slot request + slotRequestTimeoutFuture.get(); + assertEquals(0L, (long) pool.getNumberOfPendingRequests().get()); AllocationID allocationId = allocationIdFuture.get(); From 445cdfd57941c4c888a6e73356558fd5f11d443c Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 13 Mar 2018 08:13:44 +0100 Subject: [PATCH 0166/2294] [FLINK-8934] [flip6] Properly cancel slot requests of otherwisely fulfilled requests Cancel slot requests at the ResourceManager if they have been completed with a different allocation. This closes #5687. --- .../runtime/jobmaster/slotpool/SlotPool.java | 2 +- .../jobmaster/slotpool/SlotPoolTest.java | 26 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java index 8a2dd45ea7ef83..42264b53219898 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java @@ -691,7 +691,7 @@ private void requestSlotFromResourceManager( pendingRequest.getAllocatedSlotFuture().whenComplete( (AllocatedSlot allocatedSlot, Throwable throwable) -> { - if (throwable != null || allocationId.equals(allocatedSlot.getAllocationId())) { + if (throwable != null || !allocationId.equals(allocatedSlot.getAllocationId())) { // cancel the slot request if there is a failure or if the pending request has // been completed with another allocated slot resourceManagerGateway.cancelSlotRequest(allocationId); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java index c529ceb3c1a004..c3819747595b8c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java @@ -457,16 +457,21 @@ public void testSlotRequestCancellationUponFailingRequest() throws Exception { * Tests that unused offered slots are directly used to fulfill pending slot * requests. * - *

    See FLINK-8089 + * Moreover it tests that the old slot request is canceled + * + *

    See FLINK-8089, FLINK-8934 */ @Test public void testFulfillingSlotRequestsWithUnusedOfferedSlots() throws Exception { final SlotPool slotPool = new SlotPool(rpcService, jobId); - final CompletableFuture allocationIdFuture = new CompletableFuture<>(); + final ArrayBlockingQueue allocationIds = new ArrayBlockingQueue<>(2); resourceManagerGateway.setRequestSlotConsumer( - (SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId())); + (SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId())); + + final ArrayBlockingQueue canceledAllocations = new ArrayBlockingQueue<>(2); + resourceManagerGateway.setCancelSlotConsumer(canceledAllocations::offer); final SlotRequestId slotRequestId1 = new SlotRequestId(); final SlotRequestId slotRequestId2 = new SlotRequestId(); @@ -487,7 +492,7 @@ public void testFulfillingSlotRequestsWithUnusedOfferedSlots() throws Exception timeout); // wait for the first slot request - final AllocationID allocationId = allocationIdFuture.get(); + final AllocationID allocationId1 = allocationIds.take(); CompletableFuture slotFuture2 = slotPoolGateway.allocateSlot( slotRequestId2, @@ -496,6 +501,9 @@ public void testFulfillingSlotRequestsWithUnusedOfferedSlots() throws Exception true, timeout); + // wait for the second slot request + final AllocationID allocationId2 = allocationIds.take(); + slotPoolGateway.releaseSlot(slotRequestId1, null, null); try { @@ -505,17 +513,21 @@ public void testFulfillingSlotRequestsWithUnusedOfferedSlots() throws Exception } catch (ExecutionException ee) { // expected assertTrue(ExceptionUtils.stripExecutionException(ee) instanceof FlinkException); - } - final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.UNKNOWN); + assertEquals(allocationId1, canceledAllocations.take()); + + final SlotOffer slotOffer = new SlotOffer(allocationId1, 0, ResourceProfile.UNKNOWN); slotPoolGateway.registerTaskManager(taskManagerLocation.getResourceID()).get(); assertTrue(slotPoolGateway.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer).get()); // the slot offer should fulfill the second slot request - assertEquals(allocationId, slotFuture2.get().getAllocationId()); + assertEquals(allocationId1, slotFuture2.get().getAllocationId()); + + // check that the second slot allocation has been canceled + assertEquals(allocationId2, canceledAllocations.take()); } finally { RpcUtils.terminateRpcEndpoint(slotPool, timeout); } From 6f46d6db182fa768417ac1397c0abe3f30da1942 Mon Sep 17 00:00:00 2001 From: zjureel Date: Thu, 7 Sep 2017 10:39:39 +0800 Subject: [PATCH 0167/2294] [FLINK-7521] Add config option to set the content length limit of REST server and client --- .../flink/configuration/RestOptions.java | 15 ++++++++++++ .../apache/flink/runtime/rest/RestClient.java | 2 +- .../runtime/rest/RestClientConfiguration.java | 24 +++++++++++++++++-- .../runtime/rest/RestServerEndpoint.java | 5 +++- .../rest/RestServerEndpointConfiguration.java | 22 +++++++++++++++-- 5 files changed, 62 insertions(+), 6 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java index 888be082398888..61bb0853471bb2 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java @@ -81,4 +81,19 @@ public class RestOptions { key("rest.connection-timeout") .defaultValue(15_000L) .withDescription("The maximum time in ms for the client to establish a TCP connection."); + + /** + * The max content length that the server will handle. + */ + public static final ConfigOption REST_SERVER_CONTENT_MAX_MB = + key("rest.server.content.max.mb") + .defaultValue(10); + + /** + * The max content length that the client will handle. + */ + public static final ConfigOption REST_CLIENT_CONTENT_MAX_MB = + key("rest.client.content.max.mb") + .defaultValue(1); + } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java index 3a0f6df70ac255..801119dfde40e4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java @@ -103,7 +103,7 @@ protected void initChannel(SocketChannel socketChannel) throws Exception { socketChannel.pipeline() .addLast(new HttpClientCodec()) - .addLast(new HttpObjectAggregator(1024 * 1024)) + .addLast(new HttpObjectAggregator(configuration.getMaxContentLength())) .addLast(new ClientHandler()) .addLast(new PipelineErrorHandler(LOG)); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java index 86578a2ce726fb..782cb4e518e1f8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java @@ -39,9 +39,15 @@ public final class RestClientConfiguration { private final long connectionTimeout; - private RestClientConfiguration(@Nullable SSLEngine sslEngine, final long connectionTimeout) { + private final int maxContentLength; + + private RestClientConfiguration( + @Nullable final SSLEngine sslEngine, + final long connectionTimeout, + final int maxContentLength) { this.sslEngine = sslEngine; this.connectionTimeout = connectionTimeout; + this.maxContentLength = maxContentLength; } /** @@ -61,6 +67,15 @@ public long getConnectionTimeout() { return connectionTimeout; } + /** + * Returns the max content length that the REST client endpoint could handle. + * + * @return max content length that the REST client endpoint could handle + */ + public int getMaxContentLength() { + return maxContentLength; + } + /** * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}. * @@ -89,6 +104,11 @@ public static RestClientConfiguration fromConfiguration(Configuration config) th final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT); - return new RestClientConfiguration(sslEngine, connectionTimeout); + int maxContentLength = config.getInteger(RestOptions.REST_CLIENT_CONTENT_MAX_MB) * 1024 * 1024; + if (maxContentLength <= 0) { + throw new ConfigurationException("Max content length for client must be a positive integer: " + maxContentLength); + } + + return new RestClientConfiguration(sslEngine, connectionTimeout, maxContentLength); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java index f131ec186cbcfb..42af4c6ee4ae9c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java @@ -78,6 +78,7 @@ public abstract class RestServerEndpoint { private final int configuredPort; private final SSLEngine sslEngine; private final Path uploadDir; + private final int maxContentLength; private final CompletableFuture terminationFuture; @@ -96,6 +97,8 @@ public RestServerEndpoint(RestServerEndpointConfiguration configuration) throws this.uploadDir = configuration.getUploadDir(); createUploadDir(uploadDir, log); + this.maxContentLength = configuration.getMaxContentLength(); + terminationFuture = new CompletableFuture<>(); this.restAddress = null; @@ -156,7 +159,7 @@ protected void initChannel(SocketChannel ch) { ch.pipeline() .addLast(new HttpServerCodec()) .addLast(new FileUploadHandler(uploadDir)) - .addLast(new HttpObjectAggregator(MAX_REQUEST_SIZE_BYTES)) + .addLast(new HttpObjectAggregator(maxContentLength)) .addLast(handler.name(), handler) .addLast(new PipelineErrorHandler(log)); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java index c411b51cea75f4..3685e2d656e50e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java @@ -51,11 +51,14 @@ public final class RestServerEndpointConfiguration { private final Path uploadDir; + private final int maxContentLength; + private RestServerEndpointConfiguration( @Nullable String restBindAddress, int restBindPort, @Nullable SSLEngine sslEngine, - final Path uploadDir) { + final Path uploadDir, + final int maxContentLength) { Preconditions.checkArgument(0 <= restBindPort && restBindPort < 65536, "The bing rest port " + restBindPort + " is out of range (0, 65536["); @@ -63,6 +66,7 @@ private RestServerEndpointConfiguration( this.restBindPort = restBindPort; this.sslEngine = sslEngine; this.uploadDir = requireNonNull(uploadDir); + this.maxContentLength = maxContentLength; } /** @@ -99,6 +103,15 @@ public Path getUploadDir() { return uploadDir; } + /** + * Returns the max content length that the REST server endpoint could handle. + * + * @return max content length that the REST server endpoint could handle + */ + public int getMaxContentLength() { + return maxContentLength; + } + /** * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}. * @@ -131,6 +144,11 @@ public static RestServerEndpointConfiguration fromConfiguration(Configuration co config.getString(WebOptions.UPLOAD_DIR, config.getString(WebOptions.TMP_DIR)), "flink-web-upload-" + UUID.randomUUID()); - return new RestServerEndpointConfiguration(address, port, sslEngine, uploadDir); + int maxContentLength = config.getInteger(RestOptions.REST_SERVER_CONTENT_MAX_MB) * 1024 * 1024; + if (maxContentLength <= 0) { + throw new ConfigurationException("Max content length for server must be a positive integer: " + maxContentLength); + } + + return new RestServerEndpointConfiguration(address, port, sslEngine, uploadDir, maxContentLength); } } From 9905700ad3f461d93862eb469ccda1682351579f Mon Sep 17 00:00:00 2001 From: gyao Date: Mon, 12 Mar 2018 15:44:27 +0100 Subject: [PATCH 0168/2294] [FLINK-7521][flip6] Remove RestServerEndpoint#MAX_REQUEST_SIZE_BYTES --- .../apache/flink/runtime/rest/RestServerEndpoint.java | 1 - .../runtime/rest/messages/job/JobSubmitRequestBody.java | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java index 42af4c6ee4ae9c..8b392508aeaced 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java @@ -69,7 +69,6 @@ */ public abstract class RestServerEndpoint { - public static final int MAX_REQUEST_SIZE_BYTES = 1024 * 1024 * 10; protected final Logger log = LoggerFactory.getLogger(getClass()); private final Object lock = new Object(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobSubmitRequestBody.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobSubmitRequestBody.java index c472ff103d1c52..3f550f0baa7753 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobSubmitRequestBody.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/JobSubmitRequestBody.java @@ -19,7 +19,6 @@ package org.apache.flink.runtime.rest.messages.job; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.rest.RestServerEndpoint; import org.apache.flink.runtime.rest.messages.RequestBody; import org.apache.flink.util.Preconditions; @@ -52,13 +51,7 @@ public JobSubmitRequestBody(JobGraph jobGraph) throws IOException { @JsonCreator public JobSubmitRequestBody( - @JsonProperty(FIELD_NAME_SERIALIZED_JOB_GRAPH) byte[] serializedJobGraph) { - - // check that job graph can be read completely by the HttpObjectAggregator on the server - // we subtract 1024 bytes to account for http headers and such. - if (serializedJobGraph.length > RestServerEndpoint.MAX_REQUEST_SIZE_BYTES - 1024) { - throw new IllegalArgumentException("Serialized job graph exceeded max request size."); - } + @JsonProperty(FIELD_NAME_SERIALIZED_JOB_GRAPH) byte[] serializedJobGraph) { this.serializedJobGraph = Preconditions.checkNotNull(serializedJobGraph); } From da3fc4fde2796af262dd275f3ea87a5b7bc69c5a Mon Sep 17 00:00:00 2001 From: gyao Date: Mon, 12 Mar 2018 23:16:25 +0100 Subject: [PATCH 0169/2294] [FLINK-7521][flip6] Return HTTP 413 if request limit is exceeded. Remove unnecessary PipelineErrorHandler from RestClient. Rename config keys for configuring request and response limits. Set response headers for all error responses. This closes #5685. --- .../flink/configuration/RestOptions.java | 18 +-- .../dispatcher/DispatcherRestEndpoint.java | 2 - .../rest/FlinkHttpObjectAggregator.java | 67 +++++++++++ .../apache/flink/runtime/rest/RestClient.java | 16 ++- .../runtime/rest/RestClientConfiguration.java | 8 +- .../runtime/rest/RestServerEndpoint.java | 10 +- .../rest/RestServerEndpointConfiguration.java | 34 +++++- .../rest/handler/PipelineErrorHandler.java | 16 ++- .../handler/RestHandlerConfiguration.java | 22 +--- .../runtime/rest/handler/RouterHandler.java | 13 +- .../rest/handler/util/HandlerUtils.java | 56 ++++++++- .../webmonitor/WebMonitorEndpoint.java | 2 - .../rest/RestServerEndpointITCase.java | 113 +++++++++++++++--- ...btaskCurrentAttemptDetailsHandlerTest.java | 2 +- ...ecutionAttemptAccumulatorsHandlerTest.java | 3 +- ...askExecutionAttemptDetailsHandlerTest.java | 2 +- 16 files changed, 304 insertions(+), 80 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/FlinkHttpObjectAggregator.java diff --git a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java index 61bb0853471bb2..94d7977b72541f 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java @@ -83,17 +83,19 @@ public class RestOptions { .withDescription("The maximum time in ms for the client to establish a TCP connection."); /** - * The max content length that the server will handle. + * The maximum content length that the server will handle. */ - public static final ConfigOption REST_SERVER_CONTENT_MAX_MB = - key("rest.server.content.max.mb") - .defaultValue(10); + public static final ConfigOption REST_SERVER_MAX_CONTENT_LENGTH = + key("rest.server.max-content-length") + .defaultValue(104_857_600) + .withDescription("The maximum content length in bytes that the server will handle."); /** - * The max content length that the client will handle. + * The maximum content length that the client will handle. */ - public static final ConfigOption REST_CLIENT_CONTENT_MAX_MB = - key("rest.client.content.max.mb") - .defaultValue(1); + public static final ConfigOption REST_CLIENT_MAX_CONTENT_LENGTH = + key("rest.client.max-content-length") + .defaultValue(104_857_600) + .withDescription("The maximum content length in bytes that the client will handle."); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java index 9df6deec49fcac..45185528395d31 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java @@ -44,7 +44,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.List; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -92,7 +91,6 @@ protected List> initiali // Add the Dispatcher specific handlers final Time timeout = restConfiguration.getTimeout(); - final Map responseHeaders = restConfiguration.getResponseHeaders(); BlobServerPortHandler blobServerPortHandler = new BlobServerPortHandler( restAddressFuture, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FlinkHttpObjectAggregator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FlinkHttpObjectAggregator.java new file mode 100644 index 00000000000000..4ee0256cbe20fa --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/FlinkHttpObjectAggregator.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest; + +import org.apache.flink.configuration.RestOptions; +import org.apache.flink.runtime.rest.handler.util.HandlerUtils; +import org.apache.flink.runtime.rest.messages.ErrorResponseBody; + +import org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.flink.shaded.netty4.io.netty.handler.codec.TooLongFrameException; +import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpObject; +import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus; + +import javax.annotation.Nonnull; + +import java.util.List; +import java.util.Map; + +/** + * Same as {@link org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpObjectDecoder} + * but returns HTTP 413 to the client if the payload exceeds {@link #maxContentLength}. + */ +public class FlinkHttpObjectAggregator extends org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpObjectAggregator { + + private final Map responseHeaders; + + public FlinkHttpObjectAggregator(final int maxContentLength, @Nonnull final Map responseHeaders) { + super(maxContentLength); + this.responseHeaders = responseHeaders; + } + + @Override + protected void decode( + final ChannelHandlerContext ctx, + final HttpObject msg, + final List out) throws Exception { + + try { + super.decode(ctx, msg, out); + } catch (final TooLongFrameException e) { + HandlerUtils.sendErrorResponse( + ctx, + false, + new ErrorResponseBody(String.format( + e.getMessage() + " Try to raise [%s]", + RestOptions.REST_SERVER_MAX_CONTENT_LENGTH.key())), + HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, + responseHeaders); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java index 801119dfde40e4..6319634fe7587f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClient.java @@ -20,7 +20,7 @@ import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.runtime.rest.handler.PipelineErrorHandler; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.runtime.rest.messages.ErrorResponseBody; import org.apache.flink.runtime.rest.messages.MessageHeaders; import org.apache.flink.runtime.rest.messages.MessageParameters; @@ -50,6 +50,7 @@ import org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoopGroup; import org.apache.flink.shaded.netty4.io.netty.channel.socket.SocketChannel; import org.apache.flink.shaded.netty4.io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.flink.shaded.netty4.io.netty.handler.codec.TooLongFrameException; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.DefaultFullHttpRequest; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.FullHttpRequest; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.FullHttpResponse; @@ -104,8 +105,7 @@ protected void initChannel(SocketChannel socketChannel) throws Exception { socketChannel.pipeline() .addLast(new HttpClientCodec()) .addLast(new HttpObjectAggregator(configuration.getMaxContentLength())) - .addLast(new ClientHandler()) - .addLast(new PipelineErrorHandler(LOG)); + .addLast(new ClientHandler()); } }; NioEventLoopGroup group = new NioEventLoopGroup(1, new DefaultThreadFactory("flink-rest-client-netty")); @@ -269,8 +269,14 @@ protected void channelRead0(ChannelHandlerContext ctx, Object msg) { } @Override - public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) throws Exception { - jsonFuture.completeExceptionally(cause); + public void exceptionCaught(final ChannelHandlerContext ctx, final Throwable cause) { + if (cause instanceof TooLongFrameException) { + jsonFuture.completeExceptionally(new TooLongFrameException(String.format( + cause.getMessage() + " Try to raise [%s]", + RestOptions.REST_CLIENT_MAX_CONTENT_LENGTH.key()))); + } else { + jsonFuture.completeExceptionally(cause); + } ctx.close(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java index 782cb4e518e1f8..17d4264565b64f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestClientConfiguration.java @@ -29,6 +29,8 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; +import static org.apache.flink.util.Preconditions.checkArgument; + /** * A configuration object for {@link RestClient}s. */ @@ -45,6 +47,7 @@ private RestClientConfiguration( @Nullable final SSLEngine sslEngine, final long connectionTimeout, final int maxContentLength) { + checkArgument(maxContentLength > 0, "maxContentLength must be positive, was: %d", maxContentLength); this.sslEngine = sslEngine; this.connectionTimeout = connectionTimeout; this.maxContentLength = maxContentLength; @@ -104,10 +107,7 @@ public static RestClientConfiguration fromConfiguration(Configuration config) th final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT); - int maxContentLength = config.getInteger(RestOptions.REST_CLIENT_CONTENT_MAX_MB) * 1024 * 1024; - if (maxContentLength <= 0) { - throw new ConfigurationException("Max content length for client must be a positive integer: " + maxContentLength); - } + int maxContentLength = config.getInteger(RestOptions.REST_CLIENT_MAX_CONTENT_LENGTH); return new RestClientConfiguration(sslEngine, connectionTimeout, maxContentLength); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java index 8b392508aeaced..a3d48431f81e43 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java @@ -37,7 +37,6 @@ import org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoopGroup; import org.apache.flink.shaded.netty4.io.netty.channel.socket.SocketChannel; import org.apache.flink.shaded.netty4.io.netty.channel.socket.nio.NioServerSocketChannel; -import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpObjectAggregator; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpServerCodec; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.router.Handler; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.router.Router; @@ -59,6 +58,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.TimeUnit; @@ -78,6 +78,7 @@ public abstract class RestServerEndpoint { private final SSLEngine sslEngine; private final Path uploadDir; private final int maxContentLength; + protected final Map responseHeaders; private final CompletableFuture terminationFuture; @@ -97,6 +98,7 @@ public RestServerEndpoint(RestServerEndpointConfiguration configuration) throws createUploadDir(uploadDir, log); this.maxContentLength = configuration.getMaxContentLength(); + this.responseHeaders = configuration.getResponseHeaders(); terminationFuture = new CompletableFuture<>(); @@ -148,7 +150,7 @@ public final void start() throws Exception { @Override protected void initChannel(SocketChannel ch) { - Handler handler = new RouterHandler(router); + Handler handler = new RouterHandler(router, responseHeaders); // SSL should be the first handler in the pipeline if (sslEngine != null) { @@ -158,9 +160,9 @@ protected void initChannel(SocketChannel ch) { ch.pipeline() .addLast(new HttpServerCodec()) .addLast(new FileUploadHandler(uploadDir)) - .addLast(new HttpObjectAggregator(maxContentLength)) + .addLast(new FlinkHttpObjectAggregator(maxContentLength, responseHeaders)) .addLast(handler.name(), handler) - .addLast(new PipelineErrorHandler(log)); + .addLast(new PipelineErrorHandler(log, responseHeaders)); } }; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java index 3685e2d656e50e..35bd6ea15d77b4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java @@ -26,12 +26,16 @@ import org.apache.flink.util.ConfigurationException; import org.apache.flink.util.Preconditions; +import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaders; + import javax.annotation.Nullable; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collections; +import java.util.Map; import java.util.UUID; import static java.util.Objects.requireNonNull; @@ -53,20 +57,24 @@ public final class RestServerEndpointConfiguration { private final int maxContentLength; + private final Map responseHeaders; + private RestServerEndpointConfiguration( @Nullable String restBindAddress, int restBindPort, @Nullable SSLEngine sslEngine, final Path uploadDir, - final int maxContentLength) { + final int maxContentLength, final Map responseHeaders) { Preconditions.checkArgument(0 <= restBindPort && restBindPort < 65536, "The bing rest port " + restBindPort + " is out of range (0, 65536["); + Preconditions.checkArgument(maxContentLength > 0, "maxContentLength must be positive, was: %d", maxContentLength); this.restBindAddress = restBindAddress; this.restBindPort = restBindPort; this.sslEngine = sslEngine; this.uploadDir = requireNonNull(uploadDir); this.maxContentLength = maxContentLength; + this.responseHeaders = requireNonNull(Collections.unmodifiableMap(responseHeaders)); } /** @@ -112,6 +120,13 @@ public int getMaxContentLength() { return maxContentLength; } + /** + * Response headers that should be added to every HTTP response. + */ + public Map getResponseHeaders() { + return responseHeaders; + } + /** * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}. * @@ -144,11 +159,18 @@ public static RestServerEndpointConfiguration fromConfiguration(Configuration co config.getString(WebOptions.UPLOAD_DIR, config.getString(WebOptions.TMP_DIR)), "flink-web-upload-" + UUID.randomUUID()); - int maxContentLength = config.getInteger(RestOptions.REST_SERVER_CONTENT_MAX_MB) * 1024 * 1024; - if (maxContentLength <= 0) { - throw new ConfigurationException("Max content length for server must be a positive integer: " + maxContentLength); - } + int maxContentLength = config.getInteger(RestOptions.REST_SERVER_MAX_CONTENT_LENGTH); + + final Map responseHeaders = Collections.singletonMap( + HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN, + config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN)); - return new RestServerEndpointConfiguration(address, port, sslEngine, uploadDir, maxContentLength); + return new RestServerEndpointConfiguration( + address, + port, + sslEngine, + uploadDir, + maxContentLength, + responseHeaders); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/PipelineErrorHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/PipelineErrorHandler.java index 046118a3401633..a16b01fcbb736d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/PipelineErrorHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/PipelineErrorHandler.java @@ -30,6 +30,9 @@ import org.slf4j.Logger; import java.util.Collections; +import java.util.Map; + +import static java.util.Objects.requireNonNull; /** * This is the last handler in the pipeline. It logs all error messages. @@ -40,8 +43,11 @@ public class PipelineErrorHandler extends SimpleChannelInboundHandler responseHeaders; + + public PipelineErrorHandler(Logger logger, final Map responseHeaders) { + this.logger = requireNonNull(logger); + this.responseHeaders = requireNonNull(responseHeaders); } @Override @@ -59,5 +65,11 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest message) { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { logger.warn("Unhandled exception", cause); + HandlerUtils.sendErrorResponse( + ctx, + false, + new ErrorResponseBody("Internal server error: " + cause.getMessage()), + HttpResponseStatus.INTERNAL_SERVER_ERROR, + responseHeaders); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java index acdd63c92f76f7..f92946bd0f5f4d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java @@ -23,11 +23,7 @@ import org.apache.flink.configuration.WebOptions; import org.apache.flink.util.Preconditions; -import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpHeaders; - import java.io.File; -import java.util.Collections; -import java.util.Map; import java.util.UUID; /** @@ -43,14 +39,11 @@ public class RestHandlerConfiguration { private final File tmpDir; - private final Map responseHeaders; - public RestHandlerConfiguration( long refreshInterval, int maxCheckpointStatisticCacheEntries, Time timeout, - File tmpDir, - Map responseHeaders) { + File tmpDir) { Preconditions.checkArgument(refreshInterval > 0L, "The refresh interval (ms) should be larger than 0."); this.refreshInterval = refreshInterval; @@ -58,8 +51,6 @@ public RestHandlerConfiguration( this.timeout = Preconditions.checkNotNull(timeout); this.tmpDir = Preconditions.checkNotNull(tmpDir); - - this.responseHeaders = Preconditions.checkNotNull(responseHeaders); } public long getRefreshInterval() { @@ -78,10 +69,6 @@ public File getTmpDir() { return tmpDir; } - public Map getResponseHeaders() { - return Collections.unmodifiableMap(responseHeaders); - } - public static RestHandlerConfiguration fromConfiguration(Configuration configuration) { final long refreshInterval = configuration.getLong(WebOptions.REFRESH_INTERVAL); @@ -92,15 +79,10 @@ public static RestHandlerConfiguration fromConfiguration(Configuration configura final String rootDir = "flink-web-" + UUID.randomUUID(); final File tmpDir = new File(configuration.getString(WebOptions.TMP_DIR), rootDir); - final Map responseHeaders = Collections.singletonMap( - HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN, - configuration.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN)); - return new RestHandlerConfiguration( refreshInterval, maxCheckpointStatisticCacheEntries, timeout, - tmpDir, - responseHeaders); + tmpDir); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RouterHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RouterHandler.java index d1d08373ee6b4b..fc02250ff805d5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RouterHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RouterHandler.java @@ -27,20 +27,21 @@ import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.router.Handler; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.router.Router; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.Map; -import java.util.Collections; +import static java.util.Objects.requireNonNull; /** * This class is an extension of {@link Handler} that replaces the standard error response to be identical with those * sent by the {@link AbstractRestHandler}. */ public class RouterHandler extends Handler { - private static final Logger LOG = LoggerFactory.getLogger(RouterHandler.class); - public RouterHandler(Router router) { + private final Map responseHeaders; + + public RouterHandler(Router router, final Map responseHeaders) { super(router); + this.responseHeaders = requireNonNull(responseHeaders); } @Override @@ -50,6 +51,6 @@ protected void respondNotFound(ChannelHandlerContext ctx, HttpRequest request) { request, new ErrorResponseBody("Not found."), HttpResponseStatus.NOT_FOUND, - Collections.emptyMap()); + responseHeaders); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java index a69f4aaf4576bf..b407ada46e6420 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/util/HandlerUtils.java @@ -112,6 +112,30 @@ public static void sendErrorResponse( HttpResponseStatus statusCode, Map headers) { + sendErrorResponse( + channelHandlerContext, + HttpHeaders.isKeepAlive(httpRequest), + errorMessage, + statusCode, + headers); + } + + /** + * Sends the given error response and status code to the given channel. + * + * @param channelHandlerContext identifying the open channel + * @param keepAlive If the connection should be kept alive. + * @param errorMessage which should be sent + * @param statusCode of the message to send + * @param headers additional header values + */ + public static void sendErrorResponse( + ChannelHandlerContext channelHandlerContext, + boolean keepAlive, + ErrorResponseBody errorMessage, + HttpResponseStatus statusCode, + Map headers) { + StringWriter sw = new StringWriter(); try { mapper.writeValue(sw, errorMessage); @@ -120,14 +144,14 @@ public static void sendErrorResponse( LOG.error("Internal server error. Could not map error response to JSON.", e); sendResponse( channelHandlerContext, - httpRequest, + keepAlive, "Internal server error. Could not map error response to JSON.", HttpResponseStatus.INTERNAL_SERVER_ERROR, headers); } sendResponse( channelHandlerContext, - httpRequest, + keepAlive, sw.toString(), statusCode, headers); @@ -148,6 +172,30 @@ public static void sendResponse( @Nonnull String message, @Nonnull HttpResponseStatus statusCode, @Nonnull Map headers) { + + sendResponse( + channelHandlerContext, + HttpHeaders.isKeepAlive(httpRequest), + message, + statusCode, + headers); + } + + /** + * Sends the given response and status code to the given channel. + * + * @param channelHandlerContext identifying the open channel + * @param keepAlive If the connection should be kept alive. + * @param message which should be sent + * @param statusCode of the message to send + * @param headers additional header values + */ + public static void sendResponse( + @Nonnull ChannelHandlerContext channelHandlerContext, + boolean keepAlive, + @Nonnull String message, + @Nonnull HttpResponseStatus statusCode, + @Nonnull Map headers) { HttpResponse response = new DefaultHttpResponse(HTTP_1_1, statusCode); response.headers().set(CONTENT_TYPE, RestConstants.REST_CONTENT_TYPE); @@ -156,7 +204,7 @@ public static void sendResponse( response.headers().set(headerEntry.getKey(), headerEntry.getValue()); } - if (HttpHeaders.isKeepAlive(httpRequest)) { + if (keepAlive) { response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE); } @@ -172,7 +220,7 @@ public static void sendResponse( ChannelFuture lastContentFuture = channelHandlerContext.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); // close the connection, if no keep-alive is needed - if (!HttpHeaders.isKeepAlive(httpRequest)) { + if (!keepAlive) { lastContentFuture.addListener(ChannelFutureListener.CLOSE); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java index 10a3650ec2323c..dfb2fc8591d97d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java @@ -129,7 +129,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -199,7 +198,6 @@ protected List> initiali ArrayList> handlers = new ArrayList<>(30); final Time timeout = restConfiguration.getTimeout(); - final Map responseHeaders = restConfiguration.getResponseHeaders(); ClusterOverviewHandler clusterOverviewHandler = new ClusterOverviewHandler( restAddressFuture, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java index c9817ff19e437c..32f3ec89cadd61 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java @@ -51,10 +51,10 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler; +import org.apache.flink.shaded.netty4.io.netty.handler.codec.TooLongFrameException; import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -81,7 +81,12 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -96,6 +101,7 @@ public class RestServerEndpointITCase extends TestLogger { private static final JobID QUERY_JOB_ID = new JobID(); private static final String JOB_ID_KEY = "jobid"; private static final Time timeout = Time.seconds(10L); + private static final int TEST_REST_MAX_CONTENT_LENGTH = 4096; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -103,12 +109,15 @@ public class RestServerEndpointITCase extends TestLogger { private RestServerEndpoint serverEndpoint; private RestClient restClient; private TestUploadHandler testUploadHandler; + private InetSocketAddress serverAddress; @Before public void setup() throws Exception { Configuration config = new Configuration(); config.setInteger(RestOptions.REST_PORT, 0); config.setString(WebOptions.UPLOAD_DIR, temporaryFolder.newFolder().getCanonicalPath()); + config.setInteger(RestOptions.REST_SERVER_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH); + config.setInteger(RestOptions.REST_CLIENT_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH); RestServerEndpointConfiguration serverConfig = RestServerEndpointConfiguration.fromConfiguration(config); RestClientConfiguration clientConfig = RestClientConfiguration.fromConfiguration(config); @@ -133,6 +142,7 @@ public void setup() throws Exception { restClient = new TestRestClient(clientConfig); serverEndpoint.start(); + serverAddress = serverEndpoint.getServerAddress(); } @After @@ -161,7 +171,6 @@ public void testRequestInterleaving() throws Exception { // send first request and wait until the handler blocks CompletableFuture response1; - final InetSocketAddress serverAddress = serverEndpoint.getServerAddress(); synchronized (TestHandler.LOCK) { response1 = restClient.sendRequest( @@ -198,8 +207,6 @@ public void testRequestInterleaving() throws Exception { */ @Test public void testBadHandlerRequest() throws Exception { - final InetSocketAddress serverAddress = serverEndpoint.getServerAddress(); - final FaultyTestParameters parameters = new FaultyTestParameters(); parameters.faultyJobIDPathParameter.resolve(PATH_JOB_ID); @@ -215,11 +222,11 @@ public void testBadHandlerRequest() throws Exception { try { response.get(); - Assert.fail("The request should fail with a bad request return code."); + fail("The request should fail with a bad request return code."); } catch (ExecutionException ee) { Throwable t = ExceptionUtils.stripExecutionException(ee); - Assert.assertTrue(t instanceof RestClientException); + assertTrue(t instanceof RestClientException); RestClientException rce = (RestClientException) t; @@ -227,6 +234,50 @@ public void testBadHandlerRequest() throws Exception { } } + /** + * Tests that requests and responses larger than {@link #TEST_REST_MAX_CONTENT_LENGTH} + * are rejected by the server and client, respectively. + */ + @Test + public void testMaxContentLengthLimit() throws Exception { + final TestParameters parameters = new TestParameters(); + parameters.jobIDPathParameter.resolve(PATH_JOB_ID); + parameters.jobIDQueryParameter.resolve(Collections.singletonList(QUERY_JOB_ID)); + + CompletableFuture response; + response = restClient.sendRequest( + serverAddress.getHostName(), + serverAddress.getPort(), + new TestHeaders(), + parameters, + new TestRequest(2, createStringOfSize(TEST_REST_MAX_CONTENT_LENGTH))); + + try { + response.get(); + fail("Expected exception not thrown"); + } catch (final ExecutionException e) { + final Throwable throwable = ExceptionUtils.stripExecutionException(e); + assertThat(throwable, instanceOf(RestClientException.class)); + assertThat(throwable.getMessage(), containsString("Try to raise")); + } + + response = restClient.sendRequest( + serverAddress.getHostName(), + serverAddress.getPort(), + new TestHeaders(), + parameters, + new TestRequest(TestHandler.LARGE_RESPONSE_BODY_ID)); + + try { + response.get(); + fail("Expected exception not thrown"); + } catch (final ExecutionException e) { + final Throwable throwable = ExceptionUtils.stripExecutionException(e); + assertThat(throwable, instanceOf(TooLongFrameException.class)); + assertThat(throwable.getMessage(), containsString("Try to raise")); + } + } + /** * Tests that multipart/form-data uploads work correctly. * @@ -294,6 +345,14 @@ private static String generateMultiPartBoundary() { return Long.toHexString(System.currentTimeMillis()); } + private static String createStringOfSize(int size) { + StringBuilder sb = new StringBuilder(size); + for (int i = 0; i < size; i++) { + sb.append('a'); + } + return sb.toString(); + } + private static class TestRestServerEndpoint extends RestServerEndpoint { private final TestHandler testHandler; @@ -323,12 +382,14 @@ protected void startInternal() throws Exception {} private static class TestHandler extends AbstractRestHandler { - public static final Object LOCK = new Object(); + private static final Object LOCK = new Object(); + + private static final int LARGE_RESPONSE_BODY_ID = 3; TestHandler( - CompletableFuture localAddressFuture, - GatewayRetriever leaderRetriever, - Time timeout) { + CompletableFuture localAddressFuture, + GatewayRetriever leaderRetriever, + Time timeout) { super( localAddressFuture, leaderRetriever, @@ -342,7 +403,8 @@ protected CompletableFuture handleRequest(@Nonnull HandlerRequest< assertEquals(request.getPathParameter(JobIDPathParameter.class), PATH_JOB_ID); assertEquals(request.getQueryParameter(JobIDQueryParameter.class).get(0), QUERY_JOB_ID); - if (request.getRequestBody().id == 1) { + final int id = request.getRequestBody().id; + if (id == 1) { synchronized (LOCK) { try { LOCK.notifyAll(); @@ -350,8 +412,12 @@ protected CompletableFuture handleRequest(@Nonnull HandlerRequest< } catch (InterruptedException ignored) { } } + } else if (id == LARGE_RESPONSE_BODY_ID) { + return CompletableFuture.completedFuture(new TestResponse( + id, + createStringOfSize(TEST_REST_MAX_CONTENT_LENGTH))); } - return CompletableFuture.completedFuture(new TestResponse(request.getRequestBody().id)); + return CompletableFuture.completedFuture(new TestResponse(id)); } } @@ -365,18 +431,37 @@ private static class TestRestClient extends RestClient { private static class TestRequest implements RequestBody { public final int id; + public final String content; + + public TestRequest(int id) { + this(id, null); + } + @JsonCreator - public TestRequest(@JsonProperty("id") int id) { + public TestRequest( + @JsonProperty("id") int id, + @JsonProperty("content") final String content) { this.id = id; + this.content = content; } } private static class TestResponse implements ResponseBody { + public final int id; + public final String content; + + public TestResponse(int id) { + this(id, null); + } + @JsonCreator - public TestResponse(@JsonProperty("id") int id) { + public TestResponse( + @JsonProperty("id") int id, + @JsonProperty("content") String content) { this.id = id; + this.content = content; } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskCurrentAttemptDetailsHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskCurrentAttemptDetailsHandlerTest.java index 997601fe99cf8e..af8b995a3e66ff 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskCurrentAttemptDetailsHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskCurrentAttemptDetailsHandlerTest.java @@ -126,7 +126,7 @@ public void testHandleRequest() throws Exception { CompletableFuture.completedFuture("127.0.0.1:9527"), () -> null, Time.milliseconds(100), - restHandlerConfiguration.getResponseHeaders(), + Collections.emptyMap(), SubtaskCurrentAttemptDetailsHeaders.getInstance(), new ExecutionGraphCache( restHandlerConfiguration.getTimeout(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java index 5f03c5572acd5e..318541d288612f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java @@ -41,6 +41,7 @@ import org.junit.Test; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -62,7 +63,7 @@ public void testHandleRequest() throws Exception { CompletableFuture.completedFuture("127.0.0.1:9527"), () -> null, Time.milliseconds(100L), - restHandlerConfiguration.getResponseHeaders(), + Collections.emptyMap(), SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(), new ExecutionGraphCache( restHandlerConfiguration.getTimeout(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java index d55ab775a14883..8e44c0e9731b0c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptDetailsHandlerTest.java @@ -129,7 +129,7 @@ public void testHandleRequest() throws Exception { CompletableFuture.completedFuture("127.0.0.1:9527"), () -> null, Time.milliseconds(100L), - restHandlerConfiguration.getResponseHeaders(), + Collections.emptyMap(), SubtaskExecutionAttemptDetailsHeaders.getInstance(), new ExecutionGraphCache( restHandlerConfiguration.getTimeout(), From 310f3de62e52f1f977c217d918cc5aac79b87277 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 12 Mar 2018 14:56:32 +0100 Subject: [PATCH 0170/2294] [FLINK-8850] [sql-client] Add support for event-time in SQL Client This closes #5683. --- .../conf/sql-client-defaults.yaml | 32 ++++++++++++------ .../flink/table/client/config/Execution.java | 16 +++++++++ .../table/client/config/PropertyStrings.java | 6 ++++ .../flink/table/client/gateway/Executor.java | 3 +- .../gateway/local/ExecutionContext.java | 1 + .../client/gateway/local/LocalExecutor.java | 5 +-- .../client/gateway/local/ResultStore.java | 2 +- .../client/gateway/local/DependencyTest.java | 1 + .../gateway/local/LocalExecutorITCase.java | 1 + .../gateway/utils/TestTableSourceFactory.java | 33 +++++++++++++++++-- .../resources/test-sql-client-defaults.yaml | 1 + .../resources/test-sql-client-factory.yaml | 7 ++++ .../apache/flink/table/api/TableSchema.scala | 24 ++++++++++++-- 13 files changed, 113 insertions(+), 19 deletions(-) diff --git a/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml b/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml index 76ccd0c1292ef7..35584222e22f91 100644 --- a/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml +++ b/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml @@ -41,12 +41,20 @@ sources: [] # empty list # Execution properties allow for changing the behavior of a table program. execution: - type: streaming # 'batch' or 'streaming' execution - result-mode: changelog # 'changelog' or 'table' presentation of results - parallelism: 1 # parallelism of the program - max-parallelism: 128 # maximum parallelism - min-idle-state-retention: 0 # minimum idle state retention in ms - max-idle-state-retention: 0 # maximum idle state retention in ms + # 'batch' or 'streaming' execution + type: streaming + # allow 'event-time' or only 'processing-time' in sources + time-characteristic: event-time + # 'changelog' or 'table' presentation of results + result-mode: changelog + # parallelism of the program + parallelism: 1 + # maximum parallelism + max-parallelism: 128 + # minimum idle state retention in ms + min-idle-state-retention: 0 + # maximum idle state retention in ms + max-idle-state-retention: 0 #============================================================================== # Deployment properties @@ -56,9 +64,13 @@ execution: # programs are submitted to. deployment: - type: standalone # only the 'standalone' deployment is supported - response-timeout: 5000 # general cluster communication timeout in ms - gateway-address: "" # (optional) address from cluster to gateway - gateway-port: 0 # (optional) port from cluster to gateway + # only the 'standalone' deployment is supported + type: standalone + # general cluster communication timeout in ms + response-timeout: 5000 + # (optional) address from cluster to gateway + gateway-address: "" + # (optional) port from cluster to gateway + gateway-port: 0 diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Execution.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Execution.java index 37d1a34e3d4cd2..d84c35b1d2b33f 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Execution.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Execution.java @@ -18,6 +18,8 @@ package org.apache.flink.table.client.config; +import org.apache.flink.streaming.api.TimeCharacteristic; + import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -54,6 +56,20 @@ public boolean isBatchExecution() { PropertyStrings.EXECUTION_TYPE_VALUE_BATCH); } + public TimeCharacteristic getTimeCharacteristic() { + final String s = properties.getOrDefault( + PropertyStrings.EXECUTION_TIME_CHARACTERISTIC, + PropertyStrings.EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME); + switch (s) { + case PropertyStrings.EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME: + return TimeCharacteristic.EventTime; + case PropertyStrings.EXECUTION_TIME_CHARACTERISTIC_VALUE_PROCESSING_TIME: + return TimeCharacteristic.ProcessingTime; + default: + return TimeCharacteristic.EventTime; + } + } + public long getMinStateRetention() { return Long.parseLong(properties.getOrDefault(PropertyStrings.EXECUTION_MIN_STATE_RETENTION, Long.toString(Long.MIN_VALUE))); } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/PropertyStrings.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/PropertyStrings.java index ba0759d3b06309..b7a3101dbcf5b0 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/PropertyStrings.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/PropertyStrings.java @@ -35,6 +35,12 @@ private PropertyStrings() { public static final String EXECUTION_TYPE_VALUE_BATCH = "batch"; + public static final String EXECUTION_TIME_CHARACTERISTIC = "time-characteristic"; + + public static final String EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME = "event-time"; + + public static final String EXECUTION_TIME_CHARACTERISTIC_VALUE_PROCESSING_TIME = "processing-time"; + public static final String EXECUTION_MIN_STATE_RETENTION = "min-idle-state-retention"; public static final String EXECUTION_MAX_STATE_RETENTION = "max-idle-state-retention"; diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java index 4a41222700e26a..74e6a6b2dbb19b 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/Executor.java @@ -46,7 +46,8 @@ public interface Executor { List listTables(SessionContext session) throws SqlExecutionException; /** - * Returns the schema of a table. Throws an exception if the table could not be found. + * Returns the schema of a table. Throws an exception if the table could not be found. The + * schema might contain time attribute types for helping the user during debugging a query. */ TableSchema getTableSchema(SessionContext session, String name) throws SqlExecutionException; diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java index 15a3c129bb8018..a013afcc0a424e 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java @@ -149,6 +149,7 @@ private StreamExecutionEnvironment createStreamExecutionEnvironment() { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(mergedEnv.getExecution().getParallelism()); env.setMaxParallelism(mergedEnv.getExecution().getMaxParallelism()); + env.setStreamTimeCharacteristic(mergedEnv.getExecution().getTimeCharacteristic()); return env; } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java index 35d7da9bbfd285..fa6c9d2fd26cd7 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java @@ -228,6 +228,7 @@ public ResultDescriptor executeQuery(SessionContext session, String query) throw // create table here to fail quickly for wrong queries final Table table = createTable(context, query); + final TableSchema resultSchema = table.getSchema().withoutTimeAttributes(); // deployment final ClusterClient clusterClient = createDeployment(mergedEnv.getDeployment()); @@ -235,7 +236,7 @@ public ResultDescriptor executeQuery(SessionContext session, String query) throw // initialize result final DynamicResult result = resultStore.createResult( mergedEnv, - table.getSchema(), + resultSchema, context.getExecutionConfig()); // create job graph with jars @@ -275,7 +276,7 @@ public ResultDescriptor executeQuery(SessionContext session, String query) throw // start result retrieval result.startRetrieval(program); - return new ResultDescriptor(resultId, table.getSchema(), result.isMaterialized()); + return new ResultDescriptor(resultId, resultSchema, result.isMaterialized()); } @Override diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java index 1f3dc8484f0d99..19a440e22246b4 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java @@ -55,7 +55,7 @@ public ResultStore(Configuration flinkConfig) { } /** - * Creates a result. Might start thread or opens sockets so every creates result must be closed. + * Creates a result. Might start threads or opens sockets so every created result must be closed. */ public DynamicResult createResult(Environment env, TableSchema schema, ExecutionConfig config) { if (!env.getExecution().isStreamingExecution()) { diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java index 715d2db5c39c2a..40a1c2cf137c2a 100644 --- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java @@ -65,6 +65,7 @@ public void testTableSourceFactoryDiscovery() throws Exception { final TableSchema expected = TableSchema.builder() .field("IntegerField1", Types.INT()) .field("StringField1", Types.STRING()) + .field("rowtimeField", Types.SQL_TIMESTAMP()) .build(); assertEquals(expected, result); diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java index a2ae28108bfb96..45369784f563ca 100644 --- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java @@ -101,6 +101,7 @@ public void testGetSessionProperties() throws Exception { final Map expectedProperties = new HashMap<>(); expectedProperties.put("execution.type", "streaming"); + expectedProperties.put("execution.time-characteristic", "event-time"); expectedProperties.put("execution.parallelism", "1"); expectedProperties.put("execution.max-parallelism", "16"); expectedProperties.put("execution.max-idle-state-retention", "0"); diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java index 40a7e7bac67859..1b0a30e35615e7 100644 --- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactory.java @@ -25,6 +25,10 @@ import org.apache.flink.table.api.Types; import org.apache.flink.table.client.gateway.local.DependencyTest; import org.apache.flink.table.descriptors.DescriptorProperties; +import org.apache.flink.table.descriptors.SchemaValidator; +import org.apache.flink.table.sources.DefinedProctimeAttribute; +import org.apache.flink.table.sources.DefinedRowtimeAttributes; +import org.apache.flink.table.sources.RowtimeAttributeDescriptor; import org.apache.flink.table.sources.StreamTableSource; import org.apache.flink.table.sources.TableSource; import org.apache.flink.table.sources.TableSourceFactory; @@ -34,8 +38,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_TYPE; +import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_TYPE; import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA; import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_NAME; import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_TYPE; @@ -58,6 +65,8 @@ public List supportedProperties() { properties.add("connector.test-property"); properties.add(SCHEMA() + ".#." + SCHEMA_TYPE()); properties.add(SCHEMA() + ".#." + SCHEMA_NAME()); + properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_TYPE()); + properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_TYPE()); return properties; } @@ -65,9 +74,13 @@ public List supportedProperties() { public TableSource create(Map properties) { final DescriptorProperties params = new DescriptorProperties(true); params.putProperties(properties); + final Optional proctime = SchemaValidator.deriveProctimeAttribute(params); + final List rowtime = SchemaValidator.deriveRowtimeAttributes(params); return new TestTableSource( params.getTableSchema(SCHEMA()), - properties.get("connector.test-property")); + properties.get("connector.test-property"), + proctime.orElse(null), + rowtime); } // -------------------------------------------------------------------------------------------- @@ -75,14 +88,18 @@ public TableSource create(Map properties) { /** * Test table source. */ - public static class TestTableSource implements StreamTableSource { + public static class TestTableSource implements StreamTableSource, DefinedRowtimeAttributes, DefinedProctimeAttribute { private final TableSchema schema; private final String property; + private final String proctime; + private final List rowtime; - public TestTableSource(TableSchema schema, String property) { + public TestTableSource(TableSchema schema, String property, String proctime, List rowtime) { this.schema = schema; this.property = property; + this.proctime = proctime; + this.rowtime = rowtime; } public String getProperty() { @@ -108,5 +125,15 @@ public TableSchema getTableSchema() { public String explainSource() { return "TestTableSource"; } + + @Override + public List getRowtimeAttributeDescriptors() { + return rowtime; + } + + @Override + public String getProctimeAttribute() { + return proctime; + } } } diff --git a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml index 9cbecb07903ab7..5a598f15ab42bf 100644 --- a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml +++ b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml @@ -63,6 +63,7 @@ sources: execution: type: streaming + time-characteristic: event-time parallelism: 1 max-parallelism: 16 min-idle-state-retention: 0 diff --git a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml index 1bb69e537f40a8..daa1fd167b5f22 100644 --- a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml +++ b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml @@ -30,6 +30,13 @@ sources: type: INT - name: StringField1 type: VARCHAR + - name: rowtimeField + type: TIMESTAMP + rowtime: + timestamps: + type: from-source + watermarks: + type: from-source connector: type: "$VAR_0" $VAR_1: "$VAR_2" diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala index 6958b3d15da8af..6389b55b125091 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala @@ -23,6 +23,8 @@ import org.apache.flink.api.common.typeutils.CompositeType import _root_.scala.collection.mutable.ArrayBuffer import _root_.java.util.Objects +import org.apache.flink.table.calcite.FlinkTypeFactory + /** * A TableSchema represents a Table's structure. */ @@ -30,6 +32,8 @@ class TableSchema( private val columnNames: Array[String], private val columnTypes: Array[TypeInformation[_]]) { + private val columnNameToIndex: Map[String, Int] = columnNames.zipWithIndex.toMap + if (columnNames.length != columnTypes.length) { throw new TableException( s"Number of field names and field types must be equal.\n" + @@ -52,8 +56,6 @@ class TableSchema( s"List of all fields: ${columnNames.mkString("[", ", ", "]")}.") } - val columnNameToIndex: Map[String, Int] = columnNames.zipWithIndex.toMap - /** * Returns a deep copy of the TableSchema. */ @@ -115,6 +117,24 @@ class TableSchema( } } + /** + * Converts a table schema into a schema that represents the result that would be written + * into a table sink or operator outside of the Table & SQL API. Time attributes are replaced + * by proper TIMESTAMP data types. + * + * @return a table schema with no time attributes + */ + def withoutTimeAttributes: TableSchema = { + val converted = columnTypes.map { t => + if (FlinkTypeFactory.isTimeIndicatorType(t)) { + Types.SQL_TIMESTAMP + } else { + t + } + } + new TableSchema(columnNames, converted) + } + override def toString: String = { val builder = new StringBuilder builder.append("root\n") From a3478fdfa0f792104123fefbd9bdf01f5029de51 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Fri, 2 Mar 2018 14:48:20 +0100 Subject: [PATCH 0171/2294] [FLINK-8832] [sql-client] Create a SQL Client Kafka 0.11 fat-jar This closes #5673. --- .../flink-connector-kafka-0.11/pom.xml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/flink-connectors/flink-connector-kafka-0.11/pom.xml b/flink-connectors/flink-connector-kafka-0.11/pom.xml index befa33686c18fb..8b404390235e31 100644 --- a/flink-connectors/flink-connector-kafka-0.11/pom.xml +++ b/flink-connectors/flink-connector-kafka-0.11/pom.xml @@ -211,6 +211,55 @@ under the License. + + + + release + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + sql-jar + + + org.apache.kafka:* + org.apache.flink:flink-connector-kafka-base_${scala.binary.version} + org.apache.flink:flink-connector-kafka-0.9_${scala.binary.version} + org.apache.flink:flink-connector-kafka-0.10_${scala.binary.version} + + + + + *:* + + kafka/kafka-version.properties + + + + + + org.apache.kafka + org.apache.flink.kafka011.shaded.org.apache.kafka + + + + + + + + + + + From f190f5b0ea8b352f8965b8649901fcdfbcb55066 Mon Sep 17 00:00:00 2001 From: yanghua Date: Mon, 12 Mar 2018 12:12:56 +0800 Subject: [PATCH 0172/2294] [FLINK-8916][REST] Write/read checkpointing mode enum in lower case This closes #5679. --- .../checkpoints/CheckpointConfigInfo.java | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointConfigInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointConfigInfo.java index 7a5d99fae370ed..b0f6abf5215cd7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointConfigInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointConfigInfo.java @@ -24,7 +24,16 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationContext; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.SerializerProvider; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ser.std.StdSerializer; + +import java.io.IOException; import java.util.Objects; /** @@ -145,8 +154,43 @@ public int hashCode() { /** * Processing mode. */ + @JsonSerialize(using = ProcessingModeSerializer.class) + @JsonDeserialize(using = ProcessingModeDeserializer.class) public enum ProcessingMode { AT_LEAST_ONCE, EXACTLY_ONCE } + + /** + * JSON deserializer for {@link ProcessingMode}. + */ + public static class ProcessingModeSerializer extends StdSerializer { + + public ProcessingModeSerializer() { + super(ProcessingMode.class); + } + + @Override + public void serialize(ProcessingMode mode, JsonGenerator generator, SerializerProvider serializerProvider) + throws IOException { + generator.writeString(mode.name().toLowerCase()); + } + } + + /** + * Processing mode deserializer. + */ + public static class ProcessingModeDeserializer extends StdDeserializer { + + public ProcessingModeDeserializer() { + super(ProcessingMode.class); + } + + @Override + public ProcessingMode deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + return ProcessingMode.valueOf(jsonParser.getValueAsString().toUpperCase()); + } + } + } From 4c46af1b4be4c973e383091ae5ba191a0d52a30d Mon Sep 17 00:00:00 2001 From: Florian Schmidt Date: Tue, 13 Mar 2018 16:54:06 +0100 Subject: [PATCH 0173/2294] [hotfix][docs][py] Fix class name in example This closes #5692. --- docs/dev/stream/python.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/stream/python.md b/docs/dev/stream/python.md index a533819f5026d2..887d983c4040e2 100644 --- a/docs/dev/stream/python.md +++ b/docs/dev/stream/python.md @@ -464,7 +464,7 @@ Rich functions (.e.g `RichFilterFunction`) enable to define (override) the optio The user may use these functions for initialization and cleanups. {% highlight python %} -class Tockenizer(RichMapFunction): +class Tokenizer(RichMapFunction): def open(self, config): pass def close(self): @@ -472,7 +472,7 @@ class Tockenizer(RichMapFunction): def map(self, value): pass -data_stream.map(Tockenizer()) +data_stream.map(Tokenizer()) {% endhighlight %} The `open` function is called by the worker before starting the streaming pipeline. From 9898d434a66a0d461d5cebb27877fe16a6acd987 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 9 Mar 2018 10:56:31 +0100 Subject: [PATCH 0174/2294] [hotfix][cli][tests] let CliFrontendRunTest extend from TestLogger --- .../java/org/apache/flink/client/cli/CliFrontendRunTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java index ebb76d886d711f..69724f10083683 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java @@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; +import org.apache.flink.util.TestLogger; import org.junit.BeforeClass; import org.junit.Test; @@ -37,7 +38,7 @@ /** * Tests for the RUN command. */ -public class CliFrontendRunTest { +public class CliFrontendRunTest extends TestLogger { @BeforeClass public static void init() { From 80ee267386a3bc3c27c95a628255031953506128 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 6 Mar 2018 11:43:32 +0100 Subject: [PATCH 0175/2294] [FLINK-8904][cli][tests] Restore previous sysout This closes #5670. --- .../apache/flink/client/cli/CliFrontendCancelTest.java | 6 ++++++ .../org/apache/flink/client/cli/CliFrontendListTest.java | 6 ++++++ .../flink/client/cli/CliFrontendPackageProgramTest.java | 9 +++++++-- .../org/apache/flink/client/cli/CliFrontendRunTest.java | 6 ++++++ .../org/apache/flink/client/cli/CliFrontendStopTest.java | 9 +++++++-- .../apache/flink/client/cli/CliFrontendTestUtils.java | 6 ++++++ 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java index b2fa003bcb9e11..837c56408699c3 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java @@ -24,6 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.util.TestLogger; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -46,6 +47,11 @@ public static void init() { CliFrontendTestUtils.pipeSystemOutToNull(); } + @AfterClass + public static void shutdown() { + CliFrontendTestUtils.restoreSystemOut(); + } + @Test public void testCancel() throws Exception { // test cancel properly diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java index 760b376ce7afb6..42399cb65f09b3 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java @@ -23,6 +23,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.util.TestLogger; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -44,6 +45,11 @@ public static void init() { CliFrontendTestUtils.pipeSystemOutToNull(); } + @AfterClass + public static void shutdown() { + CliFrontendTestUtils.restoreSystemOut(); + } + @Test public void testList() throws Exception { // test list properly diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java index 6873e68d1376a8..48c889120343c9 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendPackageProgramTest.java @@ -27,6 +27,7 @@ import org.apache.flink.optimizer.costs.DefaultCostEstimator; import org.apache.flink.util.TestLogger; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -40,7 +41,6 @@ import static org.apache.flink.client.cli.CliFrontendTestUtils.TEST_JAR_MAIN_CLASS; import static org.apache.flink.client.cli.CliFrontendTestUtils.getNonJarFilePath; import static org.apache.flink.client.cli.CliFrontendTestUtils.getTestJarPath; -import static org.apache.flink.client.cli.CliFrontendTestUtils.pipeSystemOutToNull; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -58,7 +58,12 @@ public class CliFrontendPackageProgramTest extends TestLogger { @BeforeClass public static void init() { - pipeSystemOutToNull(); + CliFrontendTestUtils.pipeSystemOutToNull(); + } + + @AfterClass + public static void shutdown() { + CliFrontendTestUtils.restoreSystemOut(); } @Before diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java index 69724f10083683..c7789a89ac5c2d 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java @@ -25,6 +25,7 @@ import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.util.TestLogger; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -45,6 +46,11 @@ public static void init() { CliFrontendTestUtils.pipeSystemOutToNull(); } + @AfterClass + public static void shutdown() { + CliFrontendTestUtils.restoreSystemOut(); + } + @Test public void testRun() throws Exception { final Configuration configuration = GlobalConfiguration.loadConfiguration(CliFrontendTestUtils.getConfigDir()); diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java index d6049e55d5ce36..ec4ccdca7d1df8 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java @@ -26,6 +26,7 @@ import org.apache.flink.util.FlinkException; import org.apache.flink.util.TestLogger; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -34,7 +35,6 @@ import java.util.Collections; -import static org.apache.flink.client.cli.CliFrontendTestUtils.pipeSystemOutToNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; @@ -49,7 +49,12 @@ public class CliFrontendStopTest extends TestLogger { @BeforeClass public static void setup() { - pipeSystemOutToNull(); + CliFrontendTestUtils.pipeSystemOutToNull(); + } + + @AfterClass + public static void shutdown() { + CliFrontendTestUtils.restoreSystemOut(); } @Test diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestUtils.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestUtils.java index 16737dd125ffd6..b47986ff21f8d6 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestUtils.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestUtils.java @@ -41,6 +41,8 @@ public class CliFrontendTestUtils { public static final int TEST_JOB_MANAGER_PORT = 55443; + private static final PrintStream previousSysout = System.out; + public static String getTestJarPath() throws FileNotFoundException, MalformedURLException { File f = new File("target/maven-test-jar.jar"); if (!f.exists()) { @@ -68,6 +70,10 @@ public static void pipeSystemOutToNull() { System.setOut(new PrintStream(new BlackholeOutputSteam())); } + public static void restoreSystemOut() { + System.setOut(previousSysout); + } + private static final class BlackholeOutputSteam extends java.io.OutputStream { @Override public void write(int b){} From 511f388d9d000a7ac84d45f41bffde514caa21b5 Mon Sep 17 00:00:00 2001 From: Bowen Li Date: Fri, 9 Mar 2018 23:35:15 -0800 Subject: [PATCH 0176/2294] [hotfix][javadocs] Update javadoc of InternalTimerService.registerEventTimeTimer() This closes #5677. --- .../flink/streaming/api/operators/InternalTimerService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerService.java index f55cb0388b6068..cb171fb752ce20 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerService.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerService.java @@ -49,7 +49,7 @@ public interface InternalTimerService { void deleteProcessingTimeTimer(N namespace, long time); /** - * Registers a timer to be fired when processing time passes the given time. The namespace + * Registers a timer to be fired when event time watermark passes the given time. The namespace * you pass here will be provided when the timer fires. */ void registerEventTimeTimer(N namespace, long time); From aa86a86252881d5320e658b6f1315de7a62fac73 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 27 Feb 2018 11:11:59 +0100 Subject: [PATCH 0177/2294] [FLINK-8703][tests] Port KafkaShortRetentionTestBase to MiniClusterResource This closes #5666. --- .../kafka/KafkaShortRetentionTestBase.java | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java index de72985f6b9861..15d972f5570f2f 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java @@ -24,15 +24,14 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.DiscardingSink; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; -import org.apache.flink.streaming.util.TestStreamEnvironment; import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema; import org.apache.flink.streaming.util.serialization.KeyedSerializationSchemaWrapper; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.util.InstantiationUtil; import org.junit.AfterClass; @@ -68,21 +67,32 @@ public class KafkaShortRetentionTestBase implements Serializable { private static KafkaTestEnvironment kafkaServer; private static Properties standardProps; - private static LocalFlinkMiniCluster flink; + + @ClassRule + public static MiniClusterResource flink = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + NUM_TMS, + TM_SLOTS)); @ClassRule public static TemporaryFolder tempFolder = new TemporaryFolder(); protected static Properties secureProps = new Properties(); + private static Configuration getConfiguration() { + Configuration flinkConfig = new Configuration(); + flinkConfig.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L); + flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s"); + return flinkConfig; + } + @BeforeClass - public static void prepare() throws IOException, ClassNotFoundException { + public static void prepare() throws ClassNotFoundException { LOG.info("-------------------------------------------------------------------------"); LOG.info(" Starting KafkaShortRetentionTestBase "); LOG.info("-------------------------------------------------------------------------"); - Configuration flinkConfig = new Configuration(); - // dynamically load the implementation for the test Class clazz = Class.forName("org.apache.flink.streaming.connectors.kafka.KafkaTestEnvironmentImpl"); kafkaServer = (KafkaTestEnvironment) InstantiationUtil.instantiate(clazz); @@ -101,26 +111,10 @@ public static void prepare() throws IOException, ClassNotFoundException { kafkaServer.prepare(kafkaServer.createConfig().setKafkaServerProperties(specificProperties)); standardProps = kafkaServer.getStandardProperties(); - - // start also a re-usable Flink mini cluster - flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); - flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, TM_SLOTS); - flinkConfig.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L); - flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s"); - - flink = new LocalFlinkMiniCluster(flinkConfig, false); - flink.start(); - - TestStreamEnvironment.setAsContext(flink, PARALLELISM); } @AfterClass public static void shutDownServices() throws Exception { - TestStreamEnvironment.unsetAsContext(); - - if (flink != null) { - flink.stop(); - } kafkaServer.shutdown(); secureProps.clear(); @@ -238,8 +232,7 @@ public void runFailOnAutoOffsetResetNone() throws Exception { kafkaServer.createTestTopic(topic, parallelism, 1); - final StreamExecutionEnvironment env = - StreamExecutionEnvironment.createRemoteEnvironment("localhost", flink.getLeaderRPCPort()); + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); env.setRestartStrategy(RestartStrategies.noRestart()); // fail immediately env.getConfig().disableSysoutLogging(); From 12cb09bd4d46da1979e3584622db584b0a316596 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 27 Feb 2018 15:19:50 +0100 Subject: [PATCH 0178/2294] [FLINK-8703][tests] Port NotSoMiniClusterIterations to MiniClusterResource This closes #5667. --- .../manual/NotSoMiniClusterIterations.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/manual/NotSoMiniClusterIterations.java b/flink-tests/src/test/java/org/apache/flink/test/manual/NotSoMiniClusterIterations.java index 9f6bcbbf67c51f..abb8673db5684f 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/manual/NotSoMiniClusterIterations.java +++ b/flink-tests/src/test/java/org/apache/flink/test/manual/NotSoMiniClusterIterations.java @@ -24,12 +24,11 @@ import org.apache.flink.api.java.io.DiscardingOutputFormat; import org.apache.flink.api.java.operators.DeltaIteration; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.examples.java.graph.ConnectedComponents; import org.apache.flink.examples.java.graph.util.ConnectedComponentsData; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; +import org.apache.flink.test.util.MiniClusterResource; import static org.junit.Assert.fail; @@ -46,23 +45,25 @@ public static void main(String[] args) { throw new RuntimeException("This test program needs to run with at least 5GB of heap space."); } - LocalFlinkMiniCluster cluster = null; + MiniClusterResource cluster = null; try { Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, PARALLELISM); config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 8L); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1000); config.setInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE, 8 * 1024); config.setInteger("taskmanager.net.server.numThreads", 1); config.setInteger("taskmanager.net.client.numThreads", 1); - cluster = new LocalFlinkMiniCluster(config, false); - cluster.start(); + cluster = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + config, + PARALLELISM, + 1)); + cluster.before(); - runConnectedComponents(cluster.getLeaderRPCPort()); + runConnectedComponents(); } catch (Exception e) { e.printStackTrace(); @@ -70,14 +71,14 @@ public static void main(String[] args) { } finally { if (cluster != null) { - cluster.stop(); + cluster.after(); } } } - private static void runConnectedComponents(int jmPort) throws Exception { + private static void runConnectedComponents() throws Exception { - ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", jmPort); + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(PARALLELISM); env.getConfig().disableSysoutLogging(); From 1e51e4369b8807489c415a7d8aae5e5cf4f66dfb Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 27 Feb 2018 15:21:50 +0100 Subject: [PATCH 0179/2294] [FLINK-8703][tests] Port StreamingScalabilityAndLatency to MiniClusterResource This closes #5668. --- .../StreamingScalabilityAndLatency.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java b/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java index efcefebe34a01d..a5b01bcd4f4445 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java +++ b/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java @@ -20,14 +20,13 @@ import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction; +import org.apache.flink.test.util.MiniClusterResource; import static org.junit.Assert.fail; @@ -45,22 +44,24 @@ public static void main(String[] args) throws Exception { final int slotsPerTaskManager = 80; final int parallelism = taskManagers * slotsPerTaskManager; - LocalFlinkMiniCluster cluster = null; + MiniClusterResource cluster = null; try { Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, taskManagers); config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 80L); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, slotsPerTaskManager); config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 20000); config.setInteger("taskmanager.net.server.numThreads", 1); config.setInteger("taskmanager.net.client.numThreads", 1); - cluster = new LocalFlinkMiniCluster(config, false); - cluster.start(); + cluster = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + config, + taskManagers, + slotsPerTaskManager)); + cluster.before(); - runPartitioningProgram(cluster.getLeaderRPCPort(), parallelism); + runPartitioningProgram(parallelism); } catch (Exception e) { e.printStackTrace(); @@ -68,13 +69,13 @@ public static void main(String[] args) throws Exception { } finally { if (cluster != null) { - cluster.stop(); + cluster.after(); } } } - private static void runPartitioningProgram(int jobManagerPort, int parallelism) throws Exception { - StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", jobManagerPort); + private static void runPartitioningProgram(int parallelism) throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); env.getConfig().enableObjectReuse(); From a04c8a08383e329213bd74c726f9834b7d19762d Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 13 Mar 2018 13:00:47 +0100 Subject: [PATCH 0180/2294] [FLINK-4569][tests] Respect exceptions thrown in thread in JobRetrievalITCase This closes #5689. --- .../test/example/client/JobRetrievalITCase.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java b/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java index d34b6c337a06ee..57198c054b1cc5 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/example/client/JobRetrievalITCase.java @@ -42,6 +42,7 @@ import org.junit.Test; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicReference; import scala.collection.Seq; @@ -85,6 +86,7 @@ public void testJobRetrieval() throws Exception { // has been attached in resumingThread lock.acquire(); client.runDetached(jobGraph, JobRetrievalITCase.class.getClassLoader()); + final AtomicReference error = new AtomicReference<>(); final Thread resumingThread = new Thread(new Runnable() { @Override @@ -92,10 +94,10 @@ public void run() { try { assertNotNull(client.retrieveJob(jobID)); } catch (Throwable e) { - fail(e.getMessage()); + error.set(e); } } - }); + }, "Flink-Job-Retriever"); final Seq actorSystemSeq = cluster.jobManagerActorSystems().get(); final ActorSystem actorSystem = actorSystemSeq.last(); @@ -119,6 +121,11 @@ public void run() { lock.release(); resumingThread.join(); + + Throwable exception = error.get(); + if (exception != null) { + throw new AssertionError(exception); + } } @Test @@ -148,6 +155,7 @@ public SemaphoreInvokable(Environment environment) { @Override public void invoke() throws Exception { lock.acquire(); + lock.release(); } } From c71f7e1badebd36e4bd55c90a011c04b51c2e9f1 Mon Sep 17 00:00:00 2001 From: Kailash HD Date: Thu, 8 Mar 2018 10:32:23 -0800 Subject: [PATCH 0181/2294] [FLINK-8888] [Kinesis Connectors] Update the AWS SDK for flink kinesis connector This closes #5663 --- flink-connectors/flink-connector-kinesis/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-connectors/flink-connector-kinesis/pom.xml b/flink-connectors/flink-connector-kinesis/pom.xml index 43046cc4021d03..72f9b3c8d9a65d 100644 --- a/flink-connectors/flink-connector-kinesis/pom.xml +++ b/flink-connectors/flink-connector-kinesis/pom.xml @@ -33,7 +33,7 @@ under the License. flink-connector-kinesis_${scala.binary.version} flink-connector-kinesis - 1.11.171 + 1.11.272 1.8.1 0.12.6 From cb60fd29e7e46f8015587e9be7ad88f2368fe85a Mon Sep 17 00:00:00 2001 From: Kailash HD Date: Wed, 14 Mar 2018 09:20:12 -0700 Subject: [PATCH 0182/2294] [FLINK-8945] [kinesis] Allow customization of KinesisProxy This closes #5698 --- .../flink/streaming/connectors/kinesis/proxy/KinesisProxy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/proxy/KinesisProxy.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/proxy/KinesisProxy.java index da81a6575aa447..057e18df0cf942 100644 --- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/proxy/KinesisProxy.java +++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/proxy/KinesisProxy.java @@ -122,7 +122,7 @@ public class KinesisProxy implements KinesisProxyInterface { * * @param configProps configuration properties containing AWS credential and AWS region info */ - private KinesisProxy(Properties configProps) { + protected KinesisProxy(Properties configProps) { checkNotNull(configProps); this.kinesisClient = AWSUtil.createKinesisClient(configProps); From 91d346e9e7611be530509154cc7034cbde22653d Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 16 Mar 2018 11:04:44 +0100 Subject: [PATCH 0183/2294] Revert "[FLINK-7851] [scheduling] Improve scheduling balance by round robin distribution" This reverts commit d9c669d4781f095806013651c1a579eae0ca2650. --- .../instance/SlotSharingGroupAssignment.java | 46 +++++------ .../SlotSharingGroupAssignmentTest.java | 79 ------------------- 2 files changed, 20 insertions(+), 105 deletions(-) delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignmentTest.java diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignment.java index 289762c82e1980..e61ba587e5f7da 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignment.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignment.java @@ -97,7 +97,7 @@ public class SlotSharingGroupAssignment { private final Set allSlots = new LinkedHashSet(); /** The slots available per vertex type (JobVertexId), keyed by TaskManager, to make them locatable */ - private final Map>> availableSlotsPerJid = new LinkedHashMap<>(); + private final Map>> availableSlotsPerJid = new LinkedHashMap<>(); // -------------------------------------------------------------------------------------------- @@ -234,7 +234,7 @@ private SimpleSlot addSharedSlotAndAllocateSubSlot( // can place a task into this slot. boolean entryForNewJidExists = false; - for (Map.Entry>> entry : availableSlotsPerJid.entrySet()) { + for (Map.Entry>> entry : availableSlotsPerJid.entrySet()) { // there is already an entry for this groupID if (entry.getKey().equals(groupIdForMap)) { entryForNewJidExists = true; @@ -247,7 +247,7 @@ private SimpleSlot addSharedSlotAndAllocateSubSlot( // make sure an empty entry exists for this group, if no other entry exists if (!entryForNewJidExists) { - availableSlotsPerJid.put(groupIdForMap, new LinkedHashMap<>()); + availableSlotsPerJid.put(groupIdForMap, new LinkedHashMap>()); } return subSlot; @@ -393,7 +393,7 @@ public Tuple2 getSharedSlotForTask( } // get the available slots for the group - LinkedHashMap> slotsForGroup = availableSlotsPerJid.get(groupId); + Map> slotsForGroup = availableSlotsPerJid.get(groupId); if (slotsForGroup == null) { // we have a new group, so all slots are available @@ -624,26 +624,20 @@ private static SharedSlot removeFromMultiMap(Map> m private static SharedSlot pollFromMultiMap(Map> map) { Iterator>> iter = map.entrySet().iterator(); - + while (iter.hasNext()) { - Map.Entry> slotEntry = iter.next(); - - // remove first entry to add it at the back if there are still slots left - iter.remove(); - - List slots = slotEntry.getValue(); - - if (!slots.isEmpty()) { - - SharedSlot result = slots.remove(slots.size() - 1); - - if (!slots.isEmpty()) { - // reinserts the entry; since it is a LinkedHashMap, we will iterate over this entry - // only after having polled from all other entries - map.put(slotEntry.getKey(), slots); - } - - return result; + List slots = iter.next().getValue(); + + if (slots.isEmpty()) { + iter.remove(); + } + else if (slots.size() == 1) { + SharedSlot slot = slots.remove(0); + iter.remove(); + return slot; + } + else { + return slots.remove(slots.size() - 1); } } @@ -651,11 +645,11 @@ private static SharedSlot pollFromMultiMap(Map> map } private static void removeSlotFromAllEntries( - Map>> availableSlots, - SharedSlot slot) { + Map>> availableSlots, SharedSlot slot) + { final ResourceID taskManagerId = slot.getTaskManagerID(); - for (Map.Entry>> entry : availableSlots.entrySet()) { + for (Map.Entry>> entry : availableSlots.entrySet()) { Map> map = entry.getValue(); List list = map.get(taskManagerId); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignmentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignmentTest.java deleted file mode 100644 index 2407c1df01906e..00000000000000 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/SlotSharingGroupAssignmentTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.instance; - -import org.apache.flink.runtime.clusterframework.types.ResourceID; -import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.runtime.jobmanager.scheduler.Locality; -import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; -import org.apache.flink.runtime.jobmaster.SlotOwner; -import org.apache.flink.runtime.taskmanager.TaskManagerLocation; -import org.apache.flink.util.TestLogger; - -import org.junit.Test; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; - -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.mock; - -public class SlotSharingGroupAssignmentTest extends TestLogger { - - /** - * Tests that slots are allocated in a round robin fashion from the set of available resources. - */ - @Test - public void testRoundRobinPolling() throws UnknownHostException { - final SlotSharingGroupAssignment slotSharingGroupAssignment = new SlotSharingGroupAssignment(); - final int numberTaskManagers = 2; - final int numberSlots = 2; - final JobVertexID sourceId = new JobVertexID(); - final JobVertexID sinkId = new JobVertexID(); - - for (int i = 0; i < numberTaskManagers; i++) { - final TaskManagerLocation taskManagerLocation = new TaskManagerLocation(ResourceID.generate(), InetAddress.getLocalHost(), i + 1000); - - for (int j = 0; j < numberSlots; j++) { - final SharedSlot slot = new SharedSlot( - mock(SlotOwner.class), - taskManagerLocation, - j, - mock(TaskManagerGateway.class), - slotSharingGroupAssignment); - - slotSharingGroupAssignment.addSharedSlotAndAllocateSubSlot(slot, Locality.UNKNOWN, sourceId); - } - } - - SimpleSlot allocatedSlot1 = slotSharingGroupAssignment.getSlotForTask(sinkId, Collections.emptyList()); - SimpleSlot allocatedSlot2 = slotSharingGroupAssignment.getSlotForTask(sinkId, Collections.emptyList()); - - assertNotEquals(allocatedSlot1.getTaskManagerLocation(), allocatedSlot2.getTaskManagerLocation()); - - // let's check that we can still allocate all 4 slots - SimpleSlot allocatedSlot3 = slotSharingGroupAssignment.getSlotForTask(sinkId, Collections.emptyList()); - assertNotNull(allocatedSlot3); - - SimpleSlot allocatedSlot4 = slotSharingGroupAssignment.getSlotForTask(sinkId, Collections.emptyList()); - assertNotNull(allocatedSlot4); - } -} From c4a1d09ccdbd19416e15534ceba45ead5d2d6ed2 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 16 Mar 2018 19:16:49 +0100 Subject: [PATCH 0184/2294] [FLINK-9016] [flip6] Properly unregister jobs from JobMetricGroup This commit properly removes jobs from the JobMetricGroup once a job has reached a terminal state. --- .../flink/runtime/dispatcher/Dispatcher.java | 33 +++++++++++++------ .../runtime/dispatcher/MiniDispatcher.java | 8 +++-- .../dispatcher/StandaloneDispatcher.java | 8 +++-- .../runtime/entrypoint/ClusterEntrypoint.java | 27 +++++++++++++-- .../entrypoint/JobClusterEntrypoint.java | 8 +++-- .../entrypoint/SessionClusterEntrypoint.java | 26 ++++++++------- .../runtime/jobmaster/JobManagerRunner.java | 31 +++-------------- .../flink/runtime/jobmaster/JobMaster.java | 15 +++------ .../runtime/minicluster/MiniCluster.java | 14 +++++++- .../runtime/dispatcher/DispatcherTest.java | 20 +++++++---- .../dispatcher/MiniDispatcherTest.java | 10 +++--- .../jobmaster/JobManagerRunnerTest.java | 10 ++---- .../runtime/jobmaster/JobMasterTest.java | 3 +- 13 files changed, 122 insertions(+), 91 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java index 9b2411c66b789c..91a4f73bf53847 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java @@ -49,7 +49,8 @@ import org.apache.flink.runtime.messages.webmonitor.JobDetails; import org.apache.flink.runtime.messages.webmonitor.JobsOverview; import org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails; -import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.resourcemanager.ResourceOverview; import org.apache.flink.runtime.rest.handler.legacy.backpressure.OperatorBackPressureStatsResponse; @@ -97,7 +98,6 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme private final JobManagerSharedServices jobManagerSharedServices; private final HeartbeatServices heartbeatServices; private final BlobServer blobServer; - private final MetricRegistry metricRegistry; private final FatalErrorHandler fatalErrorHandler; @@ -109,6 +109,11 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme private final JobManagerRunnerFactory jobManagerRunnerFactory; + private final JobManagerMetricGroup jobManagerMetricGroup; + + @Nullable + private final String metricQueryServicePath; + @Nullable protected final String restAddress; @@ -123,7 +128,8 @@ public Dispatcher( ResourceManagerGateway resourceManagerGateway, BlobServer blobServer, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, + JobManagerMetricGroup jobManagerMetricGroup, + @Nullable String metricServiceQueryPath, ArchivedExecutionGraphStore archivedExecutionGraphStore, JobManagerRunnerFactory jobManagerRunnerFactory, FatalErrorHandler fatalErrorHandler, @@ -135,9 +141,10 @@ public Dispatcher( this.resourceManagerGateway = Preconditions.checkNotNull(resourceManagerGateway); this.heartbeatServices = Preconditions.checkNotNull(heartbeatServices); this.blobServer = Preconditions.checkNotNull(blobServer); - this.metricRegistry = Preconditions.checkNotNull(metricRegistry); this.fatalErrorHandler = Preconditions.checkNotNull(fatalErrorHandler); this.submittedJobGraphStore = Preconditions.checkNotNull(submittedJobGraphStore); + this.jobManagerMetricGroup = Preconditions.checkNotNull(jobManagerMetricGroup); + this.metricQueryServicePath = metricServiceQueryPath; this.jobManagerSharedServices = JobManagerSharedServices.fromConfiguration( configuration, @@ -192,6 +199,8 @@ public CompletableFuture postStop() { exception = ExceptionUtils.firstOrSuppressed(e, exception); } + jobManagerMetricGroup.close(); + if (exception != null) { throw exception; } else { @@ -251,7 +260,8 @@ public CompletableFuture submitJob(JobGraph jobGraph, Time timeout) heartbeatServices, blobServer, jobManagerSharedServices, - metricRegistry, + jobManagerMetricGroup.addJob(jobGraph), + metricQueryServicePath, restAddress); jobManagerRunner.getResultFuture().whenCompleteAsync( @@ -464,8 +474,6 @@ public CompletableFuture requestJobResult(JobID jobId, Time timeout) @Override public CompletableFuture> requestMetricQueryServicePaths(Time timeout) { - final String metricQueryServicePath = metricRegistry.getMetricQueryServicePath(); - if (metricQueryServicePath != null) { return CompletableFuture.completedFuture(Collections.singleton(metricQueryServicePath)); } else { @@ -513,6 +521,8 @@ private void removeJob(JobID jobId, boolean cleanupHA) throws Exception { registerOrphanedJobManagerTerminationFuture(jobManagerRunnerTerminationFuture); } + jobManagerMetricGroup.removeJob(jobId); + if (cleanupHA) { submittedJobGraphStore.removeJobGraph(jobId); } @@ -725,7 +735,8 @@ JobManagerRunner createJobManagerRunner( HeartbeatServices heartbeatServices, BlobServer blobServer, JobManagerSharedServices jobManagerServices, - MetricRegistry metricRegistry, + JobManagerJobMetricGroup jobManagerJobMetricGroup, + @Nullable String metricQueryServicePath, @Nullable String restAddress) throws Exception; } @@ -745,7 +756,8 @@ public JobManagerRunner createJobManagerRunner( HeartbeatServices heartbeatServices, BlobServer blobServer, JobManagerSharedServices jobManagerServices, - MetricRegistry metricRegistry, + JobManagerJobMetricGroup jobManagerJobMetricGroup, + @Nullable String metricQueryServicePath, @Nullable String restAddress) throws Exception { return new JobManagerRunner( resourceId, @@ -756,7 +768,8 @@ public JobManagerRunner createJobManagerRunner( heartbeatServices, blobServer, jobManagerServices, - metricRegistry, + jobManagerJobMetricGroup, + metricQueryServicePath, restAddress); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java index c648131a7c09e9..3f458248fff77d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java @@ -30,7 +30,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; @@ -60,7 +60,8 @@ public MiniDispatcher( ResourceManagerGateway resourceManagerGateway, BlobServer blobServer, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, + JobManagerMetricGroup jobManagerMetricGroup, + @Nullable String metricQueryServicePath, ArchivedExecutionGraphStore archivedExecutionGraphStore, JobManagerRunnerFactory jobManagerRunnerFactory, FatalErrorHandler fatalErrorHandler, @@ -76,7 +77,8 @@ public MiniDispatcher( resourceManagerGateway, blobServer, heartbeatServices, - metricRegistry, + jobManagerMetricGroup, + metricQueryServicePath, archivedExecutionGraphStore, jobManagerRunnerFactory, fatalErrorHandler, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java index a7d21f3faaa080..52ac7a0606a570 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java @@ -24,7 +24,7 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmaster.JobMaster; -import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; @@ -45,7 +45,8 @@ public StandaloneDispatcher( ResourceManagerGateway resourceManagerGateway, BlobServer blobServer, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, + JobManagerMetricGroup jobManagerMetricGroup, + @Nullable String metricQueryServicePath, ArchivedExecutionGraphStore archivedExecutionGraphStore, JobManagerRunnerFactory jobManagerRunnerFactory, FatalErrorHandler fatalErrorHandler, @@ -59,7 +60,8 @@ public StandaloneDispatcher( resourceManagerGateway, blobServer, heartbeatServices, - metricRegistry, + jobManagerMetricGroup, + metricQueryServicePath, archivedExecutionGraphStore, jobManagerRunnerFactory, fatalErrorHandler, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 07b3b683a3f289..676415b18e1ff3 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -51,6 +51,8 @@ import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.MetricRegistryConfiguration; import org.apache.flink.runtime.metrics.MetricRegistryImpl; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.util.MetricUtils; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.resourcemanager.ResourceManagerId; @@ -154,6 +156,9 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler { @GuardedBy("lock") private ClusterInformation clusterInformation; + @GuardedBy("lock") + private JobManagerMetricGroup jobManagerMetricGroup; + protected ClusterEntrypoint(Configuration configuration) { this.configuration = Preconditions.checkNotNull(configuration); this.terminationFuture = new CompletableFuture<>(); @@ -327,6 +332,8 @@ protected void startClusterComponents( clusterInformation, webMonitorEndpoint.getRestAddress()); + jobManagerMetricGroup = MetricUtils.instantiateJobManagerMetricGroup(metricRegistry, rpcService.getAddress()); + dispatcher = createDispatcher( configuration, rpcService, @@ -334,7 +341,8 @@ protected void startClusterComponents( resourceManager.getSelfGateway(ResourceManagerGateway.class), blobServer, heartbeatServices, - metricRegistry, + jobManagerMetricGroup, + metricRegistry.getMetricQueryServicePath(), archivedExecutionGraphStore, this, webMonitorEndpoint.getRestAddress()); @@ -488,7 +496,19 @@ protected CompletableFuture stopClusterComponents() { terminationFutures.add(FutureUtils.completedExceptionally(exception)); } - return FutureUtils.completeAll(terminationFutures); + final CompletableFuture componentTerminationFuture = FutureUtils.completeAll(terminationFutures); + + if (jobManagerMetricGroup != null) { + return FutureUtils.runAfterwards( + componentTerminationFuture, + () -> { + synchronized (lock) { + jobManagerMetricGroup.close(); + } + }); + } else { + return componentTerminationFuture; + } } } @@ -567,7 +587,8 @@ protected abstract Dispatcher createDispatcher( ResourceManagerGateway resourceManagerGateway, BlobServer blobServer, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, + JobManagerMetricGroup jobManagerMetricGroup, + @Nullable String metricQueryServicePath, ArchivedExecutionGraphStore archivedExecutionGraphStore, FatalErrorHandler fatalErrorHandler, @Nullable String restAddress) throws Exception; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java index dc211d84060537..df950a343be944 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java @@ -32,7 +32,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmaster.MiniDispatcherRestEndpoint; import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.rest.RestServerEndpointConfiguration; import org.apache.flink.runtime.rest.handler.RestHandlerConfiguration; @@ -95,7 +95,8 @@ protected Dispatcher createDispatcher( ResourceManagerGateway resourceManagerGateway, BlobServer blobServer, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, + JobManagerMetricGroup jobManagerMetricGroup, + @Nullable String metricQueryServicePath, ArchivedExecutionGraphStore archivedExecutionGraphStore, FatalErrorHandler fatalErrorHandler, @Nullable String restAddress) throws Exception { @@ -114,7 +115,8 @@ protected Dispatcher createDispatcher( resourceManagerGateway, blobServer, heartbeatServices, - metricRegistry, + jobManagerMetricGroup, + metricQueryServicePath, archivedExecutionGraphStore, Dispatcher.DefaultJobManagerRunnerFactory.INSTANCE, fatalErrorHandler, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java index 764356d036a85c..fcab796ffd54e9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java @@ -34,7 +34,7 @@ import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.rest.RestServerEndpointConfiguration; import org.apache.flink.runtime.rest.handler.RestHandlerConfiguration; @@ -104,16 +104,17 @@ protected DispatcherRestEndpoint createRestEndpoint( @Override protected Dispatcher createDispatcher( - Configuration configuration, - RpcService rpcService, - HighAvailabilityServices highAvailabilityServices, - ResourceManagerGateway resourceManagerGateway, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - FatalErrorHandler fatalErrorHandler, - @Nullable String restAddress) throws Exception { + Configuration configuration, + RpcService rpcService, + HighAvailabilityServices highAvailabilityServices, + ResourceManagerGateway resourceManagerGateway, + BlobServer blobServer, + HeartbeatServices heartbeatServices, + JobManagerMetricGroup jobManagerMetricGroup, + @Nullable String metricQueryServicePath, + ArchivedExecutionGraphStore archivedExecutionGraphStore, + FatalErrorHandler fatalErrorHandler, + @Nullable String restAddress) throws Exception { // create the default dispatcher return new StandaloneDispatcher( @@ -124,7 +125,8 @@ protected Dispatcher createDispatcher( resourceManagerGateway, blobServer, heartbeatServices, - metricRegistry, + jobManagerMetricGroup, + metricQueryServicePath, archivedExecutionGraphStore, Dispatcher.DefaultJobManagerRunnerFactory.INSTANCE, fatalErrorHandler, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java index 8b64f0ddf57e97..cd2852b7fae18d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java @@ -35,9 +35,7 @@ import org.apache.flink.runtime.leaderelection.LeaderContender; import org.apache.flink.runtime.leaderelection.LeaderElectionService; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.metrics.util.MetricUtils; +import org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup; import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.util.AutoCloseableAsync; @@ -82,8 +80,6 @@ public class JobManagerRunner implements LeaderContender, OnCompletionActions, F private final JobMaster jobManager; - private final JobManagerMetricGroup jobManagerMetricGroup; - private final Time rpcTimeout; private final CompletableFuture resultFuture; @@ -111,11 +107,10 @@ public JobManagerRunner( final HeartbeatServices heartbeatServices, final BlobServer blobServer, final JobManagerSharedServices jobManagerSharedServices, - final MetricRegistry metricRegistry, + final JobManagerJobMetricGroup jobManagerJobMetricGroup, + @Nullable final String metricQueryServicePath, @Nullable final String restAddress) throws Exception { - JobManagerMetricGroup jobManagerMetrics = null; - this.resultFuture = new CompletableFuture<>(); this.terminationFuture = new CompletableFuture<>(); @@ -126,10 +121,6 @@ public JobManagerRunner( checkArgument(jobGraph.getNumberOfVertices() > 0, "The given job is empty"); - final String hostAddress = rpcService.getAddress().isEmpty() ? "localhost" : rpcService.getAddress(); - jobManagerMetrics = MetricUtils.instantiateJobManagerMetricGroup(metricRegistry, hostAddress); - this.jobManagerMetricGroup = jobManagerMetrics; - // libraries and class loader first final LibraryCacheManager libraryCacheManager = jobManagerSharedServices.getLibraryCacheManager(); try { @@ -162,19 +153,14 @@ public JobManagerRunner( jobManagerSharedServices, heartbeatServices, blobServer, - jobManagerMetrics, + jobManagerJobMetricGroup, this, this, userCodeLoader, restAddress, - metricRegistry.getMetricQueryServicePath()); + metricQueryServicePath); } catch (Throwable t) { - // clean up everything - if (jobManagerMetrics != null) { - jobManagerMetrics.close(); - } - terminationFuture.completeExceptionally(t); resultFuture.completeExceptionally(t); @@ -230,13 +216,6 @@ public CompletableFuture closeAsync() { throwable = ExceptionUtils.firstOrSuppressed(t, ExceptionUtils.stripCompletionException(throwable)); } - // make all registered metrics go away - try { - jobManagerMetricGroup.close(); - } catch (Throwable t) { - throwable = ExceptionUtils.firstOrSuppressed(t, throwable); - } - if (throwable != null) { terminationFuture.completeExceptionally( new FlinkException("Could not properly shut down the JobManagerRunner", throwable)); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index 74f9b656c6f0ed..ced8c7c4dd89e8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -24,7 +24,6 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.core.io.InputSplit; import org.apache.flink.core.io.InputSplitAssigner; -import org.apache.flink.metrics.MetricGroup; import org.apache.flink.queryablestate.KvStateID; import org.apache.flink.runtime.JobException; import org.apache.flink.runtime.StoppingException; @@ -76,8 +75,7 @@ import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint; import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; import org.apache.flink.runtime.messages.webmonitor.JobDetails; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; +import org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup; import org.apache.flink.runtime.query.KvStateLocation; import org.apache.flink.runtime.query.KvStateLocationRegistry; import org.apache.flink.runtime.query.UnknownKvStateLocation; @@ -165,7 +163,7 @@ public class JobMaster extends FencedRpcEndpoint implements JobMast private final BlobServer blobServer; /** The metrics for the job. */ - private final MetricGroup jobMetricGroup; + private final JobManagerJobMetricGroup jobMetricGroup; /** The heartbeat manager with task managers. */ private final HeartbeatManager taskManagerHeartbeatManager; @@ -225,7 +223,7 @@ public JobMaster( JobManagerSharedServices jobManagerSharedServices, HeartbeatServices heartbeatServices, BlobServer blobServer, - @Nullable JobManagerMetricGroup jobManagerMetricGroup, + JobManagerJobMetricGroup jobMetricGroup, OnCompletionActions jobCompletionActions, FatalErrorHandler errorHandler, ClassLoader userCodeLoader, @@ -246,6 +244,7 @@ public JobMaster( this.jobCompletionActions = checkNotNull(jobCompletionActions); this.errorHandler = checkNotNull(errorHandler); this.userCodeLoader = checkNotNull(userCodeLoader); + this.jobMetricGroup = checkNotNull(jobMetricGroup); this.taskManagerHeartbeatManager = heartbeatServices.createHeartbeatManagerSender( resourceId, @@ -262,12 +261,6 @@ public JobMaster( final String jobName = jobGraph.getName(); final JobID jid = jobGraph.getJobID(); - if (jobManagerMetricGroup != null) { - this.jobMetricGroup = jobManagerMetricGroup.addJob(jobGraph); - } else { - this.jobMetricGroup = UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(); - } - log.info("Initializing job {} ({}).", jobName, jid); final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration = diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 98c8ca22e9ba43..d660c6758b3d3c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -50,6 +50,8 @@ import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.MetricRegistryConfiguration; import org.apache.flink.runtime.metrics.MetricRegistryImpl; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.util.MetricUtils; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.resourcemanager.ResourceManagerId; import org.apache.flink.runtime.resourcemanager.ResourceManagerRunner; @@ -153,6 +155,9 @@ public class MiniCluster implements JobExecutorService, AutoCloseableAsync { @GuardedBy("lock") private StandaloneDispatcher dispatcher; + @GuardedBy("lock") + private JobManagerMetricGroup jobManagerMetricGroup; + @GuardedBy("lock") private RpcGatewayRetriever dispatcherGatewayRetriever; @@ -344,6 +349,8 @@ public void start() throws Exception { // bring up the dispatcher that launches JobManagers when jobs submitted LOG.info("Starting job dispatcher(s) for JobManger"); + this.jobManagerMetricGroup = MetricUtils.instantiateJobManagerMetricGroup(metricRegistry, "localhost"); + dispatcher = new StandaloneDispatcher( jobManagerRpcService, Dispatcher.DISPATCHER_NAME + UUID.randomUUID(), @@ -352,7 +359,8 @@ public void start() throws Exception { resourceManagerRunner.getResourceManageGateway(), blobServer, heartbeatServices, - metricRegistry, + jobManagerMetricGroup, + metricRegistry.getMetricQueryServicePath(), new MemoryArchivedExecutionGraphStore(), Dispatcher.DefaultJobManagerRunnerFactory.INSTANCE, new ShutDownFatalErrorHandler(), @@ -424,6 +432,10 @@ public CompletableFuture closeAsync() { componentsTerminationFuture, () -> { synchronized (lock) { + if (jobManagerMetricGroup != null) { + jobManagerMetricGroup.close(); + jobManagerMetricGroup = null; + } // metrics shutdown if (metricRegistry != null) { metricRegistry.shutdown(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java index 82679216205fc7..71c391f20a7c3a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java @@ -45,8 +45,9 @@ import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.FlinkJobNotFoundException; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; +import org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder; import org.apache.flink.runtime.rpc.FatalErrorHandler; @@ -177,7 +178,8 @@ public void setUp() throws Exception { mock(ResourceManagerGateway.class), new BlobServer(blobServerConfig, new VoidBlobStore()), heartbeatServices, - NoOpMetricRegistry.INSTANCE, + UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), + null, new MemoryArchivedExecutionGraphStore(), fatalErrorHandler, TEST_JOB_ID); @@ -360,7 +362,8 @@ private TestingDispatcher( ResourceManagerGateway resourceManagerGateway, BlobServer blobServer, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, + JobManagerMetricGroup jobManagerMetricGroup, + @Nullable String metricQueryServicePath, ArchivedExecutionGraphStore archivedExecutionGraphStore, FatalErrorHandler fatalErrorHandler, JobID expectedJobId) throws Exception { @@ -373,7 +376,8 @@ private TestingDispatcher( resourceManagerGateway, blobServer, heartbeatServices, - metricRegistry, + jobManagerMetricGroup, + metricQueryServicePath, archivedExecutionGraphStore, new ExpectedJobIdJobManagerRunnerFactory(expectedJobId), fatalErrorHandler, @@ -421,7 +425,8 @@ public JobManagerRunner createJobManagerRunner( HeartbeatServices heartbeatServices, BlobServer blobServer, JobManagerSharedServices jobManagerSharedServices, - MetricRegistry metricRegistry, + JobManagerJobMetricGroup jobManagerJobMetricGroup, + @Nullable String metricQueryServicePath, @Nullable String restAddress) throws Exception { assertEquals(expectedJobId, jobGraph.getJobID()); @@ -434,7 +439,8 @@ public JobManagerRunner createJobManagerRunner( heartbeatServices, blobServer, jobManagerSharedServices, - metricRegistry, + jobManagerJobMetricGroup, + metricQueryServicePath, restAddress); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java index c6eda2e816a416..651200f6096e02 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java @@ -35,8 +35,8 @@ import org.apache.flink.runtime.jobmaster.JobManagerSharedServices; import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; +import org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup; +import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway; import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder; import org.apache.flink.runtime.rpc.RpcService; @@ -254,7 +254,8 @@ private MiniDispatcher createMiniDispatcher(ClusterEntrypoint.ExecutionMode exec resourceManagerGateway, blobServer, heartbeatServices, - NoOpMetricRegistry.INSTANCE, + UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), + null, archivedExecutionGraphStore, testingJobManagerRunnerFactory, testingFatalErrorHandler, @@ -283,7 +284,8 @@ public JobManagerRunner createJobManagerRunner( HeartbeatServices heartbeatServices, BlobServer blobServer, JobManagerSharedServices jobManagerSharedServices, - MetricRegistry metricRegistry, + JobManagerJobMetricGroup jobManagerJobMetricGroup, + @Nullable String metricQueryServicePath, @Nullable String restAddress) throws Exception { jobGraphFuture.complete(jobGraph); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java index 9730ddef27e5a6..1d7f0906f43081 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java @@ -32,8 +32,7 @@ import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; +import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder; import org.apache.flink.runtime.rpc.TestingRpcService; import org.apache.flink.runtime.testtasks.NoOpInvokable; @@ -76,8 +75,6 @@ public class JobManagerRunnerTest extends TestLogger { private static JobManagerSharedServices jobManagerSharedServices; - private static MetricRegistry metricRegistry; - private static JobGraph jobGraph; private static ArchivedExecutionGraph archivedExecutionGraph; @@ -97,8 +94,6 @@ public static void setupClass() throws Exception { jobManagerSharedServices = JobManagerSharedServices.fromConfiguration(configuration, blobServer); - metricRegistry = NoOpMetricRegistry.INSTANCE; - final JobVertex jobVertex = new JobVertex("Test vertex"); jobVertex.setInvokableClass(NoOpInvokable.class); jobGraph = new JobGraph(jobVertex); @@ -215,7 +210,8 @@ private JobManagerRunner createJobManagerRunner() throws Exception { heartbeatServices, blobServer, jobManagerSharedServices, - metricRegistry, + UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(), + null, null); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java index b5430569e1913c..ed5a8946b1caa4 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java @@ -47,6 +47,7 @@ import org.apache.flink.runtime.jobmanager.OnCompletionActions; import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; +import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.ResourceManagerId; import org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway; @@ -410,7 +411,7 @@ private JobMaster createJobMaster( jobManagerSharedServices, fastHeartbeatServices, blobServer, - null, + UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(), new NoOpOnCompletionActions(), testingFatalErrorHandler, JobMasterTest.class.getClassLoader(), From 7caeefdcbe6dab89bed4a9e2fa42e07722fbf08d Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Sat, 17 Mar 2018 17:53:35 +0100 Subject: [PATCH 0185/2294] [FLINK-8812] [flip6] Set managed memory for TaskExecutor to 80 MB in MiniCluster In order to avoid problems with OOM exceptions, this commit sets the managed memory to 80 MB for TaskExecutors started by the MiniCluster. This closes #5713. --- .../java/org/apache/flink/test/util/MiniClusterResource.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java index 2f12bdc347e846..dbd292cd2d07bb 100644 --- a/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java +++ b/flink-test-utils-parent/flink-test-utils/src/main/java/org/apache/flink/test/util/MiniClusterResource.java @@ -26,6 +26,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; import org.apache.flink.configuration.RestOptions; +import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.minicluster.JobExecutorService; import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; @@ -194,6 +195,10 @@ private void startFlip6MiniCluster() throws Exception { configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); } + if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) { + configuration.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, TestBaseUtils.TASK_MANAGER_MEMORY_SIZE); + } + // set rest port to 0 to avoid clashes with concurrent MiniClusters configuration.setInteger(RestOptions.REST_PORT, 0); From 9538675caca24c83201e47bd26e4e725b2695217 Mon Sep 17 00:00:00 2001 From: gyao Date: Fri, 9 Mar 2018 14:36:33 +0100 Subject: [PATCH 0186/2294] [FLINK-7804][flip6] Run AMRMClientAsync callbacks in main thread This closes #5675. --- .../flink/yarn/YarnResourceManager.java | 107 ++++++++++-------- .../flink/yarn/YarnResourceManagerTest.java | 15 +-- 2 files changed, 65 insertions(+), 57 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index af789baf7e95a3..97db2ad8a37a12 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -316,7 +316,10 @@ protected YarnWorkerNode workerStarted(ResourceID resourceID) { return workerNodeMap.get(resourceID); } - // AMRMClientAsync CallbackHandler methods + // ------------------------------------------------------------------------ + // AMRMClientAsync CallbackHandler methods + // ------------------------------------------------------------------------ + @Override public float getProgress() { // Temporarily need not record the total size of asked and allocated containers @@ -325,67 +328,68 @@ public float getProgress() { @Override public void onContainersCompleted(List list) { - for (ContainerStatus container : list) { - if (container.getExitStatus() < 0) { - closeTaskManagerConnection(new ResourceID( - container.getContainerId().toString()), new Exception(container.getDiagnostics())); + runAsync(() -> { + for (ContainerStatus container : list) { + if (container.getExitStatus() < 0) { + closeTaskManagerConnection(new ResourceID( + container.getContainerId().toString()), new Exception(container.getDiagnostics())); + } + workerNodeMap.remove(new ResourceID(container.getContainerId().toString())); + } } - workerNodeMap.remove(new ResourceID(container.getContainerId().toString())); - } + ); } @Override public void onContainersAllocated(List containers) { - for (Container container : containers) { - log.info( - "Received new container: {} - Remaining pending container requests: {}", - container.getId(), - numPendingContainerRequests); - - if (numPendingContainerRequests > 0) { - numPendingContainerRequests--; - - final String containerIdStr = container.getId().toString(); - - workerNodeMap.put(new ResourceID(containerIdStr), new YarnWorkerNode(container)); - - try { - // Context information used to start a TaskExecutor Java process - ContainerLaunchContext taskExecutorLaunchContext = createTaskExecutorLaunchContext( - container.getResource(), - containerIdStr, - container.getNodeId().getHost()); - - nodeManagerClient.startContainer(container, taskExecutorLaunchContext); - } catch (Throwable t) { - log.error("Could not start TaskManager in container {}.", container.getId(), t); - - // release the failed container + runAsync(() -> { + for (Container container : containers) { + log.info( + "Received new container: {} - Remaining pending container requests: {}", + container.getId(), + numPendingContainerRequests); + + if (numPendingContainerRequests > 0) { + numPendingContainerRequests--; + + final String containerIdStr = container.getId().toString(); + + workerNodeMap.put(new ResourceID(containerIdStr), new YarnWorkerNode(container)); + + try { + // Context information used to start a TaskExecutor Java process + ContainerLaunchContext taskExecutorLaunchContext = createTaskExecutorLaunchContext( + container.getResource(), + containerIdStr, + container.getNodeId().getHost()); + + nodeManagerClient.startContainer(container, taskExecutorLaunchContext); + } catch (Throwable t) { + log.error("Could not start TaskManager in container {}.", container.getId(), t); + + // release the failed container + resourceManagerClient.releaseAssignedContainer(container.getId()); + // and ask for a new one + requestYarnContainer(container.getResource(), container.getPriority()); + } + } else { + // return the excessive containers + log.info("Returning excess container {}.", container.getId()); resourceManagerClient.releaseAssignedContainer(container.getId()); - // and ask for a new one - requestYarnContainer(container.getResource(), container.getPriority()); } - } else { - // return the excessive containers - log.info("Returning excess container {}.", container.getId()); - resourceManagerClient.releaseAssignedContainer(container.getId()); } - } - // if we are waiting for no further containers, we can go to the - // regular heartbeat interval - if (numPendingContainerRequests <= 0) { - resourceManagerClient.setHeartbeatInterval(yarnHeartbeatIntervalMillis); - } + // if we are waiting for no further containers, we can go to the + // regular heartbeat interval + if (numPendingContainerRequests <= 0) { + resourceManagerClient.setHeartbeatInterval(yarnHeartbeatIntervalMillis); + } + }); } @Override public void onShutdownRequest() { - try { - shutDown(); - } catch (Exception e) { - log.warn("Fail to shutdown the YARN resource manager.", e); - } + shutDown(); } @Override @@ -398,7 +402,10 @@ public void onError(Throwable error) { onFatalError(error); } - //Utility methods + // ------------------------------------------------------------------------ + // Utility methods + // ------------------------------------------------------------------------ + /** * Converts a Flink application status enum to a YARN application status enum. * @param status The Flink application status. diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java index 455abc9596e9ce..0d37b8ed6bf986 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java @@ -72,8 +72,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; @@ -105,14 +103,12 @@ */ public class YarnResourceManagerTest extends TestLogger { - private static final Logger LOG = LoggerFactory.getLogger(YarnResourceManagerTest.class); + private static final Time TIMEOUT = Time.seconds(10L); private static Configuration flinkConfig = new Configuration(); private static Map env = new HashMap<>(); - private static final Time timeout = Time.seconds(10L); - @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -178,7 +174,7 @@ public TestingYarnResourceManager( } public CompletableFuture runInMainThread(Callable callable) { - return callAsync(callable, timeout); + return callAsync(callable, TIMEOUT); } public MainThreadExecutor getMainThreadExecutorForTesting() { @@ -197,6 +193,11 @@ protected AMRMClientAsync createAndStartResourceMan protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) { return mockNMClient; } + + @Override + protected void runAsync(final Runnable runnable) { + runnable.run(); + } } static class Context { @@ -292,7 +293,7 @@ class MockResourceManagerRuntimeServices { public void grantLeadership() throws Exception { rmLeaderSessionId = UUID.randomUUID(); - rmLeaderElectionService.isLeader(rmLeaderSessionId).get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS); + rmLeaderElectionService.isLeader(rmLeaderSessionId).get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS); } } From 0caff357848929b736fca0f78e358f77423ce355 Mon Sep 17 00:00:00 2001 From: gyao Date: Fri, 16 Mar 2018 06:57:24 +0100 Subject: [PATCH 0187/2294] [hotfix][flip6] Only create new terminationFuture if MiniCluster is running --- .../org/apache/flink/runtime/minicluster/MiniCluster.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index d660c6758b3d3c..74aa3886281193 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -217,9 +217,6 @@ public void start() throws Exception { LOG.info("Starting Flink Mini Cluster"); LOG.debug("Using configuration {}", miniClusterConfiguration); - // create a new termination future - terminationFuture = new CompletableFuture<>(); - final Configuration configuration = miniClusterConfiguration.getConfiguration(); final Time rpcTimeout = miniClusterConfiguration.getRpcTimeout(); final int numTaskManagers = miniClusterConfiguration.getNumTaskManagers(); @@ -384,6 +381,9 @@ public void start() throws Exception { throw e; } + // create a new termination future + terminationFuture = new CompletableFuture<>(); + // now officially mark this as running running = true; From efd7336fa693a9f82b9ecfb5d81c0ef747ab7801 Mon Sep 17 00:00:00 2001 From: gyao Date: Thu, 15 Mar 2018 22:04:58 +0100 Subject: [PATCH 0188/2294] [FLINK-8843][REST] Decouple bind REST address from advertised address By default bind REST server on wildcard address. Rename RestServerEndpoint#getRestAddress to getRestBaseUrl. This closes #5707. --- .../flink/configuration/RestOptions.java | 15 ++++-- .../runtime/entrypoint/ClusterEntrypoint.java | 4 +- .../HighAvailabilityServicesUtils.java | 6 ++- .../runtime/minicluster/MiniCluster.java | 4 +- .../minicluster/MiniClusterConfiguration.java | 4 ++ .../runtime/rest/RestServerEndpoint.java | 46 +++++++++++-------- .../rest/RestServerEndpointConfiguration.java | 32 +++++++++---- .../webmonitor/WebMonitorEndpoint.java | 6 +-- .../rest/RestServerEndpointITCase.java | 3 +- 9 files changed, 80 insertions(+), 40 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java index 94d7977b72541f..e7421c4dcffdd4 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/RestOptions.java @@ -29,12 +29,21 @@ public class RestOptions { /** - * The address that the server binds itself to / the client connects to. + * The address that the server binds itself to. + */ + public static final ConfigOption REST_BIND_ADDRESS = + key("rest.bind-address") + .noDefaultValue() + .withDescription("The address that the server binds itself."); + + /** + * The address that should be used by clients to connect to the server. */ public static final ConfigOption REST_ADDRESS = key("rest.address") - .defaultValue("localhost") - .withDescription("The address that the server binds itself to / the client connects to."); + .noDefaultValue() + .withDeprecatedKeys(JobManagerOptions.ADDRESS.key()) + .withDescription("The address that should be used by clients to connect to the server."); /** * The port that the server listens on / the client connects to. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 676415b18e1ff3..63c8072a826208 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -330,7 +330,7 @@ protected void startClusterComponents( metricRegistry, this, clusterInformation, - webMonitorEndpoint.getRestAddress()); + webMonitorEndpoint.getRestBaseUrl()); jobManagerMetricGroup = MetricUtils.instantiateJobManagerMetricGroup(metricRegistry, rpcService.getAddress()); @@ -345,7 +345,7 @@ protected void startClusterComponents( metricRegistry.getMetricQueryServicePath(), archivedExecutionGraphStore, this, - webMonitorEndpoint.getRestAddress()); + webMonitorEndpoint.getRestBaseUrl()); LOG.debug("Starting ResourceManager."); resourceManager.start(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java index 4f12f2bc4816d4..f19a421a0c7084 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/HighAvailabilityServicesUtils.java @@ -39,6 +39,8 @@ import java.util.concurrent.Executor; +import static org.apache.flink.util.Preconditions.checkNotNull; + /** * Utils class to instantiate {@link HighAvailabilityServices} implementations. */ @@ -97,7 +99,9 @@ public static HighAvailabilityServices createHighAvailabilityServices( addressResolution, configuration); - final String address = configuration.getString(RestOptions.REST_ADDRESS); + final String address = checkNotNull(configuration.getString(RestOptions.REST_ADDRESS), + "%s must be set", + RestOptions.REST_ADDRESS.key()); final int port = configuration.getInteger(RestOptions.REST_PORT); final boolean enableSSL = configuration.getBoolean(SecurityOptions.SSL_ENABLED); final String protocol = enableSSL ? "https://" : "http://"; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 74aa3886281193..dfe30afda5eb67 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -341,7 +341,7 @@ public void start() throws Exception { dispatcherRestEndpoint.start(); - restAddressURI = new URI(dispatcherRestEndpoint.getRestAddress()); + restAddressURI = new URI(dispatcherRestEndpoint.getRestBaseUrl()); // bring up the dispatcher that launches JobManagers when jobs submitted LOG.info("Starting job dispatcher(s) for JobManger"); @@ -361,7 +361,7 @@ public void start() throws Exception { new MemoryArchivedExecutionGraphStore(), Dispatcher.DefaultJobManagerRunnerFactory.INSTANCE, new ShutDownFatalErrorHandler(), - dispatcherRestEndpoint.getRestAddress()); + dispatcherRestEndpoint.getRestBaseUrl()); dispatcher.start(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java index 08af0c4fe89f6c..fe7669443365c2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniClusterConfiguration.java @@ -22,6 +22,7 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.UnmodifiableConfiguration; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.util.Preconditions; @@ -167,6 +168,9 @@ public Builder setCommonBindAddress(String commonBindAddress) { public MiniClusterConfiguration build() { final Configuration modifiedConfiguration = new Configuration(configuration); modifiedConfiguration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numSlotsPerTaskManager); + modifiedConfiguration.setString( + RestOptions.REST_ADDRESS, + modifiedConfiguration.getString(RestOptions.REST_ADDRESS, "localhost")); return new MiniClusterConfiguration( modifiedConfiguration, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java index a3d48431f81e43..dfb01ca2657c9f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java @@ -73,8 +73,9 @@ public abstract class RestServerEndpoint { private final Object lock = new Object(); - private final String configuredAddress; - private final int configuredPort; + private final String restAddress; + private final String restBindAddress; + private final int restBindPort; private final SSLEngine sslEngine; private final Path uploadDir; private final int maxContentLength; @@ -84,14 +85,16 @@ public abstract class RestServerEndpoint { private ServerBootstrap bootstrap; private Channel serverChannel; - private String restAddress; + private String restBaseUrl; private State state = State.CREATED; public RestServerEndpoint(RestServerEndpointConfiguration configuration) throws IOException { Preconditions.checkNotNull(configuration); - this.configuredAddress = configuration.getEndpointBindAddress(); - this.configuredPort = configuration.getEndpointBindPort(); + + this.restAddress = configuration.getRestAddress(); + this.restBindAddress = configuration.getRestBindAddress(); + this.restBindPort = configuration.getRestBindPort(); this.sslEngine = configuration.getSslEngine(); this.uploadDir = configuration.getUploadDir(); @@ -101,8 +104,6 @@ public RestServerEndpoint(RestServerEndpointConfiguration configuration) throws this.responseHeaders = configuration.getResponseHeaders(); terminationFuture = new CompletableFuture<>(); - - this.restAddress = null; } /** @@ -176,18 +177,23 @@ protected void initChannel(SocketChannel ch) { .childHandler(initializer); final ChannelFuture channel; - if (configuredAddress == null) { - channel = bootstrap.bind(configuredPort); + if (restBindAddress == null) { + channel = bootstrap.bind(restBindPort); } else { - channel = bootstrap.bind(configuredAddress, configuredPort); + channel = bootstrap.bind(restBindAddress, restBindPort); } serverChannel = channel.syncUninterruptibly().channel(); - InetSocketAddress bindAddress = (InetSocketAddress) serverChannel.localAddress(); - String address = bindAddress.getAddress().getHostAddress(); - int port = bindAddress.getPort(); + final InetSocketAddress bindAddress = (InetSocketAddress) serverChannel.localAddress(); + final String advertisedAddress; + if (bindAddress.getAddress().isAnyLocalAddress()) { + advertisedAddress = this.restAddress; + } else { + advertisedAddress = bindAddress.getAddress().getHostAddress(); + } + final int port = bindAddress.getPort(); - log.info("Rest endpoint listening at {}:{}", address, port); + log.info("Rest endpoint listening at {}:{}", advertisedAddress, port); final String protocol; @@ -197,9 +203,9 @@ protected void initChannel(SocketChannel ch) { protocol = "http://"; } - restAddress = protocol + address + ':' + port; + restBaseUrl = protocol + advertisedAddress + ':' + port; - restAddressFuture.complete(restAddress); + restAddressFuture.complete(restBaseUrl); state = State.RUNNING; @@ -238,14 +244,14 @@ public InetSocketAddress getServerAddress() { } /** - * Returns the address of the REST server endpoint. + * Returns the base URL of the REST server endpoint. * - * @return REST address of this endpoint + * @return REST base URL of this endpoint */ - public String getRestAddress() { + public String getRestBaseUrl() { synchronized (lock) { Preconditions.checkState(state != State.CREATED, "The RestServerEndpoint has not been started yet."); - return restAddress; + return restBaseUrl; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java index 35bd6ea15d77b4..1fac08e53edc9d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java @@ -45,6 +45,8 @@ */ public final class RestServerEndpointConfiguration { + private final String restAddress; + @Nullable private final String restBindAddress; @@ -60,6 +62,7 @@ public final class RestServerEndpointConfiguration { private final Map responseHeaders; private RestServerEndpointConfiguration( + final String restAddress, @Nullable String restBindAddress, int restBindPort, @Nullable SSLEngine sslEngine, @@ -69,12 +72,20 @@ private RestServerEndpointConfiguration( Preconditions.checkArgument(0 <= restBindPort && restBindPort < 65536, "The bing rest port " + restBindPort + " is out of range (0, 65536["); Preconditions.checkArgument(maxContentLength > 0, "maxContentLength must be positive, was: %d", maxContentLength); + this.restAddress = requireNonNull(restAddress); this.restBindAddress = restBindAddress; this.restBindPort = restBindPort; this.sslEngine = sslEngine; this.uploadDir = requireNonNull(uploadDir); this.maxContentLength = maxContentLength; - this.responseHeaders = requireNonNull(Collections.unmodifiableMap(responseHeaders)); + this.responseHeaders = Collections.unmodifiableMap(requireNonNull(responseHeaders)); + } + + /** + * @see RestOptions#REST_ADDRESS + */ + public String getRestAddress() { + return restAddress; } /** @@ -82,7 +93,7 @@ private RestServerEndpointConfiguration( * * @return address that the REST server endpoint should bind itself to */ - public String getEndpointBindAddress() { + public String getRestBindAddress() { return restBindAddress; } @@ -91,7 +102,7 @@ public String getEndpointBindAddress() { * * @return port that the REST server endpoint should listen on */ - public int getEndpointBindPort() { + public int getRestBindPort() { return restBindPort; } @@ -136,12 +147,16 @@ public Map getResponseHeaders() { */ public static RestServerEndpointConfiguration fromConfiguration(Configuration config) throws ConfigurationException { Preconditions.checkNotNull(config); - String address = config.getString(RestOptions.REST_ADDRESS); - int port = config.getInteger(RestOptions.REST_PORT); + final String restAddress = Preconditions.checkNotNull(config.getString(RestOptions.REST_ADDRESS), + "%s must be set", + RestOptions.REST_ADDRESS.key()); + + final String restBindAddress = config.getString(RestOptions.REST_BIND_ADDRESS); + final int port = config.getInteger(RestOptions.REST_PORT); SSLEngine sslEngine = null; - boolean enableSSL = config.getBoolean(SecurityOptions.SSL_ENABLED); + final boolean enableSSL = config.getBoolean(SecurityOptions.SSL_ENABLED); if (enableSSL) { try { SSLContext sslContext = SSLUtils.createSSLServerContext(config); @@ -159,14 +174,15 @@ public static RestServerEndpointConfiguration fromConfiguration(Configuration co config.getString(WebOptions.UPLOAD_DIR, config.getString(WebOptions.TMP_DIR)), "flink-web-upload-" + UUID.randomUUID()); - int maxContentLength = config.getInteger(RestOptions.REST_SERVER_MAX_CONTENT_LENGTH); + final int maxContentLength = config.getInteger(RestOptions.REST_SERVER_MAX_CONTENT_LENGTH); final Map responseHeaders = Collections.singletonMap( HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN, config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN)); return new RestServerEndpointConfiguration( - address, + restAddress, + restBindAddress, port, sslEngine, uploadDir, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java index dfb2fc8591d97d..50ad7eb1bceeb2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java @@ -666,18 +666,18 @@ protected CompletableFuture shutDownInternal() { @Override public void grantLeadership(final UUID leaderSessionID) { - log.info("{} was granted leadership with leaderSessionID={}", getRestAddress(), leaderSessionID); + log.info("{} was granted leadership with leaderSessionID={}", getRestBaseUrl(), leaderSessionID); leaderElectionService.confirmLeaderSessionID(leaderSessionID); } @Override public void revokeLeadership() { - log.info("{} lost leadership", getRestAddress()); + log.info("{} lost leadership", getRestBaseUrl()); } @Override public String getAddress() { - return getRestAddress(); + return getRestBaseUrl(); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java index 32f3ec89cadd61..784c14158a3b4d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java @@ -115,6 +115,7 @@ public class RestServerEndpointITCase extends TestLogger { public void setup() throws Exception { Configuration config = new Configuration(); config.setInteger(RestOptions.REST_PORT, 0); + config.setString(RestOptions.REST_ADDRESS, "localhost"); config.setString(WebOptions.UPLOAD_DIR, temporaryFolder.newFolder().getCanonicalPath()); config.setInteger(RestOptions.REST_SERVER_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH); config.setInteger(RestOptions.REST_CLIENT_MAX_CONTENT_LENGTH, TEST_REST_MAX_CONTENT_LENGTH); @@ -335,7 +336,7 @@ public void testMultiPartFormDataWithoutFileUpload() throws Exception { private HttpURLConnection openHttpConnectionForUpload(final String boundary) throws IOException { final HttpURLConnection connection = - (HttpURLConnection) new URL(serverEndpoint.getRestAddress() + "/upload").openConnection(); + (HttpURLConnection) new URL(serverEndpoint.getRestBaseUrl() + "/upload").openConnection(); connection.setDoOutput(true); connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary); return connection; From 096a89ae6d0db9d763648a0acca2334f14efd3b3 Mon Sep 17 00:00:00 2001 From: gyao Date: Fri, 16 Mar 2018 20:52:11 +0100 Subject: [PATCH 0189/2294] [FLINK-8894][REST] Set object codec for JsonGenerator used by CurrentJobIdsHandler This closes #5711. --- .../handler/legacy/CurrentJobIdsHandler.java | 2 + .../legacy/CurrentJobIdsHandlerTest.java | 62 ++++++++++++++++--- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandler.java index cf8a3d6d709007..ef02762e5a53a0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandler.java @@ -22,6 +22,7 @@ import org.apache.flink.runtime.jobmaster.JobManagerGateway; import org.apache.flink.runtime.messages.webmonitor.JobIdsWithStatusOverview; import org.apache.flink.runtime.rest.messages.JobIdsWithStatusesOverviewHeaders; +import org.apache.flink.runtime.rest.util.RestMapperUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; @@ -70,6 +71,7 @@ public CompletableFuture handleJsonRequest( StringWriter writer = new StringWriter(); JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer); + gen.setCodec(RestMapperUtils.getStrictObjectMapper()); gen.writeStartObject(); gen.writeArrayFieldStart(JobIdsWithStatusOverview.FIELD_NAME_JOBS); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandlerTest.java index 0ada30d5daf081..b193122c1693d5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/CurrentJobIdsHandlerTest.java @@ -18,21 +18,69 @@ package org.apache.flink.runtime.rest.handler.legacy; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.concurrent.Executors; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.jobmaster.JobManagerGateway; +import org.apache.flink.runtime.messages.webmonitor.JobIdsWithStatusOverview; +import org.apache.flink.util.TestLogger; -import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.Collections; +import java.util.concurrent.CompletableFuture; + +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; /** - * Tests for the CurrentJobIdsHandler. + * Tests for {@link CurrentJobIdsHandler}. */ -public class CurrentJobIdsHandlerTest { +public class CurrentJobIdsHandlerTest extends TestLogger { + + private CurrentJobIdsHandler currentJobIdsHandler; + + @Mock + private JobManagerGateway mockJobManagerGateway; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + currentJobIdsHandler = new CurrentJobIdsHandler(Executors.directExecutor(), Time.seconds(0L)); + } + @Test public void testGetPaths() { - CurrentJobIdsHandler handler = new CurrentJobIdsHandler(Executors.directExecutor(), Time.seconds(0L)); - String[] paths = handler.getPaths(); - Assert.assertEquals(1, paths.length); - Assert.assertEquals("/jobs", paths[0]); + final String[] paths = currentJobIdsHandler.getPaths(); + assertEquals(1, paths.length); + assertEquals("/jobs", paths[0]); } + + @Test + public void testHandleJsonRequest() throws Exception { + final JobID jobId = new JobID(); + final JobStatus jobStatus = JobStatus.RUNNING; + + when(mockJobManagerGateway.requestJobsOverview(any(Time.class))).thenReturn( + CompletableFuture.completedFuture(new JobIdsWithStatusOverview(Collections.singleton( + new JobIdsWithStatusOverview.JobIdWithStatus(jobId, jobStatus))))); + + final CompletableFuture jsonFuture = currentJobIdsHandler.handleJsonRequest( + Collections.emptyMap(), + Collections.emptyMap(), + mockJobManagerGateway); + + final String json = jsonFuture.get(); + + assertThat(json, containsString(jobId.toString())); + assertThat(json, containsString(jobStatus.name())); + } + } From 31c0754ee70265a1b440cbfbc21e1220e5d9718a Mon Sep 17 00:00:00 2001 From: yanghua Date: Thu, 15 Mar 2018 16:24:35 +0800 Subject: [PATCH 0190/2294] [FLINK-8915] CheckpointingStatisticsHandler fails to return PendingCheckpointStats This closes #5703. --- .../checkpoints/CheckpointStatistics.java | 77 ++++++++++++++++++- .../CheckpointingStatisticsTest.java | 16 +++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatistics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatistics.java index 333c0167bee1c6..f8aeb266aaacab 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatistics.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointStatistics.java @@ -22,6 +22,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointStatsStatus; import org.apache.flink.runtime.checkpoint.CompletedCheckpointStats; import org.apache.flink.runtime.checkpoint.FailedCheckpointStats; +import org.apache.flink.runtime.checkpoint.PendingCheckpointStats; import org.apache.flink.runtime.checkpoint.TaskStateStats; import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.rest.messages.ResponseBody; @@ -50,7 +51,8 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "@class") @JsonSubTypes({ @JsonSubTypes.Type(value = CheckpointStatistics.CompletedCheckpointStatistics.class, name = "completed"), - @JsonSubTypes.Type(value = CheckpointStatistics.FailedCheckpointStatistics.class, name = "failed")}) + @JsonSubTypes.Type(value = CheckpointStatistics.FailedCheckpointStatistics.class, name = "failed"), + @JsonSubTypes.Type(value = CheckpointStatistics.PendingCheckpointStatistics.class, name = "in_progress")}) public class CheckpointStatistics implements ResponseBody { public static final String FIELD_NAME_ID = "id"; @@ -272,8 +274,25 @@ public static CheckpointStatistics generateCheckpointStatistics(AbstractCheckpoi checkpointStatisticsPerTask, failedCheckpointStats.getFailureTimestamp(), failedCheckpointStats.getFailureMessage()); + } else if (checkpointStats instanceof PendingCheckpointStats) { + final PendingCheckpointStats pendingCheckpointStats = ((PendingCheckpointStats) checkpointStats); + + return new CheckpointStatistics.PendingCheckpointStatistics( + pendingCheckpointStats.getCheckpointId(), + pendingCheckpointStats.getStatus(), + pendingCheckpointStats.getProperties().isSavepoint(), + pendingCheckpointStats.getTriggerTimestamp(), + pendingCheckpointStats.getLatestAckTimestamp(), + pendingCheckpointStats.getStateSize(), + pendingCheckpointStats.getEndToEndDuration(), + pendingCheckpointStats.getAlignmentBuffered(), + pendingCheckpointStats.getNumberOfSubtasks(), + pendingCheckpointStats.getNumberOfAcknowledgedSubtasks(), + checkpointStatisticsPerTask + ); } else { - throw new IllegalArgumentException("Given checkpoint stats object of type " + checkpointStats.getClass().getName() + " cannot be converted."); + throw new IllegalArgumentException("Given checkpoint stats object of type " + + checkpointStats.getClass().getName() + " cannot be converted."); } } @@ -438,4 +457,58 @@ public int hashCode() { return Objects.hash(super.hashCode(), failureTimestamp, failureMessage); } } + + /** + * Statistics for a pending checkpoint. + */ + public static final class PendingCheckpointStatistics extends CheckpointStatistics { + + @JsonCreator + public PendingCheckpointStatistics( + @JsonProperty(FIELD_NAME_ID) long id, + @JsonProperty(FIELD_NAME_STATUS) CheckpointStatsStatus status, + @JsonProperty(FIELD_NAME_IS_SAVEPOINT) boolean savepoint, + @JsonProperty(FIELD_NAME_TRIGGER_TIMESTAMP) long triggerTimestamp, + @JsonProperty(FIELD_NAME_LATEST_ACK_TIMESTAMP) long latestAckTimestamp, + @JsonProperty(FIELD_NAME_STATE_SIZE) long stateSize, + @JsonProperty(FIELD_NAME_DURATION) long duration, + @JsonProperty(FIELD_NAME_ALIGNMENT_BUFFERED) long alignmentBuffered, + @JsonProperty(FIELD_NAME_NUM_SUBTASKS) int numSubtasks, + @JsonProperty(FIELD_NAME_NUM_ACK_SUBTASKS) int numAckSubtasks, + @JsonDeserialize(keyUsing = JobVertexIDKeyDeserializer.class) @JsonProperty(FIELD_NAME_TASKS) Map checkpointingStatisticsPerTask) { + super( + id, + status, + savepoint, + triggerTimestamp, + latestAckTimestamp, + stateSize, + duration, + alignmentBuffered, + numSubtasks, + numAckSubtasks, + checkpointingStatisticsPerTask); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode()); + } + + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointingStatisticsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointingStatisticsTest.java index 562418e64077cd..8f25a59fd282e4 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointingStatisticsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/checkpoints/CheckpointingStatisticsTest.java @@ -122,6 +122,20 @@ protected CheckpointingStatistics getTestResponseInstance() throws Exception { true, "foobar"); + CheckpointStatistics.PendingCheckpointStatistics pending = new CheckpointStatistics.PendingCheckpointStatistics( + 5L, + CheckpointStatsStatus.IN_PROGRESS, + false, + 42L, + 41L, + 1337L, + 1L, + 0L, + 10, + 10, + Collections.emptyMap() + ); + final CheckpointingStatistics.LatestCheckpoints latestCheckpoints = new CheckpointingStatistics.LatestCheckpoints( completed, savepoint, @@ -132,6 +146,6 @@ protected CheckpointingStatistics getTestResponseInstance() throws Exception { counts, summary, latestCheckpoints, - Arrays.asList(completed, savepoint, failed)); + Arrays.asList(completed, savepoint, failed, pending)); } } From 52475b3478e78f12d5e2a9ecb10e2bf3d5133687 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Fri, 9 Mar 2018 11:05:51 +0100 Subject: [PATCH 0191/2294] [FLINK-8905][rest][client] fix RestClusterClient#getMaxSlots() returning 0 --- .../main/java/org/apache/flink/client/cli/CliFrontend.java | 3 ++- .../org/apache/flink/client/program/ClusterClient.java | 7 ++++++- .../org/apache/flink/client/program/MiniClusterClient.java | 2 +- .../flink/client/program/StandaloneClusterClient.java | 2 +- .../flink/client/program/rest/RestClusterClient.java | 2 +- .../main/java/org/apache/flink/yarn/YarnClusterClient.java | 2 +- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java index 06131dc6836b01..d636ef7c9615e1 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java @@ -86,6 +86,7 @@ import static org.apache.flink.client.cli.CliFrontendParser.HELP_OPTION; import static org.apache.flink.client.cli.CliFrontendParser.MODIFY_PARALLELISM_OPTION; +import static org.apache.flink.client.program.ClusterClient.MAX_SLOTS_UNKNOWN; /** * Implementation of a simple command line frontend for executing programs. @@ -262,7 +263,7 @@ private void runProgram( int userParallelism = runOptions.getParallelism(); LOG.debug("User parallelism is set to {}", userParallelism); - if (client.getMaxSlots() != -1 && userParallelism == -1) { + if (client.getMaxSlots() != MAX_SLOTS_UNKNOWN && userParallelism == -1) { logAndSysout("Using the parallelism provided by the remote cluster (" + client.getMaxSlots() + "). " + "To use another parallelism, set it at the ./bin/flink client."); diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index 1a783fc2213849..b0c50e59d77052 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -133,6 +133,11 @@ public abstract class ClusterClient { /** Switch for blocking/detached job submission of the client. */ private boolean detachedJobSubmission = false; + /** + * Value returned by {@link #getMaxSlots()} if the number of maximum slots is unknown. + */ + public static final int MAX_SLOTS_UNKNOWN = -1; + // ------------------------------------------------------------------------ // Construction // ------------------------------------------------------------------------ @@ -1000,7 +1005,7 @@ public Configuration getFlinkConfiguration() { /** * The client may define an upper limit on the number of slots to use. - * @return -1 if unknown + * @return -1 ({@link #MAX_SLOTS_UNKNOWN}) if unknown */ public abstract int getMaxSlots(); diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index f0a7631023ea46..86e92795750501 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -181,7 +181,7 @@ public List getNewMessages() { @Override public int getMaxSlots() { - return 0; + return MAX_SLOTS_UNKNOWN; } @Override diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java index 1c9c690710ee1f..e502add468c76f 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/StandaloneClusterClient.java @@ -98,7 +98,7 @@ public StandaloneClusterId getClusterId() { @Override public int getMaxSlots() { - return -1; + return MAX_SLOTS_UNKNOWN; } @Override diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 8cf0d2c7eb2009..5558461dbf6097 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -567,7 +567,7 @@ public List getNewMessages() { @Override public int getMaxSlots() { - return 0; + return MAX_SLOTS_UNKNOWN; } //------------------------------------------------------------------------- diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java index e0010c769ae0e7..29ece26719ea74 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterClient.java @@ -139,7 +139,7 @@ public org.apache.flink.configuration.Configuration getFlinkConfiguration() { public int getMaxSlots() { // TODO: this should be retrieved from the running Flink cluster int maxSlots = numberTaskManagers * slotsPerTaskManager; - return maxSlots > 0 ? maxSlots : -1; + return maxSlots > 0 ? maxSlots : MAX_SLOTS_UNKNOWN; } @Override From 463b922abebc77420d0ec990ba18ad83ea33c0e4 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Thu, 8 Mar 2018 11:07:08 +0100 Subject: [PATCH 0192/2294] [FLINK-8906][flip6][tests] also test Flip6DefaultCLI in org.apache.flink.client.cli tests This closes #5671. --- .../client/cli/CliFrontendCancelTest.java | 19 +++--- .../flink/client/cli/CliFrontendInfoTest.java | 19 +++--- .../flink/client/cli/CliFrontendListTest.java | 7 +-- .../client/cli/CliFrontendModifyTest.java | 10 +-- .../flink/client/cli/CliFrontendRunTest.java | 50 +++++++++------ .../client/cli/CliFrontendSavepointTest.java | 15 +++-- .../flink/client/cli/CliFrontendStopTest.java | 11 ++-- .../flink/client/cli/CliFrontendTestBase.java | 61 +++++++++++++++++++ .../flink/client/cli/DefaultCLITest.java | 24 +++++--- 9 files changed, 144 insertions(+), 72 deletions(-) create mode 100644 flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestBase.java diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java index 837c56408699c3..638150a2e72a86 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendCancelTest.java @@ -22,7 +22,6 @@ import org.apache.flink.client.cli.util.MockedCliFrontend; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.TestLogger; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -40,7 +39,7 @@ /** * Tests for the CANCEL command. */ -public class CliFrontendCancelTest extends TestLogger { +public class CliFrontendCancelTest extends CliFrontendTestBase { @BeforeClass public static void init() { @@ -69,20 +68,20 @@ public void testCancel() throws Exception { @Test(expected = CliArgsException.class) public void testMissingJobId() throws Exception { String[] parameters = {}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.cancel(parameters); } @Test(expected = CliArgsException.class) public void testUnrecognizedOption() throws Exception { String[] parameters = {"-v", "-l"}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.cancel(parameters); } @@ -122,10 +121,10 @@ public void testCancelWithSavepoint() throws Exception { public void testCancelWithSavepointWithoutJobId() throws Exception { // Cancel with savepoint (with target directory), but no job ID String[] parameters = { "-s", "targetDirectory" }; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.cancel(parameters); } @@ -133,10 +132,10 @@ public void testCancelWithSavepointWithoutJobId() throws Exception { public void testCancelWithSavepointWithoutParameters() throws Exception { // Cancel with savepoint (no target directory) and no job ID String[] parameters = { "-s" }; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.cancel(parameters); } diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendInfoTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendInfoTest.java index c284c6141b7a3a..47c799ee98120b 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendInfoTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendInfoTest.java @@ -19,7 +19,6 @@ package org.apache.flink.client.cli; import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.TestLogger; import org.junit.Test; @@ -33,7 +32,7 @@ /** * Tests for the "info" command. */ -public class CliFrontendInfoTest extends TestLogger { +public class CliFrontendInfoTest extends CliFrontendTestBase { private static PrintStream stdOut; private static PrintStream capture; @@ -42,20 +41,20 @@ public class CliFrontendInfoTest extends TestLogger { @Test(expected = CliArgsException.class) public void testMissingOption() throws Exception { String[] parameters = {}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.cancel(parameters); } @Test(expected = CliArgsException.class) public void testUnrecognizedOption() throws Exception { String[] parameters = {"-v", "-l"}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.cancel(parameters); } @@ -65,10 +64,10 @@ public void testShowExecutionPlan() throws Exception { try { String[] parameters = new String[]{CliFrontendTestUtils.getTestJarPath(), "-f", "true"}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.info(parameters); assertTrue(buffer.toString().contains("\"parallelism\": \"1\"")); } @@ -82,10 +81,10 @@ public void testShowExecutionPlanWithParallelism() { replaceStdOut(); try { String[] parameters = {"-p", "17", CliFrontendTestUtils.getTestJarPath()}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.info(parameters); assertTrue(buffer.toString().contains("\"parallelism\": \"17\"")); } diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java index 42399cb65f09b3..a8a7f006bda3d2 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendListTest.java @@ -21,7 +21,6 @@ import org.apache.flink.client.cli.util.MockedCliFrontend; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.TestLogger; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -38,7 +37,7 @@ /** * Tests for the LIST command. */ -public class CliFrontendListTest extends TestLogger { +public class CliFrontendListTest extends CliFrontendTestBase { @BeforeClass public static void init() { @@ -66,10 +65,10 @@ public void testList() throws Exception { @Test(expected = CliArgsException.class) public void testUnrecognizedOption() throws Exception { String[] parameters = {"-v", "-k"}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.list(parameters); } diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendModifyTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendModifyTest.java index 00d52419537f36..a2d6c48f01d99c 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendModifyTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendModifyTest.java @@ -25,7 +25,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.util.TestLogger; import org.hamcrest.Matchers; import org.junit.Test; @@ -38,7 +37,7 @@ /** * Tests for the modify command. */ -public class CliFrontendModifyTest extends TestLogger { +public class CliFrontendModifyTest extends CliFrontendTestBase { @Test public void testModifyJob() throws Exception { @@ -106,7 +105,7 @@ public void testUnparsableJobId() throws Exception { private Tuple2 callModify(String[] args) throws Exception { final CompletableFuture> rescaleJobFuture = new CompletableFuture<>(); - final TestingClusterClient clusterClient = new TestingClusterClient(rescaleJobFuture); + final TestingClusterClient clusterClient = new TestingClusterClient(rescaleJobFuture, getConfiguration()); final MockedCliFrontend cliFrontend = new MockedCliFrontend(clusterClient); cliFrontend.modify(args); @@ -120,8 +119,9 @@ private static final class TestingClusterClient extends StandaloneClusterClient private final CompletableFuture> rescaleJobFuture; - public TestingClusterClient(CompletableFuture> rescaleJobFuture) throws Exception { - super(new Configuration(), new TestingHighAvailabilityServices(), false); + TestingClusterClient( + CompletableFuture> rescaleJobFuture, Configuration configuration) { + super(configuration, new TestingHighAvailabilityServices(), false); this.rescaleJobFuture = rescaleJobFuture; } diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java index c7789a89ac5c2d..efa6a3964a786b 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendRunTest.java @@ -21,9 +21,7 @@ import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; -import org.apache.flink.util.TestLogger; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -39,7 +37,7 @@ /** * Tests for the RUN command. */ -public class CliFrontendRunTest extends TestLogger { +public class CliFrontendRunTest extends CliFrontendTestBase { @BeforeClass public static void init() { @@ -53,33 +51,29 @@ public static void shutdown() { @Test public void testRun() throws Exception { - final Configuration configuration = GlobalConfiguration.loadConfiguration(CliFrontendTestUtils.getConfigDir()); + final Configuration configuration = getConfiguration(); // test without parallelism { String[] parameters = {"-v", getTestJarPath()}; - RunTestingCliFrontend testFrontend = new RunTestingCliFrontend(configuration, 1, true, false); - testFrontend.run(parameters); + verifyCliFrontend(getCli(configuration), parameters, 1, true, false); } // test configure parallelism { String[] parameters = {"-v", "-p", "42", getTestJarPath()}; - RunTestingCliFrontend testFrontend = new RunTestingCliFrontend(configuration, 42, true, false); - testFrontend.run(parameters); + verifyCliFrontend(getCli(configuration), parameters, 42, true, false); } // test configure sysout logging { String[] parameters = {"-p", "2", "-q", getTestJarPath()}; - RunTestingCliFrontend testFrontend = new RunTestingCliFrontend(configuration, 2, false, false); - testFrontend.run(parameters); + verifyCliFrontend(getCli(configuration), parameters, 2, false, false); } // test detached mode { String[] parameters = {"-p", "2", "-d", getTestJarPath()}; - RunTestingCliFrontend testFrontend = new RunTestingCliFrontend(configuration, 2, true, true); - testFrontend.run(parameters); + verifyCliFrontend(getCli(configuration), parameters, 2, true, true); } // test configure savepoint path (no ignore flag) @@ -119,10 +113,10 @@ public void testRun() throws Exception { public void testUnrecognizedOption() throws Exception { // test unrecognized option String[] parameters = {"-v", "-l", "-a", "some", "program", "arguments"}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.run(parameters); } @@ -130,10 +124,10 @@ public void testUnrecognizedOption() throws Exception { public void testInvalidParallelismOption() throws Exception { // test configure parallelism with non integer value String[] parameters = {"-v", "-p", "text", getTestJarPath()}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.run(parameters); } @@ -144,22 +138,38 @@ public void testParallelismWithOverflow() throws Exception { Configuration configuration = new Configuration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.run(parameters); } // -------------------------------------------------------------------------------------------- + private static void verifyCliFrontend( + AbstractCustomCommandLine cli, + String[] parameters, + int expectedParallelism, + boolean logging, + boolean isDetached) throws Exception { + RunTestingCliFrontend testFrontend = + new RunTestingCliFrontend(cli, expectedParallelism, logging, + isDetached); + testFrontend.run(parameters); // verifies the expected values (see below) + } + private static final class RunTestingCliFrontend extends CliFrontend { private final int expectedParallelism; private final boolean sysoutLogging; private final boolean isDetached; - public RunTestingCliFrontend(Configuration configuration, int expectedParallelism, boolean logging, boolean isDetached) throws Exception { + private RunTestingCliFrontend( + AbstractCustomCommandLine cli, + int expectedParallelism, + boolean logging, + boolean isDetached) throws Exception { super( - configuration, - Collections.singletonList(new DefaultCLI(configuration))); + cli.getConfiguration(), + Collections.singletonList(cli)); this.expectedParallelism = expectedParallelism; this.sysoutLogging = logging; this.isDetached = isDetached; diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java index f4c66eb08c2047..3195a6baf6dc12 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java @@ -30,7 +30,6 @@ import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.Preconditions; -import org.apache.flink.util.TestLogger; import org.hamcrest.Matchers; import org.junit.Rule; @@ -61,7 +60,7 @@ /** * Tests for the SAVEPOINT command. */ -public class CliFrontendSavepointTest extends TestLogger { +public class CliFrontendSavepointTest extends CliFrontendTestBase { private static PrintStream stdOut; private static PrintStream stdErr; @@ -137,7 +136,7 @@ public void testTriggerSavepointFailureIllegalJobID() throws Exception { try { CliFrontend frontend = new MockedCliFrontend(new StandaloneClusterClient( - new Configuration(), + getConfiguration(), new TestingHighAvailabilityServices(), false)); @@ -197,7 +196,7 @@ public void testDisposeSavepointSuccess() throws Exception { String savepointPath = "expectedSavepointPath"; ClusterClient clusterClient = new DisposeSavepointClusterClient( - (String path, Time timeout) -> CompletableFuture.completedFuture(Acknowledge.get())); + (String path, Time timeout) -> CompletableFuture.completedFuture(Acknowledge.get()), getConfiguration()); try { @@ -229,7 +228,7 @@ public void testDisposeWithJar() throws Exception { (String savepointPath, Time timeout) -> { disposeSavepointFuture.complete(savepointPath); return CompletableFuture.completedFuture(Acknowledge.get()); - }); + }, getConfiguration()); try { CliFrontend frontend = new MockedCliFrontend(clusterClient); @@ -261,7 +260,7 @@ public void testDisposeSavepointFailure() throws Exception { Exception testException = new Exception("expectedTestException"); - DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient((String path, Time timeout) -> FutureUtils.completedExceptionally(testException)); + DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient((String path, Time timeout) -> FutureUtils.completedExceptionally(testException), getConfiguration()); try { CliFrontend frontend = new MockedCliFrontend(clusterClient); @@ -288,8 +287,8 @@ private static final class DisposeSavepointClusterClient extends StandaloneClust private final BiFunction> disposeSavepointFunction; - DisposeSavepointClusterClient(BiFunction> disposeSavepointFunction) throws Exception { - super(new Configuration(), new TestingHighAvailabilityServices(), false); + DisposeSavepointClusterClient(BiFunction> disposeSavepointFunction, Configuration configuration) { + super(configuration, new TestingHighAvailabilityServices(), false); this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction); } diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java index ec4ccdca7d1df8..23bed005317c67 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendStopTest.java @@ -24,7 +24,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; -import org.apache.flink.util.TestLogger; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -45,7 +44,7 @@ /** * Tests for the STOP command. */ -public class CliFrontendStopTest extends TestLogger { +public class CliFrontendStopTest extends CliFrontendTestBase { @BeforeClass public static void setup() { @@ -76,10 +75,10 @@ public void testStop() throws Exception { public void testUnrecognizedOption() throws Exception { // test unrecognized option String[] parameters = { "-v", "-l" }; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.stop(parameters); } @@ -87,10 +86,10 @@ public void testUnrecognizedOption() throws Exception { public void testMissingJobId() throws Exception { // test missing job id String[] parameters = {}; - Configuration configuration = new Configuration(); + Configuration configuration = getConfiguration(); CliFrontend testFrontend = new CliFrontend( configuration, - Collections.singletonList(new DefaultCLI(configuration))); + Collections.singletonList(getCli(configuration))); testFrontend.stop(parameters); } diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestBase.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestBase.java new file mode 100644 index 00000000000000..e2463f214c430f --- /dev/null +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestBase.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.client.cli; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.util.TestLogger; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.List; + +/** + * Base test class for {@link CliFrontend} tests that wraps the Flip-6 vs. non-Flip-6 modes. + */ +@RunWith(Parameterized.class) +public abstract class CliFrontendTestBase extends TestLogger { + @Parameterized.Parameter + public String mode; + + @Parameterized.Parameters(name = "Mode = {0}") + public static List parameters() { + return Arrays.asList(CoreOptions.OLD_MODE, CoreOptions.FLIP6_MODE); + } + + protected Configuration getConfiguration() { + final Configuration configuration = GlobalConfiguration + .loadConfiguration(CliFrontendTestUtils.getConfigDir()); + configuration.setString(CoreOptions.MODE, mode); + return configuration; + } + + static AbstractCustomCommandLine getCli(Configuration configuration) { + switch (configuration.getString(CoreOptions.MODE)) { + case CoreOptions.OLD_MODE: + return new DefaultCLI(configuration); + case CoreOptions.FLIP6_MODE: + return new Flip6DefaultCLI(configuration); + } + throw new IllegalStateException(); + } +} diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java index d89e988aebb6df..8402b7e112099c 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/DefaultCLITest.java @@ -18,12 +18,12 @@ package org.apache.flink.client.cli; -import org.apache.flink.client.deployment.StandaloneClusterDescriptor; +import org.apache.flink.client.deployment.ClusterDescriptor; +import org.apache.flink.client.deployment.StandaloneClusterId; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.runtime.util.LeaderConnectionInfo; -import org.apache.flink.util.TestLogger; import org.apache.commons.cli.CommandLine; import org.hamcrest.Matchers; @@ -36,7 +36,7 @@ /** * Tests for the {@link DefaultCLI}. */ -public class DefaultCLITest extends TestLogger { +public class DefaultCLITest extends CliFrontendTestBase { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -47,7 +47,7 @@ public class DefaultCLITest extends TestLogger { */ @Test public void testConfigurationPassing() throws Exception { - final Configuration configuration = new Configuration(); + final Configuration configuration = getConfiguration(); final String localhost = "localhost"; final int port = 1234; @@ -55,13 +55,16 @@ public void testConfigurationPassing() throws Exception { configuration.setString(JobManagerOptions.ADDRESS, localhost); configuration.setInteger(JobManagerOptions.PORT, port); - final DefaultCLI defaultCLI = new DefaultCLI(configuration); + @SuppressWarnings("unchecked") + final AbstractCustomCommandLine defaultCLI = + (AbstractCustomCommandLine) getCli(configuration); final String[] args = {}; CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false); - final StandaloneClusterDescriptor clusterDescriptor = defaultCLI.createClusterDescriptor(commandLine); + final ClusterDescriptor clusterDescriptor = + defaultCLI.createClusterDescriptor(commandLine); final ClusterClient clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine)); @@ -78,12 +81,14 @@ public void testConfigurationPassing() throws Exception { public void testManualConfigurationOverride() throws Exception { final String localhost = "localhost"; final int port = 1234; - final Configuration configuration = new Configuration(); + final Configuration configuration = getConfiguration(); configuration.setString(JobManagerOptions.ADDRESS, localhost); configuration.setInteger(JobManagerOptions.PORT, port); - final DefaultCLI defaultCLI = new DefaultCLI(configuration); + @SuppressWarnings("unchecked") + final AbstractCustomCommandLine defaultCLI = + (AbstractCustomCommandLine) getCli(configuration); final String manualHostname = "123.123.123.123"; final int manualPort = 4321; @@ -91,7 +96,8 @@ public void testManualConfigurationOverride() throws Exception { CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false); - final StandaloneClusterDescriptor clusterDescriptor = defaultCLI.createClusterDescriptor(commandLine); + final ClusterDescriptor clusterDescriptor = + defaultCLI.createClusterDescriptor(commandLine); final ClusterClient clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine)); From b6bbd123c951d1f6e1cda108514d8e749c5ed033 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Fri, 16 Mar 2018 15:56:07 +0100 Subject: [PATCH 0193/2294] [FLINK-8948][runtime] Fix IllegalStateException when closing StreamTask This closes #5710. --- .../runtime/io/network/buffer/BufferBuilder.java | 5 +++-- .../buffer/BufferBuilderAndConsumerTest.java | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java index 63b60d2b8a080d..305f1842911c26 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java @@ -99,10 +99,11 @@ public void commit() { * Mark this {@link BufferBuilder} and associated {@link BufferConsumer} as finished - no new data writes will be * allowed. * + *

    This method should be idempotent to handle failures and task interruptions. Check FLINK-8948 for more details. + * * @return number of written bytes. */ public int finish() { - checkState(!isFinished()); positionMarker.markFinished(); commit(); return getWrittenBytes(); @@ -125,7 +126,7 @@ public int getMaxCapacity() { return memorySegment.size(); } - public int getWrittenBytes() { + private int getWrittenBytes() { return positionMarker.getCached(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java index edf2bfe319eadd..b5d9da0f1aa3e2 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java @@ -202,22 +202,30 @@ private static void testIsFinished(int writes) { for (int i = 0; i < writes; i++) { assertEquals(Integer.BYTES, bufferBuilder.appendAndCommit(toByteBuffer(42))); } + int expectedWrittenBytes = writes * Integer.BYTES; assertFalse(bufferBuilder.isFinished()); assertFalse(bufferConsumer.isFinished()); + assertEquals(0, bufferConsumer.getWrittenBytes()); bufferConsumer.build(); - assertFalse(bufferBuilder.isFinished()); assertFalse(bufferConsumer.isFinished()); + assertEquals(expectedWrittenBytes, bufferConsumer.getWrittenBytes()); - bufferBuilder.finish(); - + int actualWrittenBytes = bufferBuilder.finish(); + assertEquals(expectedWrittenBytes, actualWrittenBytes); assertTrue(bufferBuilder.isFinished()); assertFalse(bufferConsumer.isFinished()); + assertEquals(expectedWrittenBytes, bufferConsumer.getWrittenBytes()); - bufferConsumer.build(); + actualWrittenBytes = bufferBuilder.finish(); + assertEquals(expectedWrittenBytes, actualWrittenBytes); + assertTrue(bufferBuilder.isFinished()); + assertFalse(bufferConsumer.isFinished()); + assertEquals(expectedWrittenBytes, bufferConsumer.getWrittenBytes()); + assertEquals(0, bufferConsumer.build().getSize()); assertTrue(bufferConsumer.isFinished()); } From cfd8df405cb91278f4c8d81cc294daaaa8b07840 Mon Sep 17 00:00:00 2001 From: sihuazhou Date: Wed, 14 Mar 2018 16:22:44 +0800 Subject: [PATCH 0194/2294] [hotfix][javadocs] Minor javadoc fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This close sä5696 --- .../apache/flink/runtime/jobmaster/slotpool/SlotProvider.java | 3 ++- .../org/apache/flink/runtime/state/heap/AbstractHeapState.java | 3 +-- .../apache/flink/runtime/state/heap/HeapAggregatingState.java | 3 +-- .../flink/contrib/streaming/state/RocksDBAggregatingState.java | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java index 1653138949b3f2..91c0372dcb387a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotProvider.java @@ -46,9 +46,10 @@ public interface SlotProvider { /** * Allocating slot with specific requirement. * + * @param slotRequestId identifying the slot request * @param task The task to allocate the slot for * @param allowQueued Whether allow the task be queued if we do not have enough resource - * @param preferredLocations preferred locations for the slot allocation + * @param slotProfile profile of the requested slot * @param timeout after which the allocation fails with a timeout exception * @return The future of the allocation */ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java index 66360e4e9fffb6..7f629ae9e031d3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/AbstractHeapState.java @@ -19,7 +19,6 @@ package org.apache.flink.runtime.state.heap; import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.State; import org.apache.flink.api.common.state.StateDescriptor; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -29,7 +28,7 @@ import org.apache.flink.util.Preconditions; /** - * Base class for partitioned {@link ListState} implementations that are backed by a regular + * Base class for partitioned {@link State} implementations that are backed by a regular * heap hash map. The concrete implementations define how the state is checkpointed. * * @param The type of the key. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapAggregatingState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapAggregatingState.java index 3fa8cd4781dea0..6dd5cec878a3a0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapAggregatingState.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/heap/HeapAggregatingState.java @@ -21,7 +21,6 @@ import org.apache.flink.api.common.functions.AggregateFunction; import org.apache.flink.api.common.state.AggregatingState; import org.apache.flink.api.common.state.AggregatingStateDescriptor; -import org.apache.flink.api.common.state.ReducingState; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.runtime.state.StateTransformationFunction; import org.apache.flink.runtime.state.internal.InternalAggregatingState; @@ -30,7 +29,7 @@ import java.io.IOException; /** - * Heap-backed partitioned {@link ReducingState} that is + * Heap-backed partitioned {@link AggregatingState} that is * snapshotted into files. * * @param The type of the key. diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java index f2d1d865bc0566..4dfc77228f290c 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBAggregatingState.java @@ -53,7 +53,7 @@ public class RocksDBAggregatingState private final AggregateFunction aggFunction; /** - * Creates a new {@code RocksDBFoldingState}. + * Creates a new {@code RocksDBAggregatingState}. * * @param namespaceSerializer * The serializer for the namespace. From 5fa84c28fc1bfc62fa2e1165e3407fc81b3d09a9 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 7 Mar 2018 11:05:12 +0100 Subject: [PATCH 0195/2294] [FLINK-8935][tests] Implement MiniClusterClient#listJobs --- .../flink/client/program/MiniClusterClient.java | 2 +- .../flink/runtime/minicluster/MiniCluster.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 86e92795750501..961604f3a865df 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -125,7 +125,7 @@ public CompletableFuture disposeSavepoint(String savepointPath, Tim @Override public CompletableFuture> listJobs() throws Exception { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + return guardWithSingleRetry(miniCluster::listJobs, scheduledExecutor); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index dfe30afda5eb67..21b89ec1fd1b0f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -28,6 +28,7 @@ import org.apache.flink.runtime.blob.BlobCacheService; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.runtime.clusterframework.FlinkResourceManager; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.concurrent.FutureUtils; @@ -87,6 +88,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import static org.apache.flink.util.Preconditions.checkNotNull; import static org.apache.flink.util.Preconditions.checkState; @@ -473,6 +475,20 @@ public CompletableFuture closeAsync() { // Accessing jobs // ------------------------------------------------------------------------ + public CompletableFuture> listJobs() { + try { + return getDispatcherGateway().requestMultipleJobDetails(rpcTimeout) + .thenApply(jobs -> jobs.getJobs().stream() + .map(details -> new JobStatusMessage(details.getJobId(), details.getJobName(), details.getStatus(), details.getStartTime())) + .collect(Collectors.toList())); + } catch (LeaderRetrievalException | InterruptedException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + "Could not retrieve job list.", + e)); + } + } + public CompletableFuture getJobStatus(JobID jobId) { try { return getDispatcherGateway().requestJobStatus(jobId, rpcTimeout); From 3e6aa676e36824ac76258ff20723159b40c3a338 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 7 Mar 2018 11:05:42 +0100 Subject: [PATCH 0196/2294] [FLINK-8935][tests] Implement MiniClusterClient#getAccumulators --- .../flink/client/program/MiniClusterClient.java | 13 +++++++++++-- .../flink/runtime/minicluster/MiniCluster.java | 12 ++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 961604f3a865df..9c8742377959e4 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -28,6 +28,7 @@ import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.concurrent.ScheduledExecutor; import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; +import org.apache.flink.runtime.executiongraph.AccessExecutionGraph; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; @@ -40,6 +41,7 @@ import org.apache.flink.runtime.util.LeaderRetrievalUtils; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; +import org.apache.flink.util.SerializedValue; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -47,6 +49,7 @@ import java.net.URL; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -130,12 +133,18 @@ public CompletableFuture> listJobs() throws Excepti @Override public Map getAccumulators(JobID jobID) throws Exception { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + return getAccumulators(jobID, ClassLoader.getSystemClassLoader()); } @Override public Map getAccumulators(JobID jobID, ClassLoader loader) throws Exception { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + AccessExecutionGraph executionGraph = guardWithSingleRetry(() -> miniCluster.getExecutionGraph(jobID), scheduledExecutor).get(); + Map> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); + Map result = new HashMap<>(accumulatorsSerialized.size()); + for (Map.Entry> acc : accumulatorsSerialized.entrySet()) { + result.put(acc.getKey(), acc.getValue().deserializeValue(loader)); + } + return result; } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 21b89ec1fd1b0f..e958005e47b82e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -39,6 +39,7 @@ import org.apache.flink.runtime.dispatcher.MemoryArchivedExecutionGraphStore; import org.apache.flink.runtime.dispatcher.StandaloneDispatcher; import org.apache.flink.runtime.entrypoint.ClusterInformation; +import org.apache.flink.runtime.executiongraph.AccessExecutionGraph; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; @@ -522,6 +523,17 @@ public CompletableFuture triggerSavepoint(JobID jobId, String targetDire } } + public CompletableFuture getExecutionGraph(JobID jobId) { + try { + return getDispatcherGateway().requestJob(jobId, rpcTimeout); + } catch (LeaderRetrievalException | InterruptedException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Could not retrieve job job %s.", jobId), + e)); + } + } + // ------------------------------------------------------------------------ // running jobs // ------------------------------------------------------------------------ From ca514e16dc36ab01e4051e76948e7cffcf17c56d Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 6 Mar 2018 13:26:59 +0100 Subject: [PATCH 0197/2294] [FLINK-8935][tests] Implement MiniClusterClient#triggerSavepoint --- .../java/org/apache/flink/client/program/MiniClusterClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 9c8742377959e4..4354267bab993c 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -118,7 +118,7 @@ public void stop(JobID jobId) throws Exception { @Override public CompletableFuture triggerSavepoint(JobID jobId, @Nullable String savepointDirectory) throws FlinkException { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + return guardWithSingleRetry(() -> miniCluster.triggerSavepoint(jobId, savepointDirectory, false), scheduledExecutor); } @Override From 2dab4374bc5280a2b4536f7ad1e153d6361a8885 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 7 Mar 2018 13:02:27 +0100 Subject: [PATCH 0198/2294] [FLINK-8935][tests] Implement MiniClusterClient#stop This closes #5690. --- .../flink/client/program/MiniClusterClient.java | 2 +- .../apache/flink/runtime/minicluster/MiniCluster.java | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 4354267bab993c..276df62f7233d8 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -113,7 +113,7 @@ public String cancelWithSavepoint(JobID jobId, @Nullable String savepointDirecto @Override public void stop(JobID jobId) throws Exception { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + guardWithSingleRetry(() -> miniCluster.stopJob(jobId), scheduledExecutor).get(); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index e958005e47b82e..bc75a547b55f56 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -512,6 +512,17 @@ public CompletableFuture cancelJob(JobID jobId) { } } + public CompletableFuture stopJob(JobID jobId) { + try { + return getDispatcherGateway().stopJob(jobId, rpcTimeout); + } catch (LeaderRetrievalException | InterruptedException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Could not stop job %s.", jobId), + e)); + } + } + public CompletableFuture triggerSavepoint(JobID jobId, String targetDirectory, boolean cancelJob) { try { return getDispatcherGateway().triggerSavepoint(jobId, targetDirectory, cancelJob, rpcTimeout); From bcb0f324f50adc74fb6122621794d6d7e37bc933 Mon Sep 17 00:00:00 2001 From: vinoyang Date: Sun, 4 Mar 2018 17:01:35 +0800 Subject: [PATCH 0199/2294] [FLINK-8830][YARN] YarnResourceManager throws NullPointerException --- .../java/org/apache/flink/yarn/Utils.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java index ff2478ede1415d..79a670315df29f 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java @@ -496,18 +496,20 @@ static ContainerLaunchContext createTaskExecutorContext( // NOTE: must read the tokens from the local file, not from the UGI context, because if UGI is login // using Kerberos keytabs, there is no HDFS delegation token in the UGI context. String fileLocation = System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION); - Method readTokenStorageFileMethod = Credentials.class.getMethod( - "readTokenStorageFile", File.class, org.apache.hadoop.conf.Configuration.class); - - Credentials cred = - (Credentials) readTokenStorageFileMethod.invoke( - null, - new File(fileLocation), - HadoopUtils.getHadoopConfiguration(flinkConfig)); - - cred.writeTokenStorageToStream(dob); - ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - ctx.setTokens(securityTokens); + if (fileLocation != null) { + Method readTokenStorageFileMethod = Credentials.class.getMethod( + "readTokenStorageFile", File.class, org.apache.hadoop.conf.Configuration.class); + + Credentials cred = + (Credentials) readTokenStorageFileMethod.invoke( + null, + new File(fileLocation), + HadoopUtils.getHadoopConfiguration(flinkConfig)); + + cred.writeTokenStorageToStream(dob); + ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + ctx.setTokens(securityTokens); + } } catch (Throwable t) { log.error("Getting current user info failed when trying to launch the container", t); From 4536e9cbe1925c0a6dd24f74c2b73a675afb2625 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Mon, 19 Mar 2018 18:06:12 +0100 Subject: [PATCH 0200/2294] [FLINK-8830] [yarn] Log reading of Hadoop's token file This closes #5629. --- .../java/org/apache/flink/yarn/Utils.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java index 79a670315df29f..b9f7fac2978623 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java @@ -489,14 +489,15 @@ static ContainerLaunchContext createTaskExecutorContext( ctx.setEnvironment(containerEnv); - try (DataOutputBuffer dob = new DataOutputBuffer()) { - log.debug("Adding security tokens to Task Executor Container launch Context...."); + // For TaskManager YARN container context, read the tokens from the jobmanager yarn container local file. + // NOTE: must read the tokens from the local file, not from the UGI context, because if UGI is login + // using Kerberos keytabs, there is no HDFS delegation token in the UGI context. + final String fileLocation = System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION); + + if (fileLocation != null) { + log.debug("Adding security tokens to TaskExecutor's container launch context."); - // For TaskManager YARN container context, read the tokens from the jobmanager yarn container local flie. - // NOTE: must read the tokens from the local file, not from the UGI context, because if UGI is login - // using Kerberos keytabs, there is no HDFS delegation token in the UGI context. - String fileLocation = System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION); - if (fileLocation != null) { + try (DataOutputBuffer dob = new DataOutputBuffer()) { Method readTokenStorageFileMethod = Credentials.class.getMethod( "readTokenStorageFile", File.class, org.apache.hadoop.conf.Configuration.class); @@ -509,10 +510,11 @@ static ContainerLaunchContext createTaskExecutorContext( cred.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); ctx.setTokens(securityTokens); + } catch (Throwable t) { + log.error("Failed to add Hadoop's security tokens.", t); } - } - catch (Throwable t) { - log.error("Getting current user info failed when trying to launch the container", t); + } else { + log.info("Could not set security tokens because Hadoop's token file location is unknown."); } return ctx; From f9fbbc3a137276cab4b8abf272199f1cd4633d29 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 14 Mar 2018 14:21:27 +0100 Subject: [PATCH 0201/2294] [FLINK-8942][runtime] Pass heartbeat target ResourceID received payload field now volatile Add HeartbeatMonitor#getHeartbeatTargetId This closes #5699. --- .../runtime/heartbeat/HeartbeatListener.java | 3 +- .../heartbeat/HeartbeatManagerImpl.java | 8 +- .../heartbeat/HeartbeatManagerSenderImpl.java | 2 +- .../flink/runtime/jobmaster/JobMaster.java | 4 +- .../resourcemanager/ResourceManager.java | 4 +- .../runtime/taskexecutor/TaskExecutor.java | 4 +- .../heartbeat/HeartbeatManagerTest.java | 167 +++++++++++++++++- 7 files changed, 177 insertions(+), 15 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java index 734eb4c853b414..01a4754dfb116b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatListener.java @@ -57,7 +57,8 @@ public interface HeartbeatListener { * Retrieves the payload value for the next heartbeat message. Since the operation can happen * asynchronously, the result is returned wrapped in a future. * + * @param resourceID Resource ID identifying the receiver of the payload * @return Future containing the next payload for heartbeats */ - CompletableFuture retrievePayload(); + CompletableFuture retrievePayload(ResourceID resourceID); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java index 09c4b461b3f389..42268fc46e76dc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerImpl.java @@ -106,7 +106,7 @@ HeartbeatListener getHeartbeatListener() { return heartbeatListener; } - Collection> getHeartbeatTargets() { + Collection> getHeartbeatTargets() { return heartbeatTargets.values(); } @@ -202,7 +202,7 @@ public void requestHeartbeat(final ResourceID requestOrigin, I heartbeatPayload) heartbeatListener.reportPayload(requestOrigin, heartbeatPayload); } - CompletableFuture futurePayload = heartbeatListener.retrievePayload(); + CompletableFuture futurePayload = heartbeatListener.retrievePayload(requestOrigin); if (futurePayload != null) { CompletableFuture sendHeartbeatFuture = futurePayload.thenAcceptAsync( @@ -289,6 +289,10 @@ HeartbeatTarget getHeartbeatTarget() { return heartbeatTarget; } + ResourceID getHeartbeatTargetId() { + return resourceID; + } + public long getLastHeartbeat() { return lastHeartbeat; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java index eb8234369b9d4f..e3b939c068b721 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerSenderImpl.java @@ -63,7 +63,7 @@ public void run() { if (!stopped) { log.debug("Trigger heartbeat request."); for (HeartbeatMonitor heartbeatMonitor : getHeartbeatTargets()) { - CompletableFuture futurePayload = getHeartbeatListener().retrievePayload(); + CompletableFuture futurePayload = getHeartbeatListener().retrievePayload(heartbeatMonitor.getHeartbeatTargetId()); final HeartbeatTarget heartbeatTarget = heartbeatMonitor.getHeartbeatTarget(); if (futurePayload != null) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index ced8c7c4dd89e8..f0b29bf49a347d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -1527,7 +1527,7 @@ public void reportPayload(ResourceID resourceID, Void payload) { } @Override - public CompletableFuture retrievePayload() { + public CompletableFuture retrievePayload(ResourceID resourceID) { return CompletableFuture.completedFuture(null); } } @@ -1551,7 +1551,7 @@ public void reportPayload(ResourceID resourceID, Void payload) { } @Override - public CompletableFuture retrievePayload() { + public CompletableFuture retrievePayload(ResourceID resourceID) { return CompletableFuture.completedFuture(null); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java index 77e43621dbb201..0ae4ab6af3a08e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java @@ -1076,7 +1076,7 @@ public void run() { } @Override - public CompletableFuture retrievePayload() { + public CompletableFuture retrievePayload(ResourceID resourceID) { return CompletableFuture.completedFuture(null); } } @@ -1109,7 +1109,7 @@ public void reportPayload(ResourceID resourceID, Void payload) { } @Override - public CompletableFuture retrievePayload() { + public CompletableFuture retrievePayload(ResourceID resourceID) { return CompletableFuture.completedFuture(null); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index fc69984c8a0c87..7409175d44aa12 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -1515,7 +1515,7 @@ public void reportPayload(ResourceID resourceID, Void payload) { } @Override - public CompletableFuture retrievePayload() { + public CompletableFuture retrievePayload(ResourceID resourceID) { return CompletableFuture.completedFuture(null); } } @@ -1544,7 +1544,7 @@ public void reportPayload(ResourceID resourceID, Void payload) { } @Override - public CompletableFuture retrievePayload() { + public CompletableFuture retrievePayload(ResourceID resourceID) { return callAsync( () -> taskSlotTable.createSlotReport(getResourceID()), taskManagerConfiguration.getTimeout()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java index 390a1312e54136..77d12d54f0c6e9 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/heartbeat/HeartbeatManagerTest.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.heartbeat; +import org.apache.flink.core.testutils.OneShotLatch; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.concurrent.Executors; import org.apache.flink.runtime.concurrent.ScheduledExecutor; @@ -75,7 +76,7 @@ public void testRegularHeartbeat() { Object expectedObject = new Object(); - when(heartbeatListener.retrievePayload()).thenReturn(CompletableFuture.completedFuture(expectedObject)); + when(heartbeatListener.retrievePayload(any(ResourceID.class))).thenReturn(CompletableFuture.completedFuture(expectedObject)); HeartbeatManagerImpl heartbeatManager = new HeartbeatManagerImpl<>( heartbeatTimeout, @@ -93,7 +94,7 @@ public void testRegularHeartbeat() { heartbeatManager.requestHeartbeat(targetResourceID, expectedObject); verify(heartbeatListener, times(1)).reportPayload(targetResourceID, expectedObject); - verify(heartbeatListener, times(1)).retrievePayload(); + verify(heartbeatListener, times(1)).retrievePayload(any(ResourceID.class)); verify(heartbeatTarget, times(1)).receiveHeartbeat(ownResourceID, expectedObject); heartbeatManager.receiveHeartbeat(targetResourceID, expectedObject); @@ -118,7 +119,7 @@ public void testHeartbeatMonitorUpdate() { Object expectedObject = new Object(); - when(heartbeatListener.retrievePayload()).thenReturn(CompletableFuture.completedFuture(expectedObject)); + when(heartbeatListener.retrievePayload(any(ResourceID.class))).thenReturn(CompletableFuture.completedFuture(expectedObject)); HeartbeatManagerImpl heartbeatManager = new HeartbeatManagerImpl<>( heartbeatTimeout, @@ -207,7 +208,7 @@ public void testHeartbeatCluster() throws Exception { @SuppressWarnings("unchecked") HeartbeatListener heartbeatListener = mock(HeartbeatListener.class); - when(heartbeatListener.retrievePayload()).thenReturn(CompletableFuture.completedFuture(object)); + when(heartbeatListener.retrievePayload(any(ResourceID.class))).thenReturn(CompletableFuture.completedFuture(object)); TestingHeartbeatListener heartbeatListener2 = new TestingHeartbeatListener(object2); @@ -347,6 +348,162 @@ public void testLastHeartbeatFrom() { } } + /** + * Tests that the heartbeat target {@link ResourceID} is properly passed to the {@link HeartbeatListener} by the + * {@link HeartbeatManagerImpl}. + */ + @Test + public void testHeartbeatManagerTargetPayload() { + final long heartbeatTimeout = 100L; + + final ResourceID someTargetId = ResourceID.generate(); + final ResourceID specialTargetId = ResourceID.generate(); + final TargetDependentHeartbeatReceiver someHeartbeatTarget = new TargetDependentHeartbeatReceiver(); + final TargetDependentHeartbeatReceiver specialHeartbeatTarget = new TargetDependentHeartbeatReceiver(); + + final int defaultResponse = 0; + final int specialResponse = 1; + + HeartbeatManager heartbeatManager = new HeartbeatManagerImpl<>( + heartbeatTimeout, + ResourceID.generate(), + new TargetDependentHeartbeatSender(specialTargetId, specialResponse, defaultResponse), + Executors.directExecutor(), + mock(ScheduledExecutor.class), + LOG); + + try { + heartbeatManager.monitorTarget(someTargetId, someHeartbeatTarget); + heartbeatManager.monitorTarget(specialTargetId, specialHeartbeatTarget); + + heartbeatManager.requestHeartbeat(someTargetId, null); + assertEquals(defaultResponse, someHeartbeatTarget.getLastReceivedHeartbeatPayload()); + + heartbeatManager.requestHeartbeat(specialTargetId, null); + assertEquals(specialResponse, specialHeartbeatTarget.getLastReceivedHeartbeatPayload()); + } finally { + heartbeatManager.stop(); + } + } + + /** + * Tests that the heartbeat target {@link ResourceID} is properly passed to the {@link HeartbeatListener} by the + * {@link HeartbeatManagerSenderImpl}. + */ + @Test + public void testHeartbeatManagerSenderTargetPayload() throws Exception { + final long heartbeatTimeout = 100L; + final long heartbeatPeriod = 2000L; + + final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1); + + final ResourceID someTargetId = ResourceID.generate(); + final ResourceID specialTargetId = ResourceID.generate(); + + final OneShotLatch someTargetReceivedLatch = new OneShotLatch(); + final OneShotLatch specialTargetReceivedLatch = new OneShotLatch(); + + final TargetDependentHeartbeatReceiver someHeartbeatTarget = new TargetDependentHeartbeatReceiver(someTargetReceivedLatch); + final TargetDependentHeartbeatReceiver specialHeartbeatTarget = new TargetDependentHeartbeatReceiver(specialTargetReceivedLatch); + + final int defaultResponse = 0; + final int specialResponse = 1; + + HeartbeatManager heartbeatManager = new HeartbeatManagerSenderImpl<>( + heartbeatPeriod, + heartbeatTimeout, + ResourceID.generate(), + new TargetDependentHeartbeatSender(specialTargetId, specialResponse, defaultResponse), + Executors.directExecutor(), + new ScheduledExecutorServiceAdapter(scheduledThreadPoolExecutor), + LOG); + + try { + heartbeatManager.monitorTarget(someTargetId, someHeartbeatTarget); + heartbeatManager.monitorTarget(specialTargetId, specialHeartbeatTarget); + + someTargetReceivedLatch.await(5, TimeUnit.SECONDS); + specialTargetReceivedLatch.await(5, TimeUnit.SECONDS); + + assertEquals(defaultResponse, someHeartbeatTarget.getLastRequestedHeartbeatPayload()); + assertEquals(specialResponse, specialHeartbeatTarget.getLastRequestedHeartbeatPayload()); + } finally { + heartbeatManager.stop(); + scheduledThreadPoolExecutor.shutdown(); + } + } + + /** + * Test {@link HeartbeatTarget} that exposes the last received payload. + */ + private static class TargetDependentHeartbeatReceiver implements HeartbeatTarget { + + private volatile int lastReceivedHeartbeatPayload = -1; + private volatile int lastRequestedHeartbeatPayload = -1; + + private final OneShotLatch latch; + + public TargetDependentHeartbeatReceiver() { + this(new OneShotLatch()); + } + + public TargetDependentHeartbeatReceiver(OneShotLatch latch) { + this.latch = latch; + } + + @Override + public void receiveHeartbeat(ResourceID heartbeatOrigin, Integer heartbeatPayload) { + this.lastReceivedHeartbeatPayload = heartbeatPayload; + latch.trigger(); + } + + @Override + public void requestHeartbeat(ResourceID requestOrigin, Integer heartbeatPayload) { + this.lastRequestedHeartbeatPayload = heartbeatPayload; + latch.trigger(); + } + + public int getLastReceivedHeartbeatPayload() { + return lastReceivedHeartbeatPayload; + } + + public int getLastRequestedHeartbeatPayload() { + return lastRequestedHeartbeatPayload; + } + } + + /** + * Test {@link HeartbeatListener} that returns different payloads based on the target {@link ResourceID}. + */ + private static class TargetDependentHeartbeatSender implements HeartbeatListener { + private final ResourceID specialId; + private final int specialResponse; + private final int defaultResponse; + + TargetDependentHeartbeatSender(ResourceID specialId, int specialResponse, int defaultResponse) { + this.specialId = specialId; + this.specialResponse = specialResponse; + this.defaultResponse = defaultResponse; + } + + @Override + public void notifyHeartbeatTimeout(ResourceID resourceID) { + } + + @Override + public void reportPayload(ResourceID resourceID, Object payload) { + } + + @Override + public CompletableFuture retrievePayload(ResourceID resourceID) { + if (resourceID.equals(specialId)) { + return CompletableFuture.completedFuture(specialResponse); + } else { + return CompletableFuture.completedFuture(defaultResponse); + } + } + } + static class TestingHeartbeatListener implements HeartbeatListener { private final CompletableFuture future = new CompletableFuture<>(); @@ -378,7 +535,7 @@ public void reportPayload(ResourceID resourceID, Object payload) { } @Override - public CompletableFuture retrievePayload() { + public CompletableFuture retrievePayload(ResourceID resourceID) { return CompletableFuture.completedFuture(payload); } } From 6eb91a1006590e6806ec0e6c381fca411d0e23d7 Mon Sep 17 00:00:00 2001 From: yanghua Date: Tue, 20 Mar 2018 10:02:21 +0800 Subject: [PATCH 0202/2294] [FLINK-9019] Unclosed closeableRegistry in StreamTaskStateInitializerImpl#rawOperatorStateInputs This closes #5723. --- .../api/operators/StreamTaskStateInitializerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java index acbc2f8bc9ed5d..7e915544e4c115 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java @@ -279,8 +279,6 @@ protected CloseableIterable rawOperatorStateInputs if (restoreStateAlternatives.hasNext()) { - final CloseableRegistry closeableRegistry = new CloseableRegistry(); - Collection rawOperatorState = restoreStateAlternatives.next(); // TODO currently this does not support local state recovery, so we expect there is only one handle. Preconditions.checkState( @@ -288,8 +286,10 @@ protected CloseableIterable rawOperatorStateInputs "Local recovery is currently not implemented for raw operator state, but found state alternative."); if (rawOperatorState != null) { - return new CloseableIterable() { + + final CloseableRegistry closeableRegistry = new CloseableRegistry(); + @Override public void close() throws IOException { closeableRegistry.close(); From c90a757b29f168144b1bae99df532911ae682e63 Mon Sep 17 00:00:00 2001 From: Nico Kruber Date: Tue, 27 Feb 2018 17:23:20 +0100 Subject: [PATCH 0203/2294] [FLINK-8801][yarn/s3] fix Utils#setupLocalResource() relying on consistent read-after-write "Amazon S3 provides read-after-write consistency for PUTS of new objects in your S3 bucket in all regions with one caveat. The caveat is that if you make a HEAD or GET request to the key name (to find if the object exists) before creating the object, Amazon S3 provides eventual consistency for read-after-write." https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel Some S3 file system implementations may actually execute such a request for the about-to-write object and thus the read-after-write is only eventually consistent. org.apache.flink.yarn.Utils#setupLocalResource() currently relies on a consistent read-after-write since it accesses the remote resource to get file size and modification timestamp. Since there we have access to the local resource, we can use this metadata directly instead and circumvent the problem. This closes #5602. --- .../java/org/apache/flink/yarn/Utils.java | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java index b9f7fac2978623..b895784766f475 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java @@ -141,7 +141,8 @@ static Tuple2 setupLocalResource( Path homedir, String relativeTargetPath) throws IOException { - if (new File(localSrcPath.toUri().getPath()).isDirectory()) { + File localFile = new File(localSrcPath.toUri().getPath()); + if (localFile.isDirectory()) { throw new IllegalArgumentException("File to copy must not be a directory: " + localSrcPath); } @@ -159,11 +160,40 @@ static Tuple2 setupLocalResource( fs.copyFromLocalFile(false, true, localSrcPath, dst); + // Note: If we used registerLocalResource(FileSystem, Path) here, we would access the remote + // file once again which has problems with eventually consistent read-after-write file + // systems. Instead, we decide to preserve the modification time at the remote + // location because this and the size of the resource will be checked by YARN based on + // the values we provide to #registerLocalResource() below. + fs.setTimes(dst, localFile.lastModified(), -1); // now create the resource instance - LocalResource resource = registerLocalResource(fs, dst); + LocalResource resource = registerLocalResource(dst, localFile.length(), localFile.lastModified()); + return Tuple2.of(dst, resource); } + /** + * Creates a YARN resource for the remote object at the given location. + * + * @param remoteRsrcPath remote location of the resource + * @param resourceSize size of the resource + * @param resourceModificationTime last modification time of the resource + * + * @return YARN resource + */ + private static LocalResource registerLocalResource( + Path remoteRsrcPath, + long resourceSize, + long resourceModificationTime) { + LocalResource localResource = Records.newRecord(LocalResource.class); + localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri())); + localResource.setSize(resourceSize); + localResource.setTimestamp(resourceModificationTime); + localResource.setType(LocalResourceType.FILE); + localResource.setVisibility(LocalResourceVisibility.APPLICATION); + return localResource; + } + private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException { LocalResource localResource = Records.newRecord(LocalResource.class); FileStatus jarStat = fs.getFileStatus(remoteRsrcPath); From 95d4c0170339585cb4876c4bafc1c2a42c44be3c Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 14 Mar 2018 18:52:16 +0100 Subject: [PATCH 0204/2294] [FLINK-8881][runtime] Send accumulator updates via heartbeats --- .../flink/runtime/jobmaster/JobMaster.java | 16 +++++--- .../runtime/jobmaster/JobMasterGateway.java | 6 ++- .../taskexecutor/AccumulatorReport.java | 40 +++++++++++++++++++ .../runtime/taskexecutor/TaskExecutor.java | 32 +++++++++++---- .../utils/TestingJobMasterGateway.java | 3 +- 5 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/AccumulatorReport.java diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index f0b29bf49a347d..6878032f0e17f6 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -27,6 +27,7 @@ import org.apache.flink.queryablestate.KvStateID; import org.apache.flink.runtime.JobException; import org.apache.flink.runtime.StoppingException; +import org.apache.flink.runtime.accumulators.AccumulatorSnapshot; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.checkpoint.CheckpointCoordinator; import org.apache.flink.runtime.checkpoint.CheckpointDeclineReason; @@ -93,6 +94,7 @@ import org.apache.flink.runtime.rpc.RpcTimeout; import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils; import org.apache.flink.runtime.state.KeyGroupRange; +import org.apache.flink.runtime.taskexecutor.AccumulatorReport; import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway; import org.apache.flink.runtime.taskexecutor.slot.SlotOffer; import org.apache.flink.runtime.taskmanager.TaskExecutionState; @@ -166,7 +168,7 @@ public class JobMaster extends FencedRpcEndpoint implements JobMast private final JobManagerJobMetricGroup jobMetricGroup; /** The heartbeat manager with task managers. */ - private final HeartbeatManager taskManagerHeartbeatManager; + private final HeartbeatManager taskManagerHeartbeatManager; /** The heartbeat manager with resource manager. */ private final HeartbeatManager resourceManagerHeartbeatManager; @@ -938,8 +940,8 @@ public void disconnectResourceManager( } @Override - public void heartbeatFromTaskManager(final ResourceID resourceID) { - taskManagerHeartbeatManager.receiveHeartbeat(resourceID, null); + public void heartbeatFromTaskManager(final ResourceID resourceID, AccumulatorReport accumulatorReport) { + taskManagerHeartbeatManager.receiveHeartbeat(resourceID, accumulatorReport); } @Override @@ -1504,7 +1506,7 @@ public void jobStatusChanges( } } - private class TaskManagerHeartbeatListener implements HeartbeatListener { + private class TaskManagerHeartbeatListener implements HeartbeatListener { private final JobMasterGateway jobMasterGateway; @@ -1522,8 +1524,10 @@ public void notifyHeartbeatTimeout(ResourceID resourceID) { } @Override - public void reportPayload(ResourceID resourceID, Void payload) { - // nothing to do since there is no payload + public void reportPayload(ResourceID resourceID, AccumulatorReport payload) { + for (AccumulatorSnapshot snapshot : payload.getAccumulatorSnapshots()) { + executionGraph.updateAccumulators(snapshot); + } } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java index 1e1bdda45117a5..4ea93577fe3a14 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterGateway.java @@ -40,6 +40,7 @@ import org.apache.flink.runtime.rest.handler.legacy.backpressure.OperatorBackPressureStatsResponse; import org.apache.flink.runtime.rpc.FencedRpcGateway; import org.apache.flink.runtime.rpc.RpcTimeout; +import org.apache.flink.runtime.taskexecutor.AccumulatorReport; import org.apache.flink.runtime.taskexecutor.slot.SlotOffer; import org.apache.flink.runtime.taskmanager.TaskExecutionState; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; @@ -219,8 +220,11 @@ CompletableFuture registerTaskManager( * Sends the heartbeat to job manager from task manager. * * @param resourceID unique id of the task manager + * @param accumulatorReport report containing accumulator updates */ - void heartbeatFromTaskManager(final ResourceID resourceID); + void heartbeatFromTaskManager( + final ResourceID resourceID, + final AccumulatorReport accumulatorReport); /** * Sends heartbeat request from the resource manager. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/AccumulatorReport.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/AccumulatorReport.java new file mode 100644 index 00000000000000..7c8c767fe0c99b --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/AccumulatorReport.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.taskexecutor; + +import org.apache.flink.runtime.accumulators.AccumulatorSnapshot; + +import java.io.Serializable; +import java.util.Collection; +import java.util.List; + +/** + * A report about the current values of all accumulators of the TaskExecutor for a given job. + */ +public class AccumulatorReport implements Serializable { + private final Collection accumulatorSnapshots; + + public AccumulatorReport(List accumulatorSnapshots) { + this.accumulatorSnapshots = accumulatorSnapshots; + } + + public Collection getAccumulatorSnapshots() { + return accumulatorSnapshots; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index 7409175d44aa12..f25601e534b0c0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -107,6 +107,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -138,7 +139,7 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { private final TaskManagerConfiguration taskManagerConfiguration; /** The heartbeat manager for job manager in the task manager. */ - private final HeartbeatManager jobManagerHeartbeatManager; + private final HeartbeatManager jobManagerHeartbeatManager; /** The heartbeat manager for resource manager in the task manager. */ private final HeartbeatManager resourceManagerHeartbeatManager; @@ -1050,14 +1051,14 @@ private void establishJobManagerConnection(JobID jobId, final JobMasterGateway j jobManagerTable.put(jobId, newJobManagerConnection); // monitor the job manager as heartbeat target - jobManagerHeartbeatManager.monitorTarget(jobManagerResourceID, new HeartbeatTarget() { + jobManagerHeartbeatManager.monitorTarget(jobManagerResourceID, new HeartbeatTarget() { @Override - public void receiveHeartbeat(ResourceID resourceID, Void payload) { - jobMasterGateway.heartbeatFromTaskManager(resourceID); + public void receiveHeartbeat(ResourceID resourceID, AccumulatorReport payload) { + jobMasterGateway.heartbeatFromTaskManager(resourceID, payload); } @Override - public void requestHeartbeat(ResourceID resourceID, Void payload) { + public void requestHeartbeat(ResourceID resourceID, AccumulatorReport payload) { // request heartbeat will never be called on the task manager side } }); @@ -1488,7 +1489,7 @@ public void timeoutSlot(final AllocationID allocationId, final UUID ticket) { } } - private class JobManagerHeartbeatListener implements HeartbeatListener { + private class JobManagerHeartbeatListener implements HeartbeatListener { @Override public void notifyHeartbeatTimeout(final ResourceID resourceID) { @@ -1515,8 +1516,23 @@ public void reportPayload(ResourceID resourceID, Void payload) { } @Override - public CompletableFuture retrievePayload(ResourceID resourceID) { - return CompletableFuture.completedFuture(null); + public CompletableFuture retrievePayload(ResourceID resourceID) { + validateRunsInMainThread(); + JobManagerConnection jobManagerConnection = jobManagerConnections.get(resourceID); + if (jobManagerConnection != null) { + JobID jobId = jobManagerConnection.getJobID(); + + List accumulatorSnapshots = new ArrayList<>(16); + Iterator allTasks = taskSlotTable.getTasks(jobId); + + while (allTasks.hasNext()) { + Task task = allTasks.next(); + accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot()); + } + return CompletableFuture.completedFuture(new AccumulatorReport(accumulatorSnapshots)); + } else { + return CompletableFuture.completedFuture(new AccumulatorReport(Collections.emptyList())); + } } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java index 0d57a56b2ceb38..65117af970b7dc 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/TestingJobMasterGateway.java @@ -44,6 +44,7 @@ import org.apache.flink.runtime.resourcemanager.ResourceManagerId; import org.apache.flink.runtime.rest.handler.legacy.backpressure.OperatorBackPressureStatsResponse; import org.apache.flink.runtime.state.KeyGroupRange; +import org.apache.flink.runtime.taskexecutor.AccumulatorReport; import org.apache.flink.runtime.taskexecutor.slot.SlotOffer; import org.apache.flink.runtime.taskmanager.TaskExecutionState; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; @@ -130,7 +131,7 @@ public CompletableFuture registerTaskManager(String taskMa } @Override - public void heartbeatFromTaskManager(ResourceID resourceID) { + public void heartbeatFromTaskManager(ResourceID resourceID, AccumulatorReport accumulatorReport) { throw new UnsupportedOperationException(); } From 0231460259b410f42dd4933cb3ae993450bf1694 Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 26 Feb 2018 14:54:07 +0100 Subject: [PATCH 0205/2294] [FLINK-8703][tests] Port SavepointMigrationTestBase to MiniClusterResource This closes #5701. --- ...cyStatefulJobSavepointMigrationITCase.java | 2 +- .../utils/SavepointMigrationTestBase.java | 99 +++++++------------ .../StatefulJobSavepointMigrationITCase.java | 2 +- 3 files changed, 39 insertions(+), 64 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java index 45a691163e154e..eee13500e3c604 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/LegacyStatefulJobSavepointMigrationITCase.java @@ -89,7 +89,7 @@ public static Collection> parameters () { private final MigrationVersion testMigrateVersion; private final String testStateBackend; - public LegacyStatefulJobSavepointMigrationITCase(Tuple2 testMigrateVersionAndBackend) { + public LegacyStatefulJobSavepointMigrationITCase(Tuple2 testMigrateVersionAndBackend) throws Exception { this.testMigrateVersion = testMigrateVersionAndBackend.f0; this.testStateBackend = testMigrateVersionAndBackend.f1; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java index 28825040d4811c..91b5de8ca5e256 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java @@ -21,25 +21,21 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobSubmissionResult; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.client.program.StandaloneClusterClient; +import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.checkpoint.savepoint.SavepointSerializers; -import org.apache.flink.runtime.client.JobListeningContext; -import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; -import org.apache.flink.runtime.messages.JobManagerMessages; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.test.util.TestBaseUtils; import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Before; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; @@ -49,34 +45,35 @@ import java.net.URI; import java.net.URL; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import scala.Option; -import scala.concurrent.Await; -import scala.concurrent.Future; import scala.concurrent.duration.Deadline; -import scala.concurrent.duration.Duration; import scala.concurrent.duration.FiniteDuration; import static junit.framework.Assert.fail; +import static org.junit.Assert.assertNotEquals; /** * Test savepoint migration. */ -public class SavepointMigrationTestBase extends TestBaseUtils { +public abstract class SavepointMigrationTestBase extends TestBaseUtils { @BeforeClass public static void before() { SavepointSerializers.setFailWhenLegacyStateDetected(false); } + @ClassRule + public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); + @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); + public final MiniClusterResource miniClusterResource; private static final Logger LOG = LoggerFactory.getLogger(SavepointMigrationTestBase.class); private static final Deadline DEADLINE = new FiniteDuration(5, TimeUnit.MINUTES).fromNow(); protected static final int DEFAULT_PARALLELISM = 4; - protected LocalFlinkMiniCluster cluster = null; protected static String getResourceFilename(String filename) { ClassLoader cl = SavepointMigrationTestBase.class.getClassLoader(); @@ -87,17 +84,25 @@ protected static String getResourceFilename(String filename) { return resource.getFile(); } - @Before - public void setup() throws Exception { + protected SavepointMigrationTestBase() throws Exception { + miniClusterResource = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + 1, + DEFAULT_PARALLELISM), + true); + } + private Configuration getConfiguration() throws Exception { // Flink configuration final Configuration config = new Configuration(); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, DEFAULT_PARALLELISM); - final File checkpointDir = tempFolder.newFolder("checkpoints").getAbsoluteFile(); - final File savepointDir = tempFolder.newFolder("savepoints").getAbsoluteFile(); + UUID id = UUID.randomUUID(); + final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile(); + final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile(); if (!checkpointDir.exists() || !savepointDir.exists()) { throw new Exception("Test setup failed: failed to create (temporary) directories."); @@ -111,12 +116,7 @@ public void setup() throws Exception { config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); - cluster = TestBaseUtils.startCluster(config, false); - } - - @After - public void teardown() throws Exception { - stopCluster(cluster, TestBaseUtils.DEFAULT_TIMEOUT); + return config; } @SafeVarargs @@ -125,22 +125,20 @@ protected final void executeAndSavepoint( String savepointPath, Tuple2... expectedAccumulators) throws Exception { - // Retrieve the job manager - ActorGateway jobManager = Await.result(cluster.leaderGateway().future(), DEADLINE.timeLeft()); + ClusterClient client = miniClusterResource.getClusterClient(); + client.setDetached(true); // Submit the job JobGraph jobGraph = env.getStreamGraph().getJobGraph(); - JobSubmissionResult jobSubmissionResult = cluster.submitJobDetached(jobGraph); + JobSubmissionResult jobSubmissionResult = client.submitJob(jobGraph, SavepointMigrationTestBase.class.getClassLoader()); LOG.info("Submitted job {} and waiting...", jobSubmissionResult.getJobID()); - StandaloneClusterClient clusterClient = new StandaloneClusterClient(cluster.configuration()); - boolean done = false; while (DEADLINE.hasTimeLeft()) { Thread.sleep(100); - Map accumulators = clusterClient.getAccumulators(jobSubmissionResult.getJobID()); + Map accumulators = client.getAccumulators(jobSubmissionResult.getJobID()); boolean allDone = true; for (Tuple2 acc : expectedAccumulators) { @@ -166,18 +164,9 @@ protected final void executeAndSavepoint( LOG.info("Triggering savepoint."); - final Future savepointResultFuture = - jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobSubmissionResult.getJobID(), Option.empty()), DEADLINE.timeLeft()); + CompletableFuture savepointPathFuture = client.triggerSavepoint(jobSubmissionResult.getJobID(), null); - Object savepointResult = Await.result(savepointResultFuture, DEADLINE.timeLeft()); - - if (savepointResult instanceof JobManagerMessages.TriggerSavepointFailure) { - fail("Error drawing savepoint: " + ((JobManagerMessages.TriggerSavepointFailure) savepointResult).cause()); - } - - // jobmanager will store savepoint in heap, we have to retrieve it - final String jobmanagerSavepointPath = ((JobManagerMessages.TriggerSavepointSuccess) savepointResult).savepointPath(); - LOG.info("Saved savepoint: " + jobmanagerSavepointPath); + String jobmanagerSavepointPath = savepointPathFuture.get(DEADLINE.timeLeft().toMillis(), TimeUnit.MILLISECONDS); File jobManagerSavepoint = new File(new URI(jobmanagerSavepointPath).getPath()); // savepoints were changed to be directories in Flink 1.3 @@ -194,18 +183,15 @@ protected final void restoreAndExecute( String savepointPath, Tuple2... expectedAccumulators) throws Exception { - // Retrieve the job manager - Await.result(cluster.leaderGateway().future(), DEADLINE.timeLeft()); + ClusterClient client = miniClusterResource.getClusterClient(); + client.setDetached(true); // Submit the job JobGraph jobGraph = env.getStreamGraph().getJobGraph(); jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath)); - JobSubmissionResult jobSubmissionResult = cluster.submitJobDetached(jobGraph); - - StandaloneClusterClient clusterClient = new StandaloneClusterClient(cluster.configuration()); - JobListeningContext jobListeningContext = clusterClient.connectToJob(jobSubmissionResult.getJobID()); + JobSubmissionResult jobSubmissionResult = client.submitJob(jobGraph, SavepointMigrationTestBase.class.getClassLoader()); boolean done = false; while (DEADLINE.hasTimeLeft()) { @@ -213,30 +199,19 @@ protected final void restoreAndExecute( // try and get a job result, this will fail if the job already failed. Use this // to get out of this loop JobID jobId = jobSubmissionResult.getJobID(); - FiniteDuration timeout = FiniteDuration.apply(5, TimeUnit.SECONDS); try { + CompletableFuture jobStatusFuture = client.getJobStatus(jobSubmissionResult.getJobID()); - Future future = clusterClient - .getJobManagerGateway() - .ask(JobManagerMessages.getRequestJobStatus(jobSubmissionResult.getJobID()), timeout); + JobStatus jobStatus = jobStatusFuture.get(5, TimeUnit.SECONDS); - Object result = Await.result(future, timeout); - - if (result instanceof JobManagerMessages.CurrentJobStatus) { - if (((JobManagerMessages.CurrentJobStatus) result).status() == JobStatus.FAILED) { - Object jobResult = Await.result( - jobListeningContext.getJobResultFuture(), - Duration.apply(5, TimeUnit.SECONDS)); - fail("Job failed: " + jobResult); - } - } + assertNotEquals(JobStatus.FAILED, jobStatus); } catch (Exception e) { fail("Could not connect to job: " + e); } Thread.sleep(100); - Map accumulators = clusterClient.getAccumulators(jobId); + Map accumulators = client.getAccumulators(jobId); boolean allDone = true; for (Tuple2 acc : expectedAccumulators) { diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java index 53a535323861d9..d2de8810eb6d46 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java @@ -95,7 +95,7 @@ public static Collection> parameters () { private final MigrationVersion testMigrateVersion; private final String testStateBackend; - public StatefulJobSavepointMigrationITCase(Tuple2 testMigrateVersionAndBackend) { + public StatefulJobSavepointMigrationITCase(Tuple2 testMigrateVersionAndBackend) throws Exception { this.testMigrateVersion = testMigrateVersionAndBackend.f0; this.testStateBackend = testMigrateVersionAndBackend.f1; } From 328f72d14ea88d082fbbaae9193065be575ef846 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 15:43:33 +0100 Subject: [PATCH 0206/2294] [hotfix] [core] Add @FunctionalInterface to KeySelector That clarifies that this interface should always be a SAM interface to allow that users created lambdas for its use. --- .../flink/api/java/functions/KeySelector.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/java/functions/KeySelector.java b/flink-core/src/main/java/org/apache/flink/api/java/functions/KeySelector.java index 4aa84693318c29..5594f9ebf3b09d 100644 --- a/flink-core/src/main/java/org/apache/flink/api/java/functions/KeySelector.java +++ b/flink-core/src/main/java/org/apache/flink/api/java/functions/KeySelector.java @@ -27,19 +27,20 @@ * The {@link KeySelector} allows to use deterministic objects for operations such as * reduce, reduceGroup, join, coGroup, etc. If invoked multiple times on the same object, * the returned key must be the same. - * - * The extractor takes an object and returns the deterministic key for that object. + * + *

    The extractor takes an object and returns the deterministic key for that object. * * @param Type of objects to extract the key from. * @param Type of key. */ @Public +@FunctionalInterface public interface KeySelector extends Function, Serializable { /** * User-defined function that deterministically extracts the key from an object. - * - * For example for a class: + * + *

    For example for a class: * 

     	 * 	public class Word {
 	 * 		String word;
@@ -48,19 +49,19 @@ public interface KeySelector extends Function, Serializable {
 	 *
    * The key extractor could return the word as * a key to group all Word objects by the String they contain. - * - * The code would look like this + * + *

    The code would look like this * 

     	 * 	public String getKey(Word w) {
 	 * 		return w.word;
 	 * 	}
 	 *
    - * + * * @param value The object to get the key from. * @return The extracted key. - * + * * @throws Exception Throwing an exception will cause the execution of the respective task to fail, - * and trigger recovery or cancellation of the program. + * and trigger recovery or cancellation of the program. */ KEY getKey(IN value) throws Exception; } From 38aa863d5a710b283b5c9b2eb9225d6fb9cc0c70 Mon Sep 17 00:00:00 2001 From: sihuazhou Date: Tue, 20 Mar 2018 15:59:33 +0800 Subject: [PATCH 0207/2294] [FLINK-9028] [yarn] Perform parameters checking before Yarn starting cluster This closes #5726. --- .../ContaineredTaskManagerParameters.java | 41 +++++++++++++------ .../flink/runtime/dispatcher/Dispatcher.java | 2 +- .../ContaineredTaskManagerParametersTest.java | 32 +++++++++++++++ .../taskexecutor/TaskManagerServicesTest.java | 1 - .../yarn/AbstractYarnClusterDescriptor.java | 27 ++++++++++++ .../flink/yarn/cli/FlinkYarnSessionCli.java | 3 +- .../flink/yarn/FlinkYarnSessionCliTest.java | 4 +- .../flink/yarn/YarnClusterDescriptorTest.java | 15 ++++--- 8 files changed, 103 insertions(+), 22 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java index c4dd486c5aeb72..fa7fdf445a61ce 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java @@ -21,6 +21,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ResourceManagerOptions; import org.apache.flink.runtime.taskexecutor.TaskManagerServices; +import org.apache.flink.util.Preconditions; import java.util.HashMap; import java.util.Map; @@ -101,46 +102,62 @@ public String toString() { // ------------------------------------------------------------------------ // Factory // ------------------------------------------------------------------------ - + /** - * Computes the parameters to be used to start a TaskManager Java process. + * calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException} + * if the config is invalid or return the cutoff value if valid. * * @param config The Flink configuration. * @param containerMemoryMB The size of the complete container, in megabytes. - * @return The parameters to start the TaskManager processes with. + * + * @return cutoff memory size used by container. */ - public static ContaineredTaskManagerParameters create( - Configuration config, long containerMemoryMB, int numSlots) - { - // (1) compute how much memory we subtract from the total memory, to get the Java memory + public static long calculateCutoffMB(Configuration config, long containerMemoryMB) { + Preconditions.checkArgument(containerMemoryMB > 0); + // (1) check cutoff ratio final float memoryCutoffRatio = config.getFloat( ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO); - final int minCutoff = config.getInteger( - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN); - if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) { throw new IllegalArgumentException("The configuration value '" + ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given=" + memoryCutoffRatio); } + // (2) check min cutoff value + final int minCutoff = config.getInteger( + ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN); + if (minCutoff >= containerMemoryMB) { throw new IllegalArgumentException("The configuration value '" + ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff + "' is larger than the total container memory " + containerMemoryMB); } + // (3) check between heap and off-heap long cutoff = (long) (containerMemoryMB * memoryCutoffRatio); if (cutoff < minCutoff) { cutoff = minCutoff; } + return cutoff; + } - final long javaMemorySizeMB = containerMemoryMB - cutoff; + /** + * Computes the parameters to be used to start a TaskManager Java process. + * + * @param config The Flink configuration. + * @param containerMemoryMB The size of the complete container, in megabytes. + * @return The parameters to start the TaskManager processes with. + */ + public static ContaineredTaskManagerParameters create( + Configuration config, long containerMemoryMB, int numSlots) + { + // (1) try to compute how much memory used by container + final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); // (2) split the remaining Java memory between heap and off-heap - final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(javaMemorySizeMB, config); + final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config); // use the cut-off memory for off-heap (that was its intention) final long offHeapSizeMB = containerMemoryMB - heapSizeMB; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java index 91a4f73bf53847..68b40468a29b81 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java @@ -313,7 +313,7 @@ public CompletableFuture submitJob(JobGraph jobGraph, Time timeout) @Override public CompletableFuture> listJobs(Time timeout) { if (jobManagerRunners.isEmpty()) { - System.out.println("empty"); + log.info("empty"); } return CompletableFuture.completedFuture( Collections.unmodifiableSet(new HashSet<>(jobManagerRunners.keySet()))); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java index 8d9ea88a575561..230a9340cf8f6a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java @@ -20,6 +20,7 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.ResourceManagerOptions; import org.apache.flink.util.TestLogger; import org.junit.Test; @@ -27,6 +28,7 @@ import static org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateNetworkBufferMemory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class ContaineredTaskManagerParametersTest extends TestLogger { private static final long CONTAINER_MEMORY = 8192L; @@ -91,4 +93,34 @@ public void testTotalMemoryDoesNotExceedContainerMemoryOffHeap() { assertTrue(params.taskManagerHeapSizeMB() + params.taskManagerDirectMemoryLimitMB() <= CONTAINER_MEMORY); } + + /** + * Test to guard {@link ContaineredTaskManagerParameters#calculateCutoffMB(Configuration, long)}. + */ + @Test + public void testCalculateCutoffMB() throws Exception { + + Configuration config = new Configuration(); + long containerMemoryMB = 1000; + + config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.1f); + config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 128); + + assertEquals(128, + ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB)); + + config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.2f); + assertEquals(200, + ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB)); + + config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 1000); + + try { + ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB); + } catch (IllegalArgumentException expected) { + // we expected it. + return; + } + fail(); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java index b0c6c60a9b2b95..d3d5444b67c971 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java @@ -214,5 +214,4 @@ public void calculateHeapSizeMB() throws Exception { config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 0.1f); // 10% assertEquals(810, TaskManagerServices.calculateHeapSizeMB(1000, config)); } - } diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java index bdb471a142f96e..eab5e39f2cffb3 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java @@ -36,9 +36,11 @@ import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.clusterframework.BootstrapTools; +import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; +import org.apache.flink.runtime.taskexecutor.TaskManagerServices; import org.apache.flink.util.FlinkException; import org.apache.flink.util.Preconditions; import org.apache.flink.util.ShutdownHookUtil; @@ -409,6 +411,28 @@ public void terminateCluster(ApplicationId applicationId) throws FlinkException } } + /** + * Method to validate cluster specification before deploy it, it will throw + * an {@link IllegalConfigurationException} if the {@link ClusterSpecification} is invalid. + */ + private void validateClusterSpecification(ClusterSpecification clusterSpecification) { + long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); + long cutoff; + try { + // We do the validation by calling the calculation methods here + cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); + } catch (IllegalArgumentException cutoffConfigurationInvalidEx) { + throw new IllegalConfigurationException("Configurations related to cutoff checked failed.", cutoffConfigurationInvalidEx); + } + + try { + // We do the validation by calling the calculation methods here + TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); + } catch (IllegalArgumentException heapSizeConfigurationInvalidEx) { + throw new IllegalConfigurationException("Configurations related to heap size checked failed.", heapSizeConfigurationInvalidEx); + } + } + /** * This method will block until the ApplicationMaster/JobManager have been deployed on YARN. * @@ -423,6 +447,9 @@ protected ClusterClient deployInternal( @Nullable JobGraph jobGraph, boolean detached) throws Exception { + // ------------------ Check if configuration is valid -------------------- + validateClusterSpecification(clusterSpecification); + if (UserGroupInformation.isSecurityEnabled()) { // note: UGI::hasKerberosCredentials inaccurately reports false // for logins based on a keytab (fixed in Hadoop 2.6.1, see HADOOP-10786), diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index 2cdc19d3c93600..1443f9957cd359 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -31,6 +31,7 @@ import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.clusterframework.ApplicationStatus; +import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse; import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; import org.apache.flink.runtime.security.SecurityConfiguration; @@ -631,7 +632,7 @@ public int run(String[] args) throws CliArgsException, FlinkException { if (detachedMode) { LOG.info("The Flink YARN client has been started in detached mode. In order to stop " + "Flink on YARN, use the following command or a YARN web interface to stop it:\n" + - "yarn application -kill " + applicationId.getOpt()); + "yarn application -kill " + yarnApplicationId); } else { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java index 20ce314399f5bc..5b0d42219bedc1 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java @@ -264,8 +264,8 @@ public void testCommandLineClusterSpecification() throws Exception { configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, 7331); configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 2); - final int jobManagerMemory = 42; - final int taskManagerMemory = 41; + final int jobManagerMemory = 1337; + final int taskManagerMemory = 7331; final int slotsPerTaskManager = 30; final String[] args = {"-yjm", String.valueOf(jobManagerMemory), "-ytm", String.valueOf(taskManagerMemory), "-ys", String.valueOf(slotsPerTaskManager)}; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java index dd8b62536c2604..52bf8bb1a6a833 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java @@ -24,6 +24,7 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; import org.apache.flink.configuration.IllegalConfigurationException; +import org.apache.flink.configuration.ResourceManagerOptions; import org.apache.flink.core.testutils.CommonTestUtils; import org.apache.flink.util.TestLogger; import org.apache.flink.yarn.cli.FlinkYarnSessionCli; @@ -91,8 +92,11 @@ public static void tearDownClass() { @Test public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentException { + final Configuration flinkConfiguration = new Configuration(); + flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0); + YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor( - new Configuration(), + flinkConfiguration, yarnConfiguration, temporaryFolder.getRoot().getAbsolutePath(), yarnClient, @@ -101,8 +105,8 @@ public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentExc clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath())); ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder() - .setMasterMemoryMB(-1) - .setTaskManagerMemoryMB(-1) + .setMasterMemoryMB(1) + .setTaskManagerMemoryMB(1) .setNumberTaskManagers(1) .setSlotsPerTaskManager(Integer.MAX_VALUE) .createClusterSpecification(); @@ -126,6 +130,7 @@ public void testConfigOverwrite() throws ClusterDeploymentException { Configuration configuration = new Configuration(); // overwrite vcores in config configuration.setInteger(YarnConfigOptions.VCORES, Integer.MAX_VALUE); + configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0); YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor( configuration, @@ -138,8 +143,8 @@ public void testConfigOverwrite() throws ClusterDeploymentException { // configure slots ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder() - .setMasterMemoryMB(-1) - .setTaskManagerMemoryMB(-1) + .setMasterMemoryMB(1) + .setTaskManagerMemoryMB(1) .setNumberTaskManagers(1) .setSlotsPerTaskManager(1) .createClusterSpecification(); From 7c952dd3a75bc64d10bf9be12e405bbc349422b1 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 20 Mar 2018 15:38:02 +0100 Subject: [PATCH 0208/2294] [FLINK-9028] [yarn] Improve failure message if cluster cannot be started --- .../ContaineredTaskManagerParameters.java | 28 +++++++++---------- .../ContaineredTaskManagerParametersTest.java | 9 +++--- .../taskexecutor/TaskManagerServicesTest.java | 2 +- .../yarn/AbstractYarnClusterDescriptor.java | 26 ++++++++--------- .../flink/yarn/cli/FlinkYarnSessionCli.java | 1 - .../flink/yarn/FlinkYarnSessionCliTest.java | 9 +++--- 6 files changed, 37 insertions(+), 38 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java index fa7fdf445a61ce..a4e7d25017ba4a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParameters.java @@ -32,23 +32,22 @@ public class ContaineredTaskManagerParameters implements java.io.Serializable { private static final long serialVersionUID = -3096987654278064670L; - - /** Total container memory, in bytes */ + + /** Total container memory, in bytes. */ private final long totalContainerMemoryMB; - /** Heap size to be used for the Java process */ + /** Heap size to be used for the Java process. */ private final long taskManagerHeapSizeMB; - /** Direct memory limit for the Java process */ + /** Direct memory limit for the Java process. */ private final long taskManagerDirectMemoryLimitMB; - /** The number of slots per TaskManager */ + /** The number of slots per TaskManager. */ private final int numSlots; - - /** Environment variables to add to the Java process */ + + /** Environment variables to add to the Java process. */ private final HashMap taskManagerEnv; - public ContaineredTaskManagerParameters( long totalContainerMemoryMB, long taskManagerHeapSizeMB, @@ -62,7 +61,7 @@ public ContaineredTaskManagerParameters( this.numSlots = numSlots; this.taskManagerEnv = taskManagerEnv; } - + // ------------------------------------------------------------------------ public long taskManagerTotalMemoryMB() { @@ -87,7 +86,7 @@ public Map taskManagerEnv() { // ------------------------------------------------------------------------ - + @Override public String toString() { return "TaskManagerParameters {" + @@ -104,7 +103,7 @@ public String toString() { // ------------------------------------------------------------------------ /** - * calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException} + * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException} * if the config is invalid or return the cutoff value if valid. * * @param config The Flink configuration. @@ -151,8 +150,9 @@ public static long calculateCutoffMB(Configuration config, long containerMemoryM * @return The parameters to start the TaskManager processes with. */ public static ContaineredTaskManagerParameters create( - Configuration config, long containerMemoryMB, int numSlots) - { + Configuration config, + long containerMemoryMB, + int numSlots) { // (1) try to compute how much memory used by container final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); @@ -164,7 +164,7 @@ public static ContaineredTaskManagerParameters create( // (3) obtain the additional environment variables from the configuration final HashMap envVars = new HashMap<>(); final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX; - + for (String key : config.keySet()) { if (key.startsWith(prefix) && key.length() > prefix.length()) { // remove prefix diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java index 230a9340cf8f6a..8537d17c1ba8f6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/ContaineredTaskManagerParametersTest.java @@ -98,10 +98,10 @@ public void testTotalMemoryDoesNotExceedContainerMemoryOffHeap() { * Test to guard {@link ContaineredTaskManagerParameters#calculateCutoffMB(Configuration, long)}. */ @Test - public void testCalculateCutoffMB() throws Exception { + public void testCalculateCutoffMB() { Configuration config = new Configuration(); - long containerMemoryMB = 1000; + long containerMemoryMB = 1000L; config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.1f); config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 128); @@ -117,10 +117,9 @@ public void testCalculateCutoffMB() throws Exception { try { ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB); - } catch (IllegalArgumentException expected) { + fail("Expected to fail with an invalid argument exception."); + } catch (IllegalArgumentException ignored) { // we expected it. - return; } - fail(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java index d3d5444b67c971..f6e7b07e07b9fa 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesTest.java @@ -44,7 +44,7 @@ public class TaskManagerServicesTest extends TestLogger { */ @SuppressWarnings("deprecation") @Test - public void calculateNetworkBufOld() throws Exception { + public void calculateNetworkBufOld() { Configuration config = new Configuration(); config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1); diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java index eab5e39f2cffb3..caf7a7614c7f21 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java @@ -413,23 +413,23 @@ public void terminateCluster(ApplicationId applicationId) throws FlinkException /** * Method to validate cluster specification before deploy it, it will throw - * an {@link IllegalConfigurationException} if the {@link ClusterSpecification} is invalid. + * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. + * + * @param clusterSpecification cluster specification to check against the configuration of the + * AbstractYarnClusterDescriptor + * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ - private void validateClusterSpecification(ClusterSpecification clusterSpecification) { - long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); - long cutoff; - try { - // We do the validation by calling the calculation methods here - cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); - } catch (IllegalArgumentException cutoffConfigurationInvalidEx) { - throw new IllegalConfigurationException("Configurations related to cutoff checked failed.", cutoffConfigurationInvalidEx); - } - + private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { + final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here + // Internally these methods will check whether the cluster can be started with the provided + // ClusterSpecification and the configured memory requirements + final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); - } catch (IllegalArgumentException heapSizeConfigurationInvalidEx) { - throw new IllegalConfigurationException("Configurations related to heap size checked failed.", heapSizeConfigurationInvalidEx); + } catch (IllegalArgumentException iae) { + throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + + "cluster specification. Please increase the memory of the cluster.", iae); } } diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index 1443f9957cd359..2311e875c2f13a 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -31,7 +31,6 @@ import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.clusterframework.ApplicationStatus; -import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; import org.apache.flink.runtime.clusterframework.messages.GetClusterStatusResponse; import org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter; import org.apache.flink.runtime.security.SecurityConfiguration; diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java index 5b0d42219bedc1..62110eda5bfdbd 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java @@ -260,13 +260,14 @@ public void testYarnIDOverridesPropertiesFile() throws Exception { @Test public void testCommandLineClusterSpecification() throws Exception { final Configuration configuration = new Configuration(); - configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, 1337); - configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, 7331); - configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 2); - final int jobManagerMemory = 1337; final int taskManagerMemory = 7331; final int slotsPerTaskManager = 30; + + configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory); + configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory); + configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager); + final String[] args = {"-yjm", String.valueOf(jobManagerMemory), "-ytm", String.valueOf(taskManagerMemory), "-ys", String.valueOf(slotsPerTaskManager)}; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, From f9df13c5058f194a5c686b9b753345d9226fc87a Mon Sep 17 00:00:00 2001 From: sihuazhou Date: Mon, 19 Mar 2018 19:48:32 +0800 Subject: [PATCH 0209/2294] [FLINK-9022][state] Fix resource release in StreamTaskStateInitializerImpl.streamOperatorStateContext() This closes #5716. --- .../streaming/state/RocksDBKeyedStateBackend.java | 3 ++- .../streaming/api/operators/AbstractStreamOperator.java | 4 ++-- .../api/operators/StreamTaskStateInitializerImpl.java | 9 +++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java index 6a661210ec703f..41b7bd00dbfc77 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBKeyedStateBackend.java @@ -1620,7 +1620,6 @@ public RunnableFuture> performSnapshot( CheckpointOptions checkpointOptions) throws Exception { long startTime = System.currentTimeMillis(); - final CloseableRegistry snapshotCloseableRegistry = new CloseableRegistry(); if (kvStateInformation.isEmpty()) { if (LOG.isDebugEnabled()) { @@ -1647,6 +1646,8 @@ public RunnableFuture> performSnapshot( CheckpointedStateScope.EXCLUSIVE, primaryStreamFactory); + final CloseableRegistry snapshotCloseableRegistry = new CloseableRegistry(); + final RocksDBFullSnapshotOperation snapshotOperation = new RocksDBFullSnapshotOperation<>( RocksDBKeyedStateBackend.this, diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java index 4d3f9f57fc77cb..e447cbeec05b07 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java @@ -248,8 +248,8 @@ public final void initializeState() throws Exception { context.isRestored(), // information whether we restore or start for the first time operatorStateBackend, // access to operator state backend keyedStateStore, // access to keyed state backend - keyedStateInputs, // access to operator state stream - operatorStateInputs); // access to keyed state stream + keyedStateInputs, // access to keyed state stream + operatorStateInputs); // access to operator state stream initializeState(initializationContext); } finally { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java index 7e915544e4c115..d9bd089b1bcb73 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java @@ -166,11 +166,12 @@ public StreamOperatorStateContext streamOperatorStateContext( // cleanup if something went wrong before results got published. if (streamTaskCloseableRegistry.unregisterCloseable(keyedStatedBackend)) { - IOUtils.closeQuietly(keyedStatedBackend); + // release resource (e.g native resource) + keyedStatedBackend.dispose(); } if (streamTaskCloseableRegistry.unregisterCloseable(operatorStateBackend)) { - IOUtils.closeQuietly(keyedStatedBackend); + operatorStateBackend.dispose(); } if (streamTaskCloseableRegistry.unregisterCloseable(rawKeyedStateInputs)) { @@ -181,10 +182,6 @@ public StreamOperatorStateContext streamOperatorStateContext( IOUtils.closeQuietly(rawOperatorStateInputs); } - if (streamTaskCloseableRegistry.unregisterCloseable(rawOperatorStateInputs)) { - IOUtils.closeQuietly(rawOperatorStateInputs); - } - throw new Exception("Exception while creating StreamOperatorStateContext.", ex); } } From 363e8d2d6513e49929dfb280c3be420fb1af0bcd Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 15:15:08 +0100 Subject: [PATCH 0210/2294] [hotfix] [core] Fix checkstyle in 'org.apache.flink.api.common.state' --- .../api/common/state/AggregatingState.java | 10 +++--- .../state/AggregatingStateDescriptor.java | 6 ++-- .../api/common/state/AppendingState.java | 26 ++++++-------- .../api/common/state/BroadcastState.java | 4 +-- .../flink/api/common/state/FoldingState.java | 4 +-- .../common/state/FoldingStateDescriptor.java | 2 +- .../api/common/state/KeyedStateStore.java | 2 +- .../flink/api/common/state/ListState.java | 9 ++--- .../api/common/state/ListStateDescriptor.java | 8 ++--- .../flink/api/common/state/MapState.java | 16 ++++----- .../api/common/state/MapStateDescriptor.java | 12 +++---- .../flink/api/common/state/MergingState.java | 2 +- .../api/common/state/OperatorStateStore.java | 11 ++---- .../flink/api/common/state/ReducingState.java | 4 +-- .../common/state/ReducingStateDescriptor.java | 8 ++--- .../flink/api/common/state/StateBinder.java | 4 +-- .../api/common/state/StateDescriptor.java | 12 +++---- .../flink/api/common/state/ValueState.java | 14 ++++---- .../common/state/ValueStateDescriptor.java | 9 ++--- .../state/AggregatingStateDescriptorTest.java | 8 +++-- .../common/state/ListStateDescriptorTest.java | 23 +++++++------ .../common/state/MapStateDescriptorTest.java | 19 ++++++----- .../state/ReducingStateDescriptorTest.java | 34 ++++++++++--------- .../state/ValueStateDescriptorTest.java | 33 ++++++++++-------- tools/maven/suppressions-core.xml | 4 --- 25 files changed, 144 insertions(+), 140 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingState.java index e69fdb411db043..5c72650412b295 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingState.java @@ -22,22 +22,22 @@ import org.apache.flink.api.common.functions.AggregateFunction; /** - * {@link State} interface for aggregating state, based on an + * {@link State} interface for aggregating state, based on an * {@link AggregateFunction}. Elements that are added to this type of state will * be eagerly pre-aggregated using a given {@code AggregateFunction}. - * + * *

    The state holds internally always the accumulator type of the {@code AggregateFunction}. - * When accessing the result of the state, the function's + * When accessing the result of the state, the function's * {@link AggregateFunction#getResult(Object)} method. * *

    The state is accessed and modified by user functions, and checkpointed consistently * by the system as part of the distributed snapshots. - * + * *

    The state is only accessible by functions applied on a {@code KeyedStream}. The key is * automatically supplied by the system, so the function always sees the value mapped to the * key of the current element. That way, the system can handle stream and state partitioning * consistently together. - * + * * @param Type of the value added to the state. * @param Type of the value extracted from the state. */ diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java index b7378d6a30b244..6f6d2f9790e086 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java @@ -30,7 +30,7 @@ * *

    The type internally stored in the state is the type of the {@code Accumulator} of the * {@code AggregateFunction}. - * + * * @param The type of the values that are added to the state. * @param The type of the accumulator (intermediate aggregation state). * @param The type of the values that are returned from the state. @@ -39,7 +39,7 @@ public class AggregatingStateDescriptor extends StateDescriptor, ACC> { private static final long serialVersionUID = 1L; - /** The aggregation function for the state */ + /** The aggregation function for the state. */ private final AggregateFunction aggFunction; /** @@ -49,7 +49,7 @@ public class AggregatingStateDescriptor extends StateDescriptorThe state is accessed and modified by user functions, and checkpointed consistently * by the system as part of the distributed snapshots. - * + * *

    The state is only accessible by functions applied on a {@code KeyedStream}. The key is * automatically supplied by the system, so the function always sees the value mapped to the * key of the current element. That way, the system can handle stream and state partitioning * consistently together. - * + * * @param Type of the value that can be added to the state. * @param Type of the value that can be retrieved from the state. */ @@ -47,29 +45,27 @@ public interface AppendingState extends State { * depends on the current operator input, as the operator maintains an * independent state for each partition. * - *

    - * NOTE TO IMPLEMENTERS: if the state is empty, then this method - * should return {@code null}. - *

    + *

    NOTE TO IMPLEMENTERS: if the state is empty, then this method + * should return {@code null}. * * @return The operator state value corresponding to the current input or {@code null} * if the state is empty. - * + * * @throws Exception Thrown if the system cannot access the state. */ - OUT get() throws Exception ; + OUT get() throws Exception; /** * Updates the operator state accessible by {@link #get()} by adding the given value * to the list of values. The next time {@link #get()} is called (for the same state * partition) the returned state will represent the updated list. * - * If `null` is passed in, the state value will remain unchanged - * + *

    If null is passed in, the state value will remain unchanged. + * * @param value The new value for the state. - * - * @throws IOException Thrown if the system cannot access the state. + * + * @throws Exception Thrown if the system cannot access the state. */ void add(IN value) throws Exception; - + } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/BroadcastState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/BroadcastState.java index 0cece41a46f2fb..fcc8bbf713feb9 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/BroadcastState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/BroadcastState.java @@ -29,7 +29,7 @@ * *

    CAUTION: the user has to guarantee that all task instances store the same elements in this type of state. * - *

    Each operator instance individually maintains and stores elements in the broadcast state. The fact that the + *

    Each operator instance individually maintains and stores elements in the broadcast state. The fact that the * incoming stream is a broadcast one guarantees that all instances see all the elements. Upon recovery * or re-scaling, the same state is given to each of the instances. To avoid hotspots, each task reads its previous * partition, and if there are more tasks (scale up), then the new instances read from the old instances in a round @@ -80,7 +80,7 @@ public interface BroadcastState extends ReadOnlyBroadcastState { Iterator> iterator() throws Exception; /** - * Returns all the mappings in the state + * Returns all the mappings in the state. * * @return An iterable view of all the key-value pairs in the state. * diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java index df9a0c6aa4bc90..928e62ba305d0d 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingState.java @@ -27,12 +27,12 @@ * *

    The state is accessed and modified by user functions, and checkpointed consistently * by the system as part of the distributed snapshots. - * + * *

    The state is only accessible by functions applied on a {@code KeyedStream}. The key is * automatically supplied by the system, so the function always sees the value mapped to the * key of the current element. That way, the system can handle stream and state partitioning * consistently together. - * + * * @param Type of the values folded into the state * @param Type of the value in the state * diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java index 09540477e9941d..261d1fe47219da 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java @@ -98,7 +98,7 @@ public FoldingStateDescriptor(String name, ACC initialValue, FoldFunction bind(StateBinder stateBinder) throws Exception { return stateBinder.createFoldingState(this); diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java b/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java index a1038a84fc98a3..e3726b6f68ed8e 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/KeyedStateStore.java @@ -281,5 +281,5 @@ public interface KeyedStateStore { * function (function is not part of a KeyedStream). */ @PublicEvolving - MapState getMapState(MapStateDescriptor stateProperties); + MapState getMapState(MapStateDescriptor stateProperties); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java index 74f275b3c64d4f..254dc1d6140763 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ListState.java @@ -26,22 +26,23 @@ * {@link State} interface for partitioned list state in Operations. * The state is accessed and modified by user functions, and checkpointed consistently * by the system as part of the distributed snapshots. - * + * *

    The state is only accessible by functions applied on a {@code KeyedStream}. The key is * automatically supplied by the system, so the function always sees the value mapped to the * key of the current element. That way, the system can handle stream and state partitioning * consistently together. - * + * * @param Type of values that this list state keeps. */ @PublicEvolving public interface ListState extends MergingState> { + /** * Updates the operator state accessible by {@link #get()} by updating existing values to * to the given list of values. The next time {@link #get()} is called (for the same state * partition) the returned state will represent the updated list. * - * If `null` or an empty list is passed in, the state value will be null + *

    If null or an empty list is passed in, the state value will be null. * * @param values The new values for the state. * @@ -54,7 +55,7 @@ public interface ListState extends MergingState> { * to existing list of values. The next time {@link #get()} is called (for the same state * partition) the returned state will represent the updated list. * - * If `null` or an empty list is passed in, the state value remains unchanged + *

    If null or an empty list is passed in, the state value remains unchanged. * * @param values The new values to be added to the state. * diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java index e59d6ee832d57c..38e56803330e5a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java @@ -29,12 +29,12 @@ /** * A {@link StateDescriptor} for {@link ListState}. This can be used to create state where the type * is a list that can be appended and iterated over. - * + * *

    Using {@code ListState} is typically more efficient than manually maintaining a list in a * {@link ValueState}, because the backing implementation can support efficient appends, rather than * replacing the full list on write. - * - *

    To create keyed list state (on a KeyedStream), use + * + *

    To create keyed list state (on a KeyedStream), use * {@link org.apache.flink.api.common.functions.RuntimeContext#getListState(ListStateDescriptor)}. * * @param The type of the values that can be added to the list state. @@ -85,7 +85,7 @@ public ListState bind(StateBinder stateBinder) throws Exception { /** * Gets the serializer for the elements contained in the list. - * + * * @return The serializer for the elements in the list. */ public TypeSerializer getElementSerializer() { diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/MapState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/MapState.java index f37fddd67e04a3..7a130d49083d2a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/MapState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/MapState.java @@ -60,7 +60,7 @@ public interface MapState extends State { * @throws Exception Thrown if the system cannot access the state. */ void put(UK key, UV value) throws Exception; - + /** * Copies all of the mappings from the given map into the state. * @@ -90,19 +90,19 @@ public interface MapState extends State { boolean contains(UK key) throws Exception; /** - * Returns all the mappings in the state + * Returns all the mappings in the state. * * @return An iterable view of all the key-value pairs in the state. - * + * * @throws Exception Thrown if the system cannot access the state. */ Iterable> entries() throws Exception; - + /** - * Returns all the keys in the state + * Returns all the keys in the state. * * @return An iterable view of all the keys in the state. - * + * * @throws Exception Thrown if the system cannot access the state. */ Iterable keys() throws Exception; @@ -111,7 +111,7 @@ public interface MapState extends State { * Returns all the values in the state. * * @return An iterable view of all the values in the state. - * + * * @throws Exception Thrown if the system cannot access the state. */ Iterable values() throws Exception; @@ -120,7 +120,7 @@ public interface MapState extends State { * Iterates over all the mappings in the state. * * @return An iterator over all the mappings in the state - * + * * @throws Exception Thrown if the system cannot access the state. */ Iterator> iterator() throws Exception; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java index 16c00cb4394302..2e7ac98778f322 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java @@ -29,12 +29,12 @@ /** * A {@link StateDescriptor} for {@link MapState}. This can be used to create state where the type * is a map that can be updated and iterated over. - * + * *

    Using {@code MapState} is typically more efficient than manually maintaining a map in a * {@link ValueState}, because the backing implementation can support efficient updates, rather then * replacing the full map on write. - * - *

    To create keyed map state (on a KeyedStream), use + * + *

    To create keyed map state (on a KeyedStream), use * {@link org.apache.flink.api.common.functions.RuntimeContext#getMapState(MapStateDescriptor)}. * * @param The type of the keys that can be added to the map state. @@ -90,7 +90,7 @@ public Type getType() { /** * Gets the serializer for the keys in the state. - * + * * @return The serializer for the keys in the state. */ public TypeSerializer getKeySerializer() { @@ -115,7 +115,7 @@ public TypeSerializer getValueSerializer() { return ((MapSerializer) rawSerializer).getValueSerializer(); } - + @Override public int hashCode() { int result = serializer.hashCode(); @@ -128,7 +128,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - + if (o == null || getClass() != o.getClass()) { return false; } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java index e79f90746252b6..8c1631336139cb 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/MergingState.java @@ -24,7 +24,7 @@ * Extension of {@link AppendingState} that allows merging of state. That is, two instances * of {@link MergingState} can be combined into a single instance that contains all the * information of the two merged states. - * + * * @param Type of the value that can be added to the state. * @param Type of the value that can be retrieved from the state. */ diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorStateStore.java b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorStateStore.java index c2037e0b584258..7a998e6149c98a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorStateStore.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorStateStore.java @@ -49,8 +49,7 @@ public interface OperatorStateStore { * @param The type of the keys in the broadcast state. * @param The type of the values in the broadcast state. * - * @return The {@link BroadcastState Broadcast State}. - * @throws Exception + * @return The Broadcast State */ BroadcastState getBroadcastState(MapStateDescriptor stateDescriptor) throws Exception; @@ -73,7 +72,6 @@ public interface OperatorStateStore { * @param The generic type of the state * * @return A list for all state partitions. - * @throws Exception */ ListState getListState(ListStateDescriptor stateDescriptor) throws Exception; @@ -97,7 +95,6 @@ public interface OperatorStateStore { * @param The generic type of the state * * @return A list for all state partitions. - * @throws Exception */ ListState getUnionListState(ListStateDescriptor stateDescriptor) throws Exception; @@ -123,13 +120,12 @@ public interface OperatorStateStore { * Creates (or restores) a list state. Each state is registered under a unique name. * The provided serializer is used to de/serialize the state in case of checkpointing (snapshot/restore). * - * The items in the list are repartitionable by the system in case of changed operator parallelism. + *

    The items in the list are repartitionable by the system in case of changed operator parallelism. * * @param stateDescriptor The descriptor for this state, providing a name and serializer. * @param The generic type of the state * * @return A list for all state partitions. - * @throws Exception * * @deprecated since 1.3.0. This was deprecated as part of a refinement to the function names. * Please use {@link #getListState(ListStateDescriptor)} instead. @@ -140,13 +136,12 @@ public interface OperatorStateStore { /** * Creates a state of the given name that uses Java serialization to persist the state. The items in the list * are repartitionable by the system in case of changed operator parallelism. - * + * *

    This is a simple convenience method. For more flexibility on how state serialization * should happen, use the {@link #getListState(ListStateDescriptor)} method. * * @param stateName The name of state to create * @return A list state using Java serialization to serialize state objects. - * @throws Exception * * @deprecated since 1.3.0. Using Java serialization for persisting state is not encouraged. * Please use {@link #getListState(ListStateDescriptor)} instead. diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java index 25777ebcc49378..0fe3ed9453b3cb 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingState.java @@ -26,12 +26,12 @@ * *

    The state is accessed and modified by user functions, and checkpointed consistently * by the system as part of the distributed snapshots. - * + * *

    The state is only accessible by functions applied on a {@code KeyedStream}. The key is * automatically supplied by the system, so the function always sees the value mapped to the * key of the current element. That way, the system can handle stream and state partitioning * consistently together. - * + * * @param Type of the value in the operator state */ @PublicEvolving diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java index 3edf1caf8a9aaf..a14b4bd1815575 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java @@ -35,9 +35,9 @@ */ @PublicEvolving public class ReducingStateDescriptor extends StateDescriptor, T> { + private static final long serialVersionUID = 1L; - - + private final ReduceFunction reduceFunction; /** @@ -47,7 +47,7 @@ public class ReducingStateDescriptor extends StateDescriptor * consider using the {@link #ReducingStateDescriptor(String, ReduceFunction, TypeInformation)} constructor. * * @param name The (unique) name for the state. - * @param reduceFunction The {@code ReduceFunction} used to aggregate the state. + * @param reduceFunction The {@code ReduceFunction} used to aggregate the state. * @param typeClass The type of the values in the state. */ public ReducingStateDescriptor(String name, ReduceFunction reduceFunction, Class typeClass) { @@ -84,7 +84,7 @@ public ReducingStateDescriptor(String name, ReduceFunction reduceFunction, Ty } // ------------------------------------------------------------------------ - + @Override public ReducingState bind(StateBinder stateBinder) throws Exception { return stateBinder.createReducingState(this); diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java index a1f7d8d2939f56..871b4a8eb371d7 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateBinder.java @@ -56,8 +56,8 @@ public interface StateBinder { * @param stateDesc The {@code StateDescriptor} that contains the name of the state. * * @param The type of the values that go into the aggregating state - * @param The type of the values that are stored in the aggregating state - * @param The type of the values that come out of the aggregating state + * @param The type of the values that are stored in the aggregating state + * @param The type of the values that come out of the aggregating state */ AggregatingState createAggregatingState( AggregatingStateDescriptor stateDesc) throws Exception; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java index b603c719ad856a..841f710db59176 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java @@ -82,7 +82,7 @@ public enum Type { /** Name for queries against state created from this StateDescriptor. */ private String queryableStateName; - /** The default value returned by the state when no other value is bound to a key */ + /** The default value returned by the state when no other value is bound to a key. */ protected transient T defaultValue; /** The type information describing the value type. Only used to lazily create the serializer @@ -111,7 +111,7 @@ protected StateDescriptor(String name, TypeSerializer serializer, T defaultVa * @param name The name of the {@code StateDescriptor}. * @param typeInfo The type information for the values in the state. * @param defaultValue The default value that will be set when requesting state without setting - * a value before. + * a value before. */ protected StateDescriptor(String name, TypeInformation typeInfo, T defaultValue) { this.name = requireNonNull(name, "name must not be null"); @@ -301,8 +301,8 @@ private void writeObject(final ObjectOutputStream out) throws IOException { byte[] serializedDefaultValue; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) - { + DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) { + TypeSerializer duplicateSerializer = serializer.duplicate(); duplicateSerializer.serialize(defaultValue, outView); @@ -333,8 +333,8 @@ private void readObject(final ObjectInputStream in) throws IOException, ClassNot in.readFully(buffer); try (ByteArrayInputStream bais = new ByteArrayInputStream(buffer); - DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(bais)) - { + DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(bais)) { + defaultValue = serializer.deserialize(inView); } catch (Exception e) { diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java index ac5571516d9709..777e84af0b0958 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueState.java @@ -28,12 +28,12 @@ * *

    The state is accessed and modified by user functions, and checkpointed consistently * by the system as part of the distributed snapshots. - * + * *

    The state is only accessible by functions applied on a {@code KeyedStream}. The key is * automatically supplied by the system, so the function always sees the value mapped to the * key of the current element. That way, the system can handle stream and state partitioning * consistently together. - * + * * @param Type of the value in the state. */ @PublicEvolving @@ -50,7 +50,7 @@ public interface ValueState extends State { * this will return {@code null} when to value was previously set using {@link #update(Object)}. * * @return The state value corresponding to the current input. - * + * * @throws IOException Thrown if the system cannot access the state. */ T value() throws IOException; @@ -59,13 +59,13 @@ public interface ValueState extends State { * Updates the operator state accessible by {@link #value()} to the given * value. The next time {@link #value()} is called (for the same state * partition) the returned state will represent the updated value. When a - * partitioned state is updated with null, the state for the current key + * partitioned state is updated with null, the state for the current key * will be removed and the default value is returned on the next access. - * + * * @param value The new value for the state. - * + * * @throws IOException Thrown if the system cannot access the state. */ void update(T value) throws IOException; - + } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java index 3afc8a7b185b7b..ef18d741209c31 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java @@ -34,11 +34,12 @@ */ @PublicEvolving public class ValueStateDescriptor extends StateDescriptor, T> { + private static final long serialVersionUID = 1L; - + /** * Creates a new {@code ValueStateDescriptor} with the given name, type, and default value. - * + * *

    If this constructor fails (because it is not possible to describe the type via a class), * consider using the {@link #ValueStateDescriptor(String, TypeInformation, Object)} constructor. * @@ -46,7 +47,7 @@ public class ValueStateDescriptor extends StateDescriptor, T> { * the default value by checking whether the contents of the state is {@code null}. * * @param name The (unique) name for the state. - * @param typeClass The type of the values in the state. + * @param typeClass The type of the values in the state. * @param defaultValue The default value that will be set when requesting state without setting * a value before. */ @@ -122,7 +123,7 @@ public ValueStateDescriptor(String name, TypeSerializer typeSerializer) { } // ------------------------------------------------------------------------ - + @Override public ValueState bind(StateBinder stateBinder) throws Exception { return stateBinder.createValueState(this); diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java index 1b27ebd89ee1ac..155f23a9c637e7 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.functions.AggregateFunction; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.util.TestLogger; + import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -29,12 +30,15 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Tests for the {@link AggregatingStateDescriptor}. + */ public class AggregatingStateDescriptorTest extends TestLogger { /** - * FLINK-6775 + * FLINK-6775. * - * Tests that the returned serializer is duplicated. This allows to + *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ @SuppressWarnings("unchecked") diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java index 0b230ad06d06ca..c6d086e695c560 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java @@ -41,16 +41,19 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Tests for the {@link ListStateDescriptor}. + */ public class ListStateDescriptorTest { - + @Test public void testValueStateDescriptorEagerSerializer() throws Exception { TypeSerializer serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); - - ListStateDescriptor descr = - new ListStateDescriptor("testName", serializer); - + + ListStateDescriptor descr = + new ListStateDescriptor<>("testName", serializer); + assertEquals("testName", descr.getName()); assertNotNull(descr.getSerializer()); assertTrue(descr.getSerializer() instanceof ListSerializer); @@ -74,8 +77,8 @@ public void testValueStateDescriptorLazySerializer() throws Exception { cfg.registerKryoType(TaskInfo.class); ListStateDescriptor descr = - new ListStateDescriptor("testName", Path.class); - + new ListStateDescriptor<>("testName", Path.class); + try { descr.getSerializer(); fail("should cause an exception"); @@ -96,7 +99,7 @@ public void testValueStateDescriptorLazySerializer() throws Exception { public void testValueStateDescriptorAutoSerializer() throws Exception { ListStateDescriptor descr = - new ListStateDescriptor("testName", String.class); + new ListStateDescriptor<>("testName", String.class); ListStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); @@ -110,9 +113,9 @@ public void testValueStateDescriptorAutoSerializer() throws Exception { } /** - * FLINK-6775 + * FLINK-6775. * - * Tests that the returned serializer is duplicated. This allows to + *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ @SuppressWarnings("unchecked") diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java index d71091141d7a1d..e2aa940351d2a4 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java @@ -42,17 +42,20 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Tests for the {@link MapStateDescriptor}. + */ public class MapStateDescriptorTest { - + @Test public void testMapStateDescriptorEagerSerializer() throws Exception { TypeSerializer keySerializer = new KryoSerializer<>(Integer.class, new ExecutionConfig()); TypeSerializer valueSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); - - MapStateDescriptor descr = + + MapStateDescriptor descr = new MapStateDescriptor<>("testName", keySerializer, valueSerializer); - + assertEquals("testName", descr.getName()); assertNotNull(descr.getSerializer()); assertTrue(descr.getSerializer() instanceof MapSerializer); @@ -81,7 +84,7 @@ public void testMapStateDescriptorLazySerializer() throws Exception { MapStateDescriptor descr = new MapStateDescriptor<>("testName", Path.class, String.class); - + try { descr.getSerializer(); fail("should cause an exception"); @@ -96,7 +99,7 @@ public void testMapStateDescriptorLazySerializer() throws Exception { assertTrue(descr.getKeySerializer() instanceof KryoSerializer); assertTrue(((KryoSerializer) descr.getKeySerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0); - + assertNotNull(descr.getValueSerializer()); assertTrue(descr.getValueSerializer() instanceof StringSerializer); } @@ -121,9 +124,9 @@ public void testMapStateDescriptorAutoSerializer() throws Exception { } /** - * FLINK-6775 + * FLINK-6775. * - * Tests that the returned serializer is duplicated. This allows to + *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ @SuppressWarnings("unchecked") diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java index aec71402265645..ef39f1496c722a 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java @@ -26,8 +26,8 @@ import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; - import org.apache.flink.util.TestLogger; + import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -36,24 +36,26 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; - import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Tests for the {@link ReducingStateDescriptor}. + */ public class ReducingStateDescriptorTest extends TestLogger { - + @Test public void testValueStateDescriptorEagerSerializer() throws Exception { @SuppressWarnings("unchecked") - ReduceFunction reducer = mock(ReduceFunction.class); - + ReduceFunction reducer = mock(ReduceFunction.class); + TypeSerializer serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); - - ReducingStateDescriptor descr = - new ReducingStateDescriptor("testName", reducer, serializer); - + + ReducingStateDescriptor descr = + new ReducingStateDescriptor<>("testName", reducer, serializer); + assertEquals("testName", descr.getName()); assertNotNull(descr.getSerializer()); assertEquals(serializer, descr.getSerializer()); @@ -70,13 +72,13 @@ public void testValueStateDescriptorLazySerializer() throws Exception { @SuppressWarnings("unchecked") ReduceFunction reducer = mock(ReduceFunction.class); - + // some different registered value ExecutionConfig cfg = new ExecutionConfig(); cfg.registerKryoType(TaskInfo.class); ReducingStateDescriptor descr = - new ReducingStateDescriptor("testName", reducer, Path.class); + new ReducingStateDescriptor<>("testName", reducer, Path.class); try { descr.getSerializer(); @@ -84,7 +86,7 @@ public void testValueStateDescriptorLazySerializer() throws Exception { } catch (IllegalStateException ignored) {} descr.initializeSerializerUnlessSet(cfg); - + assertNotNull(descr.getSerializer()); assertTrue(descr.getSerializer() instanceof KryoSerializer); @@ -98,7 +100,7 @@ public void testValueStateDescriptorAutoSerializer() throws Exception { ReduceFunction reducer = mock(ReduceFunction.class); ReducingStateDescriptor descr = - new ReducingStateDescriptor("testName", reducer, String.class); + new ReducingStateDescriptor<>("testName", reducer, String.class); ReducingStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); @@ -108,9 +110,9 @@ public void testValueStateDescriptorAutoSerializer() throws Exception { } /** - * FLINK-6775 + * FLINK-6775. * - * Tests that the returned serializer is duplicated. This allows to + *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ @SuppressWarnings("unchecked") @@ -134,5 +136,5 @@ public TypeSerializer answer(InvocationOnMock invocation) throws Throwab // check that the retrieved serializers are not the same assertNotSame(serializerA, serializerB); } - + } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java index e434e011da9e31..b43e5ad16345f5 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java @@ -26,8 +26,8 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; - import org.apache.flink.util.TestLogger; + import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -42,17 +42,20 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +/** + * Tests for the {@link ValueStateDescriptor}. + */ public class ValueStateDescriptorTest extends TestLogger { - + @Test public void testValueStateDescriptorEagerSerializer() throws Exception { TypeSerializer serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); String defaultValue = "le-value-default"; - - ValueStateDescriptor descr = - new ValueStateDescriptor("testName", serializer, defaultValue); - + + ValueStateDescriptor descr = + new ValueStateDescriptor<>("testName", serializer, defaultValue); + assertEquals("testName", descr.getName()); assertEquals(defaultValue, descr.getDefaultValue()); assertNotNull(descr.getSerializer()); @@ -68,16 +71,16 @@ public void testValueStateDescriptorEagerSerializer() throws Exception { @Test public void testValueStateDescriptorLazySerializer() throws Exception { - + // some default value that goes to the generic serializer Path defaultValue = new Path(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).toURI()); - + // some different registered value ExecutionConfig cfg = new ExecutionConfig(); cfg.registerKryoType(TaskInfo.class); ValueStateDescriptor descr = - new ValueStateDescriptor("testName", Path.class, defaultValue); + new ValueStateDescriptor<>("testName", Path.class, defaultValue); try { descr.getSerializer(); @@ -85,7 +88,7 @@ public void testValueStateDescriptorLazySerializer() throws Exception { } catch (IllegalStateException ignored) {} descr.initializeSerializerUnlessSet(cfg); - + assertNotNull(descr.getSerializer()); assertTrue(descr.getSerializer() instanceof KryoSerializer); @@ -94,11 +97,11 @@ public void testValueStateDescriptorLazySerializer() throws Exception { @Test public void testValueStateDescriptorAutoSerializer() throws Exception { - + String defaultValue = "le-value-default"; ValueStateDescriptor descr = - new ValueStateDescriptor("testName", String.class, defaultValue); + new ValueStateDescriptor<>("testName", String.class, defaultValue); ValueStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); @@ -122,7 +125,7 @@ public void testVeryLargeDefaultValue() throws Exception { String defaultValue = new String(data, ConfigConstants.DEFAULT_CHARSET); ValueStateDescriptor descr = - new ValueStateDescriptor("testName", serializer, defaultValue); + new ValueStateDescriptor<>("testName", serializer, defaultValue); assertEquals("testName", descr.getName()); assertEquals(defaultValue, descr.getDefaultValue()); @@ -138,9 +141,9 @@ public void testVeryLargeDefaultValue() throws Exception { } /** - * FLINK-6775 + * FLINK-6775. * - * Tests that the returned serializer is duplicated. This allows to + *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ @SuppressWarnings("unchecked") diff --git a/tools/maven/suppressions-core.xml b/tools/maven/suppressions-core.xml index e613fb0cfe3cca..ff9c2038583d06 100644 --- a/tools/maven/suppressions-core.xml +++ b/tools/maven/suppressions-core.xml @@ -71,10 +71,6 @@ under the License. files="(.*)test[/\\](.*)api[/\\]common[/\\]operators[/\\](.*)" checks="AvoidStarImport"/> - - From e0bc37bef69f5376d03214578e9b95816add661b Mon Sep 17 00:00:00 2001 From: vinoyang Date: Sat, 24 Feb 2018 14:50:55 +0800 Subject: [PATCH 0211/2294] [FLINK-8756][Client] Support ClusterClient.getAccumulators() in RestClusterClient This closes #5573. --- .../program/rest/RestClusterClient.java | 41 ++++++++++- .../program/rest/RestClusterClientTest.java | 68 +++++++++++++++++++ .../handler/job/JobAccumulatorsHandler.java | 35 +++++++--- ...sIncludeSerializedValueQueryParameter.java | 41 +++++++++++ .../rest/messages/JobAccumulatorsHeaders.java | 6 +- .../rest/messages/JobAccumulatorsInfo.java | 46 ++++++++++++- .../JobAccumulatorsMessageParameters.java | 36 ++++++++++ .../json/SerializedValueDeserializer.java | 6 ++ .../json/SerializedValueSerializer.java | 6 ++ .../messages/JobAccumulatorsInfoTest.java | 2 +- 10 files changed, 273 insertions(+), 14 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/AccumulatorsIncludeSerializedValueQueryParameter.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsMessageParameters.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 5558461dbf6097..f3f196182011e0 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobSubmissionResult; +import org.apache.flink.api.common.accumulators.AccumulatorHelper; import org.apache.flink.api.common.time.Time; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.ProgramInvocationException; @@ -52,6 +53,9 @@ import org.apache.flink.runtime.rest.messages.EmptyMessageParameters; import org.apache.flink.runtime.rest.messages.EmptyRequestBody; import org.apache.flink.runtime.rest.messages.EmptyResponseBody; +import org.apache.flink.runtime.rest.messages.JobAccumulatorsHeaders; +import org.apache.flink.runtime.rest.messages.JobAccumulatorsInfo; +import org.apache.flink.runtime.rest.messages.JobAccumulatorsMessageParameters; import org.apache.flink.runtime.rest.messages.JobMessageParameters; import org.apache.flink.runtime.rest.messages.JobTerminationHeaders; import org.apache.flink.runtime.rest.messages.JobTerminationMessageParameters; @@ -101,6 +105,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -403,6 +408,40 @@ private CompletableFuture triggerSavepoint( }); } + @Override + public Map getAccumulators(final JobID jobID, ClassLoader loader) throws Exception { + final JobAccumulatorsHeaders accumulatorsHeaders = JobAccumulatorsHeaders.getInstance(); + final JobAccumulatorsMessageParameters accMsgParams = accumulatorsHeaders.getUnresolvedMessageParameters(); + accMsgParams.jobPathParameter.resolve(jobID); + accMsgParams.includeSerializedAccumulatorsParameter.resolve(Collections.singletonList(true)); + + CompletableFuture responseFuture = sendRequest( + accumulatorsHeaders, + accMsgParams + ); + + Map result = Collections.emptyMap(); + + try { + result = responseFuture.thenApply((JobAccumulatorsInfo accumulatorsInfo) -> { + try { + return AccumulatorHelper.deserializeAccumulators( + accumulatorsInfo.getSerializedUserAccumulators(), + loader); + } catch (Exception e) { + throw new CompletionException( + new FlinkException( + String.format("Deserialization of accumulators for job %s failed.", jobID), + e)); + } + }).get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (ExecutionException ee) { + ExceptionUtils.rethrowException(ExceptionUtils.stripExecutionException(ee)); + } + + return result; + } + private CompletableFuture pollSavepointAsync( final JobID jobId, final TriggerId triggerID) { @@ -661,7 +700,7 @@ private CompletableFuture getDispatcherAddress() { TimeUnit.MILLISECONDS) .thenApplyAsync(leaderAddressSessionId -> { final String address = leaderAddressSessionId.f0; - final Optional host = ScalaUtils.toJava(AddressFromURIString.parse(address).host()); + final Optional host = ScalaUtils.toJava(AddressFromURIString.parse(address).host()); return host.orElseGet(() -> { // if the dispatcher address does not contain a host part, then assume it's running diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index ca2ba223ebeefa..e108a0b116eb3b 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -47,11 +47,15 @@ import org.apache.flink.runtime.rest.handler.RestHandlerSpecification; import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationResult; import org.apache.flink.runtime.rest.handler.async.TriggerResponse; +import org.apache.flink.runtime.rest.messages.AccumulatorsIncludeSerializedValueQueryParameter; import org.apache.flink.runtime.rest.messages.BlobServerPortHeaders; import org.apache.flink.runtime.rest.messages.BlobServerPortResponseBody; import org.apache.flink.runtime.rest.messages.EmptyMessageParameters; import org.apache.flink.runtime.rest.messages.EmptyRequestBody; import org.apache.flink.runtime.rest.messages.EmptyResponseBody; +import org.apache.flink.runtime.rest.messages.JobAccumulatorsHeaders; +import org.apache.flink.runtime.rest.messages.JobAccumulatorsInfo; +import org.apache.flink.runtime.rest.messages.JobAccumulatorsMessageParameters; import org.apache.flink.runtime.rest.messages.JobMessageParameters; import org.apache.flink.runtime.rest.messages.JobTerminationHeaders; import org.apache.flink.runtime.rest.messages.JobTerminationMessageParameters; @@ -102,8 +106,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -118,6 +124,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -542,6 +549,67 @@ public void testListJobs() throws Exception { } } + @Test + public void testGetAccumulators() throws Exception { + TestAccumulatorHandler accumulatorHandler = new TestAccumulatorHandler(); + + try (TestRestServerEndpoint ignored = createRestServerEndpoint(accumulatorHandler)){ + + JobID id = new JobID(); + + { + Map accumulators = restClusterClient.getAccumulators(id); + assertNotNull(accumulators); + assertEquals(1, accumulators.size()); + + assertEquals(true, accumulators.containsKey("testKey")); + assertEquals("testValue", accumulators.get("testKey").toString()); + } + } + } + + private class TestAccumulatorHandler extends TestHandler { + + public TestAccumulatorHandler() { + super(JobAccumulatorsHeaders.getInstance()); + } + + @Override + protected CompletableFuture handleRequest( + @Nonnull HandlerRequest request, + @Nonnull DispatcherGateway gateway) throws RestHandlerException { + JobAccumulatorsInfo accumulatorsInfo; + List queryParams = request.getQueryParameter(AccumulatorsIncludeSerializedValueQueryParameter.class); + + final boolean includeSerializedValue; + if (!queryParams.isEmpty()) { + includeSerializedValue = queryParams.get(0); + } else { + includeSerializedValue = false; + } + + List userTaskAccumulators = new ArrayList<>(1); + + userTaskAccumulators.add(new JobAccumulatorsInfo.UserTaskAccumulator("testName", "testType", "testValue")); + + if (includeSerializedValue) { + Map> serializedUserTaskAccumulators = new HashMap<>(1); + try { + serializedUserTaskAccumulators.put("testKey", new SerializedValue<>("testValue")); + } catch (IOException e) { + throw new RuntimeException(e); + } + + accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators); + } else { + accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap()); + } + + return CompletableFuture.completedFuture(accumulatorsInfo); + } + } + private class TestListJobsHandler extends TestHandler { private TestListJobsHandler() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java index 7dd5ff07186904..0fe920171dcfa2 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java @@ -24,12 +24,14 @@ import org.apache.flink.runtime.rest.handler.HandlerRequest; import org.apache.flink.runtime.rest.handler.RestHandlerException; import org.apache.flink.runtime.rest.handler.legacy.ExecutionGraphCache; +import org.apache.flink.runtime.rest.messages.AccumulatorsIncludeSerializedValueQueryParameter; import org.apache.flink.runtime.rest.messages.EmptyRequestBody; import org.apache.flink.runtime.rest.messages.JobAccumulatorsInfo; -import org.apache.flink.runtime.rest.messages.JobMessageParameters; +import org.apache.flink.runtime.rest.messages.JobAccumulatorsMessageParameters; import org.apache.flink.runtime.rest.messages.MessageHeaders; import org.apache.flink.runtime.webmonitor.RestfulGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; +import org.apache.flink.util.SerializedValue; import java.util.ArrayList; import java.util.Collections; @@ -41,14 +43,14 @@ /** * Request handler that returns the aggregated accumulators of a job. */ -public class JobAccumulatorsHandler extends AbstractExecutionGraphHandler { +public class JobAccumulatorsHandler extends AbstractExecutionGraphHandler { public JobAccumulatorsHandler( CompletableFuture localRestAddress, GatewayRetriever leaderRetriever, Time timeout, Map responseHeaders, - MessageHeaders messageHeaders, + MessageHeaders messageHeaders, ExecutionGraphCache executionGraphCache, Executor executor) { super( @@ -62,11 +64,21 @@ public JobAccumulatorsHandler( } @Override - protected JobAccumulatorsInfo handleRequest(HandlerRequest request, AccessExecutionGraph graph) throws RestHandlerException { - StringifiedAccumulatorResult[] accs = graph.getAccumulatorResultsStringified(); - List userTaskAccumulators = new ArrayList<>(accs.length); + protected JobAccumulatorsInfo handleRequest(HandlerRequest request, AccessExecutionGraph graph) throws RestHandlerException { + JobAccumulatorsInfo accumulatorsInfo; + List queryParams = request.getQueryParameter(AccumulatorsIncludeSerializedValueQueryParameter.class); - for (StringifiedAccumulatorResult acc : accs) { + final boolean includeSerializedValue; + if (!queryParams.isEmpty()) { + includeSerializedValue = queryParams.get(0); + } else { + includeSerializedValue = false; + } + + StringifiedAccumulatorResult[] stringifiedAccs = graph.getAccumulatorResultsStringified(); + List userTaskAccumulators = new ArrayList<>(stringifiedAccs.length); + + for (StringifiedAccumulatorResult acc : stringifiedAccs) { userTaskAccumulators.add( new JobAccumulatorsInfo.UserTaskAccumulator( acc.getName(), @@ -74,6 +86,13 @@ protected JobAccumulatorsInfo handleRequest(HandlerRequest> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized(); + accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators); + } else { + accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap()); + } + + return accumulatorsInfo; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/AccumulatorsIncludeSerializedValueQueryParameter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/AccumulatorsIncludeSerializedValueQueryParameter.java new file mode 100644 index 00000000000000..1f685c215b5852 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/AccumulatorsIncludeSerializedValueQueryParameter.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.messages; + +/** + * Query parameter for job's accumulator handler {@link org.apache.flink.runtime.rest.handler.job.JobAccumulatorsHandler}. + */ +public class AccumulatorsIncludeSerializedValueQueryParameter extends MessageQueryParameter { + + private static final String key = "includeSerializedValue"; + + public AccumulatorsIncludeSerializedValueQueryParameter() { + super(key, MessageParameterRequisiteness.OPTIONAL); + } + + @Override + public String convertValueToString(Boolean value) { + return String.valueOf(value); + } + + @Override + public Boolean convertStringToValue(String value) { + return Boolean.valueOf(value); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsHeaders.java index 00f4fd5d67833c..2e00c91cbb2a52 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsHeaders.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsHeaders.java @@ -26,7 +26,7 @@ /** * Message headers for the {@link JobAccumulatorsHandler}. */ -public class JobAccumulatorsHeaders implements MessageHeaders { +public class JobAccumulatorsHeaders implements MessageHeaders { private static final JobAccumulatorsHeaders INSTANCE = new JobAccumulatorsHeaders(); @@ -53,8 +53,8 @@ public HttpResponseStatus getResponseStatusCode() { } @Override - public JobMessageParameters getUnresolvedMessageParameters() { - return new JobMessageParameters(); + public JobAccumulatorsMessageParameters getUnresolvedMessageParameters() { + return new JobAccumulatorsMessageParameters(); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java index 367a38bba87a12..22621204a7f9aa 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java @@ -19,12 +19,19 @@ package org.apache.flink.runtime.rest.messages; import org.apache.flink.runtime.rest.handler.job.JobAccumulatorsHandler; +import org.apache.flink.runtime.rest.messages.json.SerializedValueDeserializer; +import org.apache.flink.runtime.rest.messages.json.SerializedValueSerializer; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.SerializedValue; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -33,6 +40,7 @@ public class JobAccumulatorsInfo implements ResponseBody { public static final String FIELD_NAME_JOB_ACCUMULATORS = "job-accumulators"; public static final String FIELD_NAME_USER_TASK_ACCUMULATORS = "user-task-accumulators"; + public static final String FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS = "serialized-user-task-accumulators"; @JsonProperty(FIELD_NAME_JOB_ACCUMULATORS) private List jobAccumulators; @@ -40,12 +48,33 @@ public class JobAccumulatorsInfo implements ResponseBody { @JsonProperty(FIELD_NAME_USER_TASK_ACCUMULATORS) private List userAccumulators; + @JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS) + @JsonSerialize(contentUsing = SerializedValueSerializer.class) + private Map> serializedUserAccumulators; + @JsonCreator public JobAccumulatorsInfo( @JsonProperty(FIELD_NAME_JOB_ACCUMULATORS) List jobAccumulators, - @JsonProperty(FIELD_NAME_USER_TASK_ACCUMULATORS) List userAccumulators) { + @JsonProperty(FIELD_NAME_USER_TASK_ACCUMULATORS) List userAccumulators, + @JsonDeserialize(contentUsing = SerializedValueDeserializer.class) @JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS) Map> serializedUserAccumulators) { this.jobAccumulators = Preconditions.checkNotNull(jobAccumulators); this.userAccumulators = Preconditions.checkNotNull(userAccumulators); + this.serializedUserAccumulators = Preconditions.checkNotNull(serializedUserAccumulators); + } + + @JsonIgnore + public List getJobAccumulators() { + return jobAccumulators; + } + + @JsonIgnore + public List getUserAccumulators() { + return userAccumulators; + } + + @JsonIgnore + public Map> getSerializedUserAccumulators() { + return serializedUserAccumulators; } @Override @@ -104,6 +133,21 @@ public UserTaskAccumulator( this.value = Preconditions.checkNotNull(value); } + @JsonIgnore + public String getName() { + return name; + } + + @JsonIgnore + public String getType() { + return type; + } + + @JsonIgnore + public String getValue() { + return value; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsMessageParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsMessageParameters.java new file mode 100644 index 00000000000000..ef235601785a4d --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsMessageParameters.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.messages; + +import java.util.Collection; +import java.util.Collections; + +/** + * Request parameter for job accumulator's handler {@link org.apache.flink.runtime.rest.handler.job.JobAccumulatorsHandler}. + */ +public class JobAccumulatorsMessageParameters extends JobMessageParameters { + + public final AccumulatorsIncludeSerializedValueQueryParameter + includeSerializedAccumulatorsParameter = new AccumulatorsIncludeSerializedValueQueryParameter(); + + @Override + public Collection> getQueryParameters() { + return Collections.singleton(includeSerializedAccumulatorsParameter); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueDeserializer.java index 6a2eadb15ed3ec..d7c321dc8833f4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueDeserializer.java @@ -21,9 +21,11 @@ import org.apache.flink.util.SerializedValue; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParser; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.type.TypeReference; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationContext; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JavaType; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.type.TypeFactory; import java.io.IOException; @@ -34,6 +36,10 @@ public class SerializedValueDeserializer extends StdDeserializer>() {})); + } + public SerializedValueDeserializer(final JavaType valueType) { super(valueType); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueSerializer.java index 0383d99f56db81..b63b1ef9091916 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/SerializedValueSerializer.java @@ -21,9 +21,11 @@ import org.apache.flink.util.SerializedValue; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.type.TypeReference; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JavaType; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.SerializerProvider; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ser.std.StdSerializer; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.type.TypeFactory; import java.io.IOException; @@ -36,6 +38,10 @@ public class SerializedValueSerializer extends StdSerializer> private static final long serialVersionUID = 1L; + public SerializedValueSerializer() { + super(TypeFactory.defaultInstance().constructType(new TypeReference>() {})); + } + public SerializedValueSerializer(final JavaType javaType) { super(javaType); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfoTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfoTest.java index baaa551caef55a..e0e9649b503c10 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfoTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfoTest.java @@ -47,6 +47,6 @@ protected JobAccumulatorsInfo getTestResponseInstance() throws Exception { "uta3.type", "uta3.value")); - return new JobAccumulatorsInfo(Collections.emptyList(), userAccumulatorList); + return new JobAccumulatorsInfo(Collections.emptyList(), userAccumulatorList, Collections.EMPTY_MAP); } } From e273d5fea782399c6887fe7bdc169602a7026fcc Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Mon, 19 Mar 2018 09:48:15 +0100 Subject: [PATCH 0212/2294] [FLINK-8073][kafka-tests] Disable timeout in tests To get stacktraces in case of deadlock do not timeout tests programatically. This closes #5718. --- .../kafka/FlinkKafkaProducer011ITCase.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java index 81bf0bf2db4410..361f269691905b 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java @@ -80,7 +80,7 @@ public void before() { * This test ensures that transactions reusing transactional.ids (after returning to the pool) will not clash * with previous transactions using same transactional.ids. */ - @Test(timeout = 120_000L) + @Test public void testRestoreToCheckpointAfterExceedingProducersPool() throws Exception { String topic = "flink-kafka-producer-fail-before-notify"; @@ -123,7 +123,7 @@ public void testRestoreToCheckpointAfterExceedingProducersPool() throws Exceptio } } - @Test(timeout = 120_000L) + @Test public void testFlinkKafkaProducer011FailBeforeNotify() throws Exception { String topic = "flink-kafka-producer-fail-before-notify"; @@ -166,7 +166,7 @@ public void testFlinkKafkaProducer011FailBeforeNotify() throws Exception { deleteTestTopic(topic); } - @Test(timeout = 120_000L) + @Test public void testFlinkKafkaProducer011FailTransactionCoordinatorBeforeNotify() throws Exception { String topic = "flink-kafka-producer-fail-transaction-coordinator-before-notify"; @@ -221,7 +221,7 @@ public void testFlinkKafkaProducer011FailTransactionCoordinatorBeforeNotify() th * If such transactions were left alone lingering it consumers would be unable to read committed records * that were created after this lingering transaction. */ - @Test(timeout = 120_000L) + @Test public void testFailBeforeNotifyAndResumeWorkAfterwards() throws Exception { String topic = "flink-kafka-producer-fail-before-notify"; @@ -263,7 +263,7 @@ public void testFailBeforeNotifyAndResumeWorkAfterwards() throws Exception { deleteTestTopic(topic); } - @Test(timeout = 120_000L) + @Test public void testFailAndRecoverSameCheckpointTwice() throws Exception { String topic = "flink-kafka-producer-fail-and-recover-same-checkpoint-twice"; @@ -316,7 +316,7 @@ public void testFailAndRecoverSameCheckpointTwice() throws Exception { * If such transactions were left alone lingering it consumers would be unable to read committed records * that were created after this lingering transaction. */ - @Test(timeout = 120_000L) + @Test public void testScaleDownBeforeFirstCheckpoint() throws Exception { String topic = "scale-down-before-first-checkpoint"; @@ -381,7 +381,7 @@ public void testScaleDownBeforeFirstCheckpoint() throws Exception { * new subtask have to generate new id(s), but he can not use ids that are potentially in use, so it has to generate * new ones that are greater then 4. */ - @Test(timeout = 120_000L) + @Test public void testScaleUpAfterScalingDown() throws Exception { String topic = "scale-down-before-first-checkpoint"; From ea5342f8a912abe79ffcd83c8c352b070df343f7 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Tue, 20 Mar 2018 11:17:18 +0100 Subject: [PATCH 0213/2294] [hotfix][kafka-tests] Clean up and drop unused field in KafkaProducerTestBase --- .../connectors/kafka/KafkaProducerTestBase.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java index 8104d8fa0f783a..9278b67af05825 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java @@ -482,8 +482,7 @@ private static class BrokerRestartingMapper extends RichMapFunction private static final long serialVersionUID = 6334389850158707313L; - public static volatile boolean restartedLeaderBefore; - public static volatile boolean hasBeenCheckpointedBeforeFailure; + public static volatile boolean triggeredShutdown; public static volatile int numElementsBeforeSnapshot; public static volatile Runnable shutdownAction; @@ -491,11 +490,9 @@ private static class BrokerRestartingMapper extends RichMapFunction private int numElementsTotal; private boolean failer; - private boolean hasBeenCheckpointed; public static void resetState(Runnable shutdownAction) { - restartedLeaderBefore = false; - hasBeenCheckpointedBeforeFailure = false; + triggeredShutdown = false; numElementsBeforeSnapshot = 0; BrokerRestartingMapper.shutdownAction = shutdownAction; } @@ -513,13 +510,12 @@ public void open(Configuration parameters) { public T map(T value) throws Exception { numElementsTotal++; - if (!restartedLeaderBefore) { + if (!triggeredShutdown) { Thread.sleep(10); if (failer && numElementsTotal >= failCount) { // shut down a Kafka broker - hasBeenCheckpointedBeforeFailure = hasBeenCheckpointed; - restartedLeaderBefore = true; + triggeredShutdown = true; shutdownAction.run(); } } @@ -528,7 +524,6 @@ public T map(T value) throws Exception { @Override public void notifyCheckpointComplete(long checkpointId) { - hasBeenCheckpointed = true; } @Override From b87e660ac64bebbd9a0a6aa4334a68736140053f Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Tue, 20 Mar 2018 11:23:35 +0100 Subject: [PATCH 0214/2294] [FLINK-7343][kafka-tests] Fix test at-least-once test instability Previously we could set numElementsBeforeSnapshot to some value during checkpointing AFTER executing shutdown, while at the same time FlinkKafkaProducerXXX snapshot for this value would fail. This lead to incorrectly cacluated expected set of values to be present in the test kafka topic. Fix is to remember lastSnapshotedElementBeforeShutdown - last snapshot that we expect to succeed without failure. This closes #5729. --- .../kafka/Kafka09ProducerITCase.java | 11 --------- .../kafka/KafkaProducerTestBase.java | 23 +++++++++---------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java index c619c3e2e02c33..f145e56a425381 100644 --- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java +++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java @@ -32,15 +32,4 @@ public void testExactlyOnceRegularSink() throws Exception { public void testExactlyOnceCustomOperator() throws Exception { // Kafka09 does not support exactly once semantic } - - @Override - public void testOneToOneAtLeastOnceRegularSink() throws Exception { - // For some reasons this test is sometimes failing in Kafka09 while the same code works in Kafka010. Disabling - // this test because everything indicates those failures might be caused by unfixed bugs in Kafka 0.9 branch - } - - @Override - public void testOneToOneAtLeastOnceCustomOperator() throws Exception { - // Disable this test since FlinkKafka09Producer doesn't support custom operator mode - } } diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java index 9278b67af05825..5023a7eae719b4 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java @@ -292,7 +292,7 @@ public int partition(Integer record, byte[] key, byte[] value, String targetTopi properties, topic, partition, - Collections.unmodifiableSet(new HashSet<>(getIntegersSequence(BrokerRestartingMapper.numElementsBeforeSnapshot))), + Collections.unmodifiableSet(new HashSet<>(getIntegersSequence(BrokerRestartingMapper.lastSnapshotedElementBeforeShutdown))), KAFKA_READ_TIMEOUT); deleteTestTopic(topic); @@ -483,7 +483,7 @@ private static class BrokerRestartingMapper extends RichMapFunction private static final long serialVersionUID = 6334389850158707313L; public static volatile boolean triggeredShutdown; - public static volatile int numElementsBeforeSnapshot; + public static volatile int lastSnapshotedElementBeforeShutdown; public static volatile Runnable shutdownAction; private final int failCount; @@ -493,7 +493,7 @@ private static class BrokerRestartingMapper extends RichMapFunction public static void resetState(Runnable shutdownAction) { triggeredShutdown = false; - numElementsBeforeSnapshot = 0; + lastSnapshotedElementBeforeShutdown = 0; BrokerRestartingMapper.shutdownAction = shutdownAction; } @@ -509,15 +509,12 @@ public void open(Configuration parameters) { @Override public T map(T value) throws Exception { numElementsTotal++; + Thread.sleep(10); - if (!triggeredShutdown) { - Thread.sleep(10); - - if (failer && numElementsTotal >= failCount) { - // shut down a Kafka broker - triggeredShutdown = true; - shutdownAction.run(); - } + if (!triggeredShutdown && failer && numElementsTotal >= failCount) { + // shut down a Kafka broker + triggeredShutdown = true; + shutdownAction.run(); } return value; } @@ -528,7 +525,9 @@ public void notifyCheckpointComplete(long checkpointId) { @Override public void snapshotState(FunctionSnapshotContext context) throws Exception { - numElementsBeforeSnapshot = numElementsTotal; + if (!triggeredShutdown) { + lastSnapshotedElementBeforeShutdown = numElementsTotal; + } } @Override From 91707e35d39a1b9c11448f21eafc27f0fd949370 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Fri, 16 Mar 2018 13:28:08 +0100 Subject: [PATCH 0215/2294] [FLINK-8984][network] Drop taskmanager.exactly-once.blocking.data.enabled config option Previously there were twe options: taskmanager.network.credit-based-flow-control.enabled and taskmanager.exactly-once.blocking.data.enabled If we disabled first one, but keept default value for the second one deadlocks will occur. By dropping taskmanager.exactly-once.blocking.data.enabled we can always use: - blocking BarrierBuffer for credit based flow control - spilling BarrierBuffer for non credit based flow control. This closes #5708. --- .../flink/configuration/TaskManagerOptions.java | 12 ------------ .../streaming/runtime/io/InputProcessorUtil.java | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java index 4e08fdaa9910a2..c7b0782ba4f68c 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java @@ -326,18 +326,6 @@ public class TaskManagerOptions { .defaultValue(true) .withDescription("Boolean flag to enable/disable network credit-based flow control."); - /** - * Config parameter defining whether to spill data for channels with barrier or not in exactly-once - * mode based on credit-based flow control. - * - * @deprecated Will be removed for Flink 1.6 when the old code will be dropped in favour of - * credit-based flow control. - */ - @Deprecated - public static final ConfigOption EXACTLY_ONCE_BLOCKING_DATA_ENABLED = - key("taskmanager.exactly-once.blocking.data.enabled") - .defaultValue(true); - // ------------------------------------------------------------------------ // Task Options // ------------------------------------------------------------------------ diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/InputProcessorUtil.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/InputProcessorUtil.java index cb56eeefac47c2..1ae34b349772a8 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/InputProcessorUtil.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/InputProcessorUtil.java @@ -51,7 +51,7 @@ public static CheckpointBarrierHandler createCheckpointBarrierHandler( + " must be positive or -1 (infinite)"); } - if (taskManagerConfig.getBoolean(TaskManagerOptions.EXACTLY_ONCE_BLOCKING_DATA_ENABLED)) { + if (taskManagerConfig.getBoolean(TaskManagerOptions.NETWORK_CREDIT_BASED_FLOW_CONTROL_ENABLED)) { barrierHandler = new BarrierBuffer(inputGate, new CachedBufferBlocker(inputGate.getPageSize()), maxAlign); } else { barrierHandler = new BarrierBuffer(inputGate, new BufferSpiller(ioManager, inputGate.getPageSize()), maxAlign); From 129e215961895ef3c759b4fbb919a3be2ce2f1a0 Mon Sep 17 00:00:00 2001 From: Florian Schmidt Date: Mon, 19 Mar 2018 13:26:14 +0100 Subject: [PATCH 0216/2294] [FLINK-9020][E2ETests] Use separate modules per testcase This closes #5717. --- .../pom.xml | 108 ++++++++++++++++++ .../runtime/taskmanager/TaskManager.java | 0 .../tests/ClassLoaderTestProgram.java | 0 .../src/main/resources/.version.properties | 0 flink-end-to-end-tests/pom.xml | 78 +------------ .../test_streaming_classloader.sh | 4 +- 6 files changed, 116 insertions(+), 74 deletions(-) create mode 100644 flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml rename flink-end-to-end-tests/{ => flink-parent-child-classloading-test}/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java (100%) rename flink-end-to-end-tests/{ => flink-parent-child-classloading-test}/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java (100%) rename flink-end-to-end-tests/{ => flink-parent-child-classloading-test}/src/main/resources/.version.properties (100%) diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml b/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml new file mode 100644 index 00000000000000..ee43515097fbf5 --- /dev/null +++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml @@ -0,0 +1,108 @@ + + + + + + flink-end-to-end-tests + org.apache.flink + 1.6-SNAPSHOT + .. + + + 4.0.0 + + flink-parent-child-classloading-test_${scala.binary.version} + flink-parent-child-classloading-test + jar + + + + org.apache.flink + flink-core + ${project.version} + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + + ClassLoaderTestProgram + package + + jar + + + ClassLoaderTestProgram + + + + org.apache.flink.streaming.tests.ClassLoaderTestProgram + + + + + org/apache/flink/streaming/tests/ClassLoaderTestProgram.class + org/apache/flink/runtime/taskmanager/TaskManager.class + .version.properties + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + rename + package + + run + + + + + + + + + + + + + + diff --git a/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java similarity index 100% rename from flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java rename to flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java diff --git a/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java similarity index 100% rename from flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java rename to flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java diff --git a/flink-end-to-end-tests/src/main/resources/.version.properties b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties similarity index 100% rename from flink-end-to-end-tests/src/main/resources/.version.properties rename to flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml index a5bbc52ff462b3..32a6e78ae50c3e 100644 --- a/flink-end-to-end-tests/pom.xml +++ b/flink-end-to-end-tests/pom.xml @@ -29,79 +29,13 @@ under the License. .. - flink-end-to-end-tests_${scala.binary.version} - flink-end-to-end-tests - - jar - - - - org.apache.flink - flink-core - ${project.version} - - - org.apache.flink - flink-streaming-java_${scala.binary.version} - ${project.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 2.4 + pom - - - - ClassLoaderTestProgram - package - - jar - - - ClassLoaderTestProgram - - - - org.apache.flink.streaming.tests.ClassLoaderTestProgram - - - - - org/apache/flink/streaming/tests/ClassLoaderTestProgram.class - org/apache/flink/runtime/taskmanager/TaskManager.class - .version.properties - - - - - + flink-end-to-end-tests + flink-end-to-end-tests - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - rename - package - - run - - - - - - - - - - - + + flink-parent-child-classloading-test + diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh index 95c58f8a1f584b..34c55f70ccac4a 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh @@ -19,7 +19,7 @@ source "$(dirname "$0")"/common.sh -TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/target/ClassLoaderTestProgram.jar +TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/flink-parent-child-classloading-test/target/ClassLoaderTestProgram.jar echo "Testing parent-first class loading" @@ -111,4 +111,4 @@ if [[ "$OUTPUT" != "$EXPECTED" ]]; then echo -e "EXPECTED: $EXPECTED" echo -e "ACTUAL: $OUTPUT" PASS="" -fi \ No newline at end of file +fi From 7e43f81866d3df8e839c9e01e71d6edca06bc8cd Mon Sep 17 00:00:00 2001 From: Gabor Gevay Date: Tue, 13 Feb 2018 18:09:05 +0100 Subject: [PATCH 0217/2294] [FLINK-8649] [scala api] Pass on TypeInfo in StreamExecutionEnvironment.createInput This closes #5478. --- .../streaming/api/scala/StreamExecutionEnvironment.scala | 7 ++++++- .../apache/flink/streaming/api/scala/DataStreamTest.scala | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala index cd96dbf8ea044c..9410a95dea2d30 100644 --- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala +++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala @@ -23,6 +23,7 @@ import org.apache.flink.annotation.{Internal, Public, PublicEvolving} import org.apache.flink.api.common.io.{FileInputFormat, FilePathFilter, InputFormat} import org.apache.flink.api.common.restartstrategy.RestartStrategies.RestartStrategyConfiguration import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.ResultTypeQueryable import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer import org.apache.flink.api.scala.ClosureCleaner import org.apache.flink.configuration.Configuration @@ -594,7 +595,11 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) { */ @PublicEvolving def createInput[T: TypeInformation](inputFormat: InputFormat[T, _]): DataStream[T] = - asScalaStream(javaEnv.createInput(inputFormat)) + if (inputFormat.isInstanceOf[ResultTypeQueryable[_]]) { + asScalaStream(javaEnv.createInput(inputFormat)) + } else { + asScalaStream(javaEnv.createInput(inputFormat, implicitly[TypeInformation[T]])) + } /** * Create a DataStream using a user defined source function for arbitrary diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala index 51ec5e382307ae..9e1c49393654a9 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/DataStreamTest.scala @@ -21,6 +21,7 @@ package org.apache.flink.streaming.api.scala import java.lang import org.apache.flink.api.common.functions._ +import org.apache.flink.api.java.io.ParallelIteratorInputFormat import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.api.collector.selector.OutputSelector import org.apache.flink.streaming.api.functions.{KeyedProcessFunction, ProcessFunction} @@ -673,6 +674,12 @@ class DataStreamTest extends AbstractTestBase { assert(sg.getIterationSourceSinkPairs.size() == 2) } + @Test + def testCreateInputPassesOnTypeInfo(): Unit = { + StreamExecutionEnvironment.getExecutionEnvironment.createInput[Tuple1[Integer]]( + new ParallelIteratorInputFormat[Tuple1[Integer]](null)) + } + ///////////////////////////////////////////////////////////// // Utilities ///////////////////////////////////////////////////////////// From 00b73ef79434b4adb302c6b9126a5c24825db121 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 21 Mar 2018 20:59:05 +0100 Subject: [PATCH 0218/2294] Revert "[FLINK-8703][tests] Port AutoParallelismITCase to flip6" The test does not actually run on Flip6, see FLINK-8813. --- .../test/misc/AutoParallelismITCase.java | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java index c25dbf060db7d0..9cafee624f0a26 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/misc/AutoParallelismITCase.java @@ -22,15 +22,17 @@ import org.apache.flink.api.common.functions.RichMapPartitionFunction; import org.apache.flink.api.common.io.GenericInputFormat; import org.apache.flink.api.java.DataSet; -import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.io.LocalCollectionOutputFormat; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.io.GenericInputSplit; -import org.apache.flink.test.util.MiniClusterResource; +import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; +import org.apache.flink.test.util.TestEnvironment; import org.apache.flink.util.Collector; import org.apache.flink.util.TestLogger; -import org.junit.ClassRule; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; @@ -50,17 +52,37 @@ public class AutoParallelismITCase extends TestLogger { private static final int SLOTS_PER_TM = 7; private static final int PARALLELISM = NUM_TM * SLOTS_PER_TM; - @ClassRule - public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( - new MiniClusterResource.MiniClusterResourceConfiguration( - new Configuration(), - 2, - 7)); + private static LocalFlinkMiniCluster cluster; + + private static TestEnvironment env; + + @BeforeClass + public static void setupCluster() { + Configuration config = new Configuration(); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TM); + config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, SLOTS_PER_TM); + cluster = new LocalFlinkMiniCluster(config, false); + + cluster.start(); + + env = new TestEnvironment(cluster, NUM_TM * SLOTS_PER_TM, false); + } + + @AfterClass + public static void teardownCluster() { + try { + cluster.stop(); + } + catch (Throwable t) { + System.err.println("Error stopping cluster on shutdown"); + t.printStackTrace(); + fail("ClusterClient shutdown caused an exception: " + t.getMessage()); + } + } @Test public void testProgramWithAutoParallelism() { try { - ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(ExecutionConfig.PARALLELISM_AUTO_MAX); env.getConfig().disableSysoutLogging(); From 893fabf6910b3e2ca55ba2b496f0c9530b35ef28 Mon Sep 17 00:00:00 2001 From: zentol Date: Wed, 21 Mar 2018 15:01:55 +0100 Subject: [PATCH 0219/2294] [FLINK-8925][tests] Enable flip6 on travis --- .travis.yml | 20 +++++--------------- pom.xml | 7 +++++-- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index f84b8d8b267d73..f86bbef1638c0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,27 +39,27 @@ matrix: - jdk: "oraclejdk8" env: - TEST="core" - - PROFILE="-Dhadoop.version=2.8.3" + - PROFILE="-Dhadoop.version=2.8.3 -Dflip6" - CACHE_NAME=JDK8_H280_CO - jdk: "oraclejdk8" env: - TEST="libraries" - - PROFILE="-Dhadoop.version=2.8.3" + - PROFILE="-Dhadoop.version=2.8.3 -Dflip6" - CACHE_NAME=JDK8_H280_L - jdk: "oraclejdk8" env: - TEST="connectors" - - PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis" + - PROFILE="-Dhadoop.version=2.8.3 -Dflip6 -Pinclude-kinesis" - CACHE_NAME=JDK8_H280_CN - jdk: "oraclejdk8" env: - TEST="tests" - - PROFILE="-Dhadoop.version=2.8.3" + - PROFILE="-Dhadoop.version=2.8.3 -Dflip6" - CACHE_NAME=JDK8_H280_T - jdk: "oraclejdk8" env: - TEST="misc" - - PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws" + - PROFILE="-Dhadoop.version=2.8.3 -Dflip6 -Dinclude_hadoop_aws" - CACHE_NAME=JDK8_H280_M - jdk: "openjdk8" env: @@ -86,16 +86,6 @@ matrix: - TEST="misc" - PROFILE="-Dhadoop.version=2.4.1" - CACHE_NAME=JDK8_H241_M - - jdk: "oraclejdk8" - env: - - TEST="core" - - PROFILE="-Dhadoop.version=2.8.0 -Pflip6" - - CACHE_NAME=JDK8_H280_F6_CO - - jdk: "oraclejdk8" - env: - - TEST="tests" - - PROFILE="-Dhadoop.version=2.8.0 -Pflip6" - - CACHE_NAME=JDK8_H280_F6_T git: depth: 100 diff --git a/pom.xml b/pom.xml index d83c81b2684a98..b21a2a91732c99 100644 --- a/pom.xml +++ b/pom.xml @@ -128,7 +128,6 @@ under the License. 1.3 false - org.apache.flink.testutils.category.Flip6 old flip6 From 8c042e378b65504c7d76302d508f1e33b2cfa524 Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Thu, 15 Mar 2018 21:04:00 +0100 Subject: [PATCH 0220/2294] [FLINK-8903] [table] Fix VAR_SAMP, VAR_POP, STDEV_SAMP, STDEV_POP functions on GROUP BY windows. This closes #5706. --- .../rules/AggregateReduceFunctionsRule.java | 602 ++++++++++++++++++ .../nodes/logical/FlinkLogicalAggregate.scala | 9 +- .../logical/FlinkLogicalWindowAggregate.scala | 17 + .../table/plan/rules/FlinkRuleSets.scala | 1 + .../WindowAggregateReduceFunctionsRule.scala | 75 +++ .../runtime/aggregate/AggregateUtil.scala | 4 +- .../table/api/batch/sql/GroupWindowTest.scala | 49 ++ .../api/batch/table/GroupWindowTest.scala | 45 ++ .../api/stream/sql/GroupWindowTest.scala | 46 ++ .../api/stream/table/GroupWindowTest.scala | 45 ++ 10 files changed, 888 insertions(+), 5 deletions(-) create mode 100644 flink-libraries/flink-table/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowAggregateReduceFunctionsRule.scala diff --git a/flink-libraries/flink-table/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java b/flink-libraries/flink-table/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java new file mode 100644 index 00000000000000..ce466e199c2625 --- /dev/null +++ b/flink-libraries/flink-table/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java @@ -0,0 +1,602 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.CompositeList; +import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Util; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/* + * THIS FILE HAS BEEN COPIED FROM THE APACHE CALCITE PROJECT TO MAKE IT MORE EXTENSIBLE. + * + * We have opened an issue to port this change to Apache Calcite (CALCITE-2216). + * Once CALCITE-2216 is fixed and included in a release, we can remove the copied class. + * + * Modification: + * - Added newCalcRel() method to be able to add fields to the projection. + */ + +/** + * Planner rule that reduces aggregate functions in + * {@link org.apache.calcite.rel.core.Aggregate}s to simpler forms. + * + *

    Rewrites: + *

      + * + *
    • AVG(x) → SUM(x) / COUNT(x) + * + *
    • STDDEV_POP(x) → SQRT( + * (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + * / COUNT(x)) + * + *
    • STDDEV_SAMP(x) → SQRT( + * (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + * / CASE COUNT(x) WHEN 1 THEN NULL ELSE COUNT(x) - 1 END) + * + *
    • VAR_POP(x) → (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + * / COUNT(x) + * + *
    • VAR_SAMP(x) → (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + * / CASE COUNT(x) WHEN 1 THEN NULL ELSE COUNT(x) - 1 END + *
    + * + *

    Since many of these rewrites introduce multiple occurrences of simpler + * forms like {@code COUNT(x)}, the rule gathers common sub-expressions as it + * goes. + */ +public class AggregateReduceFunctionsRule extends RelOptRule { + //~ Static fields/initializers --------------------------------------------- + + /** The singleton. */ + public static final AggregateReduceFunctionsRule INSTANCE = + new AggregateReduceFunctionsRule(operand(LogicalAggregate.class, any()), + RelFactories.LOGICAL_BUILDER); + + //~ Constructors ----------------------------------------------------------- + + /** Creates an AggregateReduceFunctionsRule. */ + public AggregateReduceFunctionsRule(RelOptRuleOperand operand, + RelBuilderFactory relBuilderFactory) { + super(operand, relBuilderFactory, null); + } + + //~ Methods ---------------------------------------------------------------- + + @Override public boolean matches(RelOptRuleCall call) { + if (!super.matches(call)) { + return false; + } + Aggregate oldAggRel = (Aggregate) call.rels[0]; + return containsAvgStddevVarCall(oldAggRel.getAggCallList()); + } + + public void onMatch(RelOptRuleCall ruleCall) { + Aggregate oldAggRel = (Aggregate) ruleCall.rels[0]; + reduceAggs(ruleCall, oldAggRel); + } + + /** + * Returns whether any of the aggregates are calls to AVG, STDDEV_*, VAR_*. + * + * @param aggCallList List of aggregate calls + */ + private boolean containsAvgStddevVarCall(List aggCallList) { + for (AggregateCall call : aggCallList) { + if (isReducible(call.getAggregation().getKind())) { + return true; + } + } + return false; + } + + /** + * Returns whether the aggregate call is a reducible function + */ + private boolean isReducible(final SqlKind kind) { + if (SqlKind.AVG_AGG_FUNCTIONS.contains(kind)) { + return true; + } + switch (kind) { + case SUM: + return true; + } + return false; + } + + /** + * Reduces all calls to AVG, STDDEV_POP, STDDEV_SAMP, VAR_POP, VAR_SAMP in + * the aggregates list to. + * + *

    It handles newly generated common subexpressions since this was done + * at the sql2rel stage. + */ + private void reduceAggs( + RelOptRuleCall ruleCall, + Aggregate oldAggRel) { + RexBuilder rexBuilder = oldAggRel.getCluster().getRexBuilder(); + + List oldCalls = oldAggRel.getAggCallList(); + final int groupCount = oldAggRel.getGroupCount(); + final int indicatorCount = oldAggRel.getIndicatorCount(); + + final List newCalls = Lists.newArrayList(); + final Map aggCallMapping = Maps.newHashMap(); + + final List projList = Lists.newArrayList(); + + // pass through group key (+ indicators if present) + for (int i = 0; i < groupCount + indicatorCount; ++i) { + projList.add( + rexBuilder.makeInputRef( + getFieldType(oldAggRel, i), + i)); + } + + // List of input expressions. If a particular aggregate needs more, it + // will add an expression to the end, and we will create an extra + // project. + final RelBuilder relBuilder = ruleCall.builder(); + relBuilder.push(oldAggRel.getInput()); + final List inputExprs = new ArrayList<>(relBuilder.fields()); + + // create new agg function calls and rest of project list together + for (AggregateCall oldCall : oldCalls) { + projList.add( + reduceAgg( + oldAggRel, oldCall, newCalls, aggCallMapping, inputExprs)); + } + + final int extraArgCount = + inputExprs.size() - relBuilder.peek().getRowType().getFieldCount(); + if (extraArgCount > 0) { + relBuilder.project(inputExprs, + CompositeList.of( + relBuilder.peek().getRowType().getFieldNames(), + Collections.nCopies(extraArgCount, null))); + } + newAggregateRel(relBuilder, oldAggRel, newCalls); + newCalcRel(relBuilder, oldAggRel, projList); + ruleCall.transformTo(relBuilder.build()); + } + + private RexNode reduceAgg( + Aggregate oldAggRel, + AggregateCall oldCall, + List newCalls, + Map aggCallMapping, + List inputExprs) { + final SqlKind kind = oldCall.getAggregation().getKind(); + if (isReducible(kind)) { + switch (kind) { + case SUM: + // replace original SUM(x) with + // case COUNT(x) when 0 then null else SUM0(x) end + return reduceSum(oldAggRel, oldCall, newCalls, aggCallMapping); + case AVG: + // replace original AVG(x) with SUM(x) / COUNT(x) + return reduceAvg(oldAggRel, oldCall, newCalls, aggCallMapping, inputExprs); + case STDDEV_POP: + // replace original STDDEV_POP(x) with + // SQRT( + // (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + // / COUNT(x)) + return reduceStddev(oldAggRel, oldCall, true, true, newCalls, + aggCallMapping, inputExprs); + case STDDEV_SAMP: + // replace original STDDEV_POP(x) with + // SQRT( + // (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + // / CASE COUNT(x) WHEN 1 THEN NULL ELSE COUNT(x) - 1 END) + return reduceStddev(oldAggRel, oldCall, false, true, newCalls, + aggCallMapping, inputExprs); + case VAR_POP: + // replace original VAR_POP(x) with + // (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + // / COUNT(x) + return reduceStddev(oldAggRel, oldCall, true, false, newCalls, + aggCallMapping, inputExprs); + case VAR_SAMP: + // replace original VAR_POP(x) with + // (SUM(x * x) - SUM(x) * SUM(x) / COUNT(x)) + // / CASE COUNT(x) WHEN 1 THEN NULL ELSE COUNT(x) - 1 END + return reduceStddev(oldAggRel, oldCall, false, false, newCalls, + aggCallMapping, inputExprs); + default: + throw Util.unexpected(kind); + } + } else { + // anything else: preserve original call + RexBuilder rexBuilder = oldAggRel.getCluster().getRexBuilder(); + final int nGroups = oldAggRel.getGroupCount(); + List oldArgTypes = + SqlTypeUtil.projectTypes( + oldAggRel.getInput().getRowType(), oldCall.getArgList()); + return rexBuilder.addAggCall(oldCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + oldArgTypes); + } + } + + private AggregateCall createAggregateCallWithBinding( + RelDataTypeFactory typeFactory, + SqlAggFunction aggFunction, + RelDataType operandType, + Aggregate oldAggRel, + AggregateCall oldCall, + int argOrdinal) { + final Aggregate.AggCallBinding binding = + new Aggregate.AggCallBinding(typeFactory, aggFunction, + ImmutableList.of(operandType), oldAggRel.getGroupCount(), + oldCall.filterArg >= 0); + return AggregateCall.create(aggFunction, + oldCall.isDistinct(), + oldCall.isApproximate(), + ImmutableIntList.of(argOrdinal), + oldCall.filterArg, + aggFunction.inferReturnType(binding), + null); + } + + private RexNode reduceAvg( + Aggregate oldAggRel, + AggregateCall oldCall, + List newCalls, + Map aggCallMapping, + List inputExprs) { + final int nGroups = oldAggRel.getGroupCount(); + final RexBuilder rexBuilder = oldAggRel.getCluster().getRexBuilder(); + final int iAvgInput = oldCall.getArgList().get(0); + final RelDataType avgInputType = + getFieldType( + oldAggRel.getInput(), + iAvgInput); + final AggregateCall sumCall = + AggregateCall.create(SqlStdOperatorTable.SUM, + oldCall.isDistinct(), + oldCall.isApproximate(), + oldCall.getArgList(), + oldCall.filterArg, + oldAggRel.getGroupCount(), + oldAggRel.getInput(), + null, + null); + final AggregateCall countCall = + AggregateCall.create(SqlStdOperatorTable.COUNT, + oldCall.isDistinct(), + oldCall.isApproximate(), + oldCall.getArgList(), + oldCall.filterArg, + oldAggRel.getGroupCount(), + oldAggRel.getInput(), + null, + null); + + // NOTE: these references are with respect to the output + // of newAggRel + RexNode numeratorRef = + rexBuilder.addAggCall(sumCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + ImmutableList.of(avgInputType)); + final RexNode denominatorRef = + rexBuilder.addAggCall(countCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + ImmutableList.of(avgInputType)); + + final RelDataTypeFactory typeFactory = oldAggRel.getCluster().getTypeFactory(); + final RelDataType avgType = typeFactory.createTypeWithNullability( + oldCall.getType(), numeratorRef.getType().isNullable()); + numeratorRef = rexBuilder.ensureType(avgType, numeratorRef, true); + final RexNode divideRef = + rexBuilder.makeCall(SqlStdOperatorTable.DIVIDE, numeratorRef, denominatorRef); + return rexBuilder.makeCast(oldCall.getType(), divideRef); + } + + private RexNode reduceSum( + Aggregate oldAggRel, + AggregateCall oldCall, + List newCalls, + Map aggCallMapping) { + final int nGroups = oldAggRel.getGroupCount(); + RexBuilder rexBuilder = oldAggRel.getCluster().getRexBuilder(); + int arg = oldCall.getArgList().get(0); + RelDataType argType = + getFieldType( + oldAggRel.getInput(), + arg); + final AggregateCall sumZeroCall = + AggregateCall.create(SqlStdOperatorTable.SUM0, oldCall.isDistinct(), + oldCall.isApproximate(), oldCall.getArgList(), oldCall.filterArg, + oldAggRel.getGroupCount(), oldAggRel.getInput(), null, + oldCall.name); + final AggregateCall countCall = + AggregateCall.create(SqlStdOperatorTable.COUNT, + oldCall.isDistinct(), + oldCall.isApproximate(), + oldCall.getArgList(), + oldCall.filterArg, + oldAggRel.getGroupCount(), + oldAggRel, + null, + null); + + // NOTE: these references are with respect to the output + // of newAggRel + RexNode sumZeroRef = + rexBuilder.addAggCall(sumZeroCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + ImmutableList.of(argType)); + if (!oldCall.getType().isNullable()) { + // If SUM(x) is not nullable, the validator must have determined that + // nulls are impossible (because the group is never empty and x is never + // null). Therefore we translate to SUM0(x). + return sumZeroRef; + } + RexNode countRef = + rexBuilder.addAggCall(countCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + ImmutableList.of(argType)); + return rexBuilder.makeCall(SqlStdOperatorTable.CASE, + rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + countRef, rexBuilder.makeExactLiteral(BigDecimal.ZERO)), + rexBuilder.makeCast(sumZeroRef.getType(), rexBuilder.constantNull()), + sumZeroRef); + } + + private RexNode reduceStddev( + Aggregate oldAggRel, + AggregateCall oldCall, + boolean biased, + boolean sqrt, + List newCalls, + Map aggCallMapping, + List inputExprs) { + // stddev_pop(x) ==> + // power( + // (sum(x * x) - sum(x) * sum(x) / count(x)) + // / count(x), + // .5) + // + // stddev_samp(x) ==> + // power( + // (sum(x * x) - sum(x) * sum(x) / count(x)) + // / nullif(count(x) - 1, 0), + // .5) + final int nGroups = oldAggRel.getGroupCount(); + final RelOptCluster cluster = oldAggRel.getCluster(); + final RexBuilder rexBuilder = cluster.getRexBuilder(); + final RelDataTypeFactory typeFactory = cluster.getTypeFactory(); + + assert oldCall.getArgList().size() == 1 : oldCall.getArgList(); + final int argOrdinal = oldCall.getArgList().get(0); + final RelDataType argOrdinalType = getFieldType(oldAggRel.getInput(), argOrdinal); + final RelDataType oldCallType = + typeFactory.createTypeWithNullability(oldCall.getType(), + argOrdinalType.isNullable()); + + final RexNode argRef = + rexBuilder.ensureType(oldCallType, inputExprs.get(argOrdinal), true); + final int argRefOrdinal = lookupOrAdd(inputExprs, argRef); + + final RexNode argSquared = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, + argRef, argRef); + final int argSquaredOrdinal = lookupOrAdd(inputExprs, argSquared); + + final AggregateCall sumArgSquaredAggCall = + createAggregateCallWithBinding(typeFactory, SqlStdOperatorTable.SUM, + argSquared.getType(), oldAggRel, oldCall, argSquaredOrdinal); + + final RexNode sumArgSquared = + rexBuilder.addAggCall(sumArgSquaredAggCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + ImmutableList.of(sumArgSquaredAggCall.getType())); + + final AggregateCall sumArgAggCall = + AggregateCall.create(SqlStdOperatorTable.SUM, + oldCall.isDistinct(), + oldCall.isApproximate(), + ImmutableIntList.of(argOrdinal), + oldCall.filterArg, + oldAggRel.getGroupCount(), + oldAggRel.getInput(), + null, + null); + + final RexNode sumArg = + rexBuilder.addAggCall(sumArgAggCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + ImmutableList.of(sumArgAggCall.getType())); + final RexNode sumArgCast = rexBuilder.ensureType(oldCallType, sumArg, true); + final RexNode sumSquaredArg = + rexBuilder.makeCall( + SqlStdOperatorTable.MULTIPLY, sumArgCast, sumArgCast); + + final AggregateCall countArgAggCall = + AggregateCall.create(SqlStdOperatorTable.COUNT, + oldCall.isDistinct(), + oldCall.isApproximate(), + oldCall.getArgList(), + oldCall.filterArg, + oldAggRel.getGroupCount(), + oldAggRel, + null, + null); + + final RexNode countArg = + rexBuilder.addAggCall(countArgAggCall, + nGroups, + oldAggRel.indicator, + newCalls, + aggCallMapping, + ImmutableList.of(argOrdinalType)); + + final RexNode avgSumSquaredArg = + rexBuilder.makeCall( + SqlStdOperatorTable.DIVIDE, sumSquaredArg, countArg); + + final RexNode diff = + rexBuilder.makeCall( + SqlStdOperatorTable.MINUS, + sumArgSquared, avgSumSquaredArg); + + final RexNode denominator; + if (biased) { + denominator = countArg; + } else { + final RexLiteral one = + rexBuilder.makeExactLiteral(BigDecimal.ONE); + final RexNode nul = + rexBuilder.makeCast(countArg.getType(), rexBuilder.constantNull()); + final RexNode countMinusOne = + rexBuilder.makeCall( + SqlStdOperatorTable.MINUS, countArg, one); + final RexNode countEqOne = + rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, countArg, one); + denominator = + rexBuilder.makeCall( + SqlStdOperatorTable.CASE, + countEqOne, nul, countMinusOne); + } + + final RexNode div = + rexBuilder.makeCall( + SqlStdOperatorTable.DIVIDE, diff, denominator); + + RexNode result = div; + if (sqrt) { + final RexNode half = + rexBuilder.makeExactLiteral(new BigDecimal("0.5")); + result = + rexBuilder.makeCall( + SqlStdOperatorTable.POWER, div, half); + } + + return rexBuilder.makeCast( + oldCall.getType(), result); + } + + /** + * Finds the ordinal of an element in a list, or adds it. + * + * @param list List + * @param element Element to lookup or add + * @param Element type + * @return Ordinal of element in list + */ + private static int lookupOrAdd(List list, T element) { + int ordinal = list.indexOf(element); + if (ordinal == -1) { + ordinal = list.size(); + list.add(element); + } + return ordinal; + } + + /** + * Do a shallow clone of oldAggRel and update aggCalls. Could be refactored + * into Aggregate and subclasses - but it's only needed for some + * subclasses. + * + * @param relBuilder Builder of relational expressions; at the top of its + * stack is its input + * @param oldAggregate LogicalAggregate to clone. + * @param newCalls New list of AggregateCalls + */ + protected void newAggregateRel(RelBuilder relBuilder, + Aggregate oldAggregate, + List newCalls) { + relBuilder.aggregate( + relBuilder.groupKey(oldAggregate.getGroupSet(), + oldAggregate.getGroupSets()), + newCalls); + } + + /** + * Add a calc with the expressions to compute the original agg calls from the + * decomposed ones. + * + * @param relBuilder Builder of relational expressions; at the top of its + * stack is its input + * @param oldAggregate The original LogicalAggregate that is replaced. + * @param exprs The expressions to compute the original agg calls. + */ + protected void newCalcRel(RelBuilder relBuilder, + Aggregate oldAggregate, + List exprs) { + relBuilder.project(exprs, oldAggregate.getRowType().getFieldNames()); + } + + private RelDataType getFieldType(RelNode relNode, int i) { + final RelDataTypeField inputField = + relNode.getRowType().getFieldList().get(i); + return inputField.getType(); + } +} + +// End AggregateReduceFunctionsRule.java diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala index e1e93c7c583b9f..17b6f1b6f8756d 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalAggregate.scala @@ -30,7 +30,7 @@ import org.apache.calcite.sql.SqlKind import org.apache.calcite.util.ImmutableBitSet import org.apache.flink.table.plan.nodes.FlinkConventions -import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ class FlinkLogicalAggregate( cluster: RelOptCluster, @@ -74,8 +74,11 @@ private class FlinkLogicalAggregateConverter // we do not support these functions natively // they have to be converted using the AggregateReduceFunctionsRule - val supported = agg.getAggCallList.map(_.getAggregation.getKind).forall { - case SqlKind.STDDEV_POP | SqlKind.STDDEV_SAMP | SqlKind.VAR_POP | SqlKind.VAR_SAMP => false + val supported = agg.getAggCallList.asScala.map(_.getAggregation.getKind).forall { + // we support AVG + case SqlKind.AVG => true + // but none of the other AVG agg functions + case k if SqlKind.AVG_AGG_FUNCTIONS.contains(k) => false case _ => true } diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalWindowAggregate.scala index 3e605e895dceb2..f2576f4c853782 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalWindowAggregate.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalWindowAggregate.scala @@ -26,6 +26,7 @@ import org.apache.calcite.rel.convert.ConverterRule import org.apache.calcite.rel.core.{Aggregate, AggregateCall} import org.apache.calcite.rel.metadata.RelMetadataQuery import org.apache.calcite.rel.{RelNode, RelShuttle} +import org.apache.calcite.sql.SqlKind import org.apache.calcite.util.ImmutableBitSet import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty import org.apache.flink.table.calcite.FlinkTypeFactory @@ -33,6 +34,8 @@ import org.apache.flink.table.plan.logical.LogicalWindow import org.apache.flink.table.plan.logical.rel.LogicalWindowAggregate import org.apache.flink.table.plan.nodes.FlinkConventions +import scala.collection.JavaConverters._ + class FlinkLogicalWindowAggregate( window: LogicalWindow, namedProperties: Seq[NamedWindowProperty], @@ -103,6 +106,20 @@ class FlinkLogicalWindowAggregateConverter FlinkConventions.LOGICAL, "FlinkLogicalWindowAggregateConverter") { + override def matches(call: RelOptRuleCall): Boolean = { + val agg = call.rel(0).asInstanceOf[LogicalWindowAggregate] + + // we do not support these functions natively + // they have to be converted using the WindowAggregateReduceFunctionsRule + agg.getAggCallList.asScala.map(_.getAggregation.getKind).forall { + // we support AVG + case SqlKind.AVG => true + // but none of the other AVG agg functions + case k if SqlKind.AVG_AGG_FUNCTIONS.contains(k) => false + case _ => true + } + } + override def convert(rel: RelNode): RelNode = { val agg = rel.asInstanceOf[LogicalWindowAggregate] val traitSet = rel.getTraitSet.replace(FlinkConventions.LOGICAL) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala index d3ad2ac5654dc0..9f3b8e99ece61d 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala @@ -93,6 +93,7 @@ object FlinkRuleSets { // reduce aggregate functions like AVG, STDDEV_POP etc. AggregateReduceFunctionsRule.INSTANCE, + WindowAggregateReduceFunctionsRule.INSTANCE, // remove unnecessary sort rule SortRemoveRule.INSTANCE, diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowAggregateReduceFunctionsRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowAggregateReduceFunctionsRule.scala new file mode 100644 index 00000000000000..4ca2b335478d60 --- /dev/null +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowAggregateReduceFunctionsRule.scala @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.plan.rules.common + +import java.util + +import org.apache.calcite.plan.RelOptRule +import org.apache.calcite.rel.core.{Aggregate, AggregateCall, RelFactories} +import org.apache.calcite.rel.logical.LogicalAggregate +import org.apache.calcite.rel.rules.AggregateReduceFunctionsRule +import org.apache.calcite.rex.RexNode +import org.apache.calcite.tools.RelBuilder +import org.apache.flink.table.plan.logical.rel.LogicalWindowAggregate + +/** + * Rule to convert complex aggregation functions into simpler ones. + * Have a look at [[AggregateReduceFunctionsRule]] for details. + */ +class WindowAggregateReduceFunctionsRule extends AggregateReduceFunctionsRule( + RelOptRule.operand(classOf[LogicalWindowAggregate], RelOptRule.any()), + RelFactories.LOGICAL_BUILDER) { + + override def newAggregateRel( + relBuilder: RelBuilder, + oldAgg: Aggregate, + newCalls: util.List[AggregateCall]): Unit = { + + // create a LogicalAggregate with simpler aggregation functions + super.newAggregateRel(relBuilder, oldAgg, newCalls) + // pop LogicalAggregate from RelBuilder + val newAgg = relBuilder.build().asInstanceOf[LogicalAggregate] + + // create a new LogicalWindowAggregate (based on the new LogicalAggregate) and push it on the + // RelBuilder + val oldWindowAgg = oldAgg.asInstanceOf[LogicalWindowAggregate] + relBuilder.push(LogicalWindowAggregate.create( + oldWindowAgg.getWindow, + oldWindowAgg.getNamedProperties, + newAgg)) + } + + override def newCalcRel( + relBuilder: RelBuilder, + oldAgg: Aggregate, + exprs: util.List[RexNode]): Unit = { + + // add all named properties of the window to the selection + val oldWindowAgg = oldAgg.asInstanceOf[LogicalWindowAggregate] + oldWindowAgg.getNamedProperties.foreach(np => exprs.add(relBuilder.field(np.name))) + + // create a LogicalCalc that computes the complex aggregates and forwards the window properties + relBuilder.project(exprs, oldAgg.getRowType.getFieldNames) + } + +} + +object WindowAggregateReduceFunctionsRule { + val INSTANCE = new WindowAggregateReduceFunctionsRule +} diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala index df9b1c5520467c..ce0a9c96e336e1 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala @@ -1259,7 +1259,7 @@ object AggregateUtil { } } - case _: SqlAvgAggFunction => + case a: SqlAvgAggFunction if a.kind == SqlKind.AVG => aggregates(index) = sqlTypeName match { case TINYINT => new ByteAvgAggFunction @@ -1413,7 +1413,7 @@ object AggregateUtil { accTypes(index) = udagg.accType case unSupported: SqlAggFunction => - throw new TableException(s"unsupported Function: '${unSupported.getName}'") + throw new TableException(s"Unsupported Function: '${unSupported.getName}'") } } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala index 8d06bcd84db394..b1369e2a9289a3 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/GroupWindowTest.scala @@ -304,4 +304,53 @@ class GroupWindowTest extends TableTestBase { util.verifySql(sql, expected) } + + @Test + def testDecomposableAggFunctions() = { + val util = batchTestUtil() + util.addTable[(Int, String, Long, Timestamp)]("MyTable", 'a, 'b, 'c, 'rowtime) + + val sql = + "SELECT " + + " VAR_POP(c), VAR_SAMP(c), STDDEV_POP(c), STDDEV_SAMP(c), " + + " TUMBLE_START(rowtime, INTERVAL '15' MINUTE), " + + " TUMBLE_END(rowtime, INTERVAL '15' MINUTE)" + + "FROM MyTable " + + "GROUP BY TUMBLE(rowtime, INTERVAL '15' MINUTE)" + + val expected = + unaryNode( + "DataSetCalc", + unaryNode( + "DataSetWindowAggregate", + unaryNode( + "DataSetCalc", + batchTableNode(0), + term("select", "rowtime", "c", + "*(c, c) AS $f2", "*(c, c) AS $f3", "*(c, c) AS $f4", "*(c, c) AS $f5") + ), + term("window", TumblingGroupWindow('w$, 'rowtime, 900000.millis)), + term("select", + "SUM($f2) AS $f0", + "SUM(c) AS $f1", + "COUNT(c) AS $f2", + "SUM($f3) AS $f3", + "SUM($f4) AS $f4", + "SUM($f5) AS $f5", + "start('w$) AS w$start", + "end('w$) AS w$end", + "rowtime('w$) AS w$rowtime") + ), + term("select", + "CAST(/(-($f0, /(*($f1, $f1), $f2)), $f2)) AS EXPR$0", + "CAST(/(-($f3, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1)))) AS EXPR$1", + "CAST(POWER(/(-($f4, /(*($f1, $f1), $f2)), $f2), 0.5)) AS EXPR$2", + "CAST(POWER(/(-($f5, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1))), 0.5)) " + + "AS EXPR$3", + "CAST(w$start) AS EXPR$4", + "CAST(w$end) AS EXPR$5") + ) + + util.verifySql(sql, expected) + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala index ad44e09c68aad8..27c1d7f6c324ac 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/table/GroupWindowTest.scala @@ -449,4 +449,49 @@ class GroupWindowTest extends TableTestBase { util.verifyTable(windowedTable, expected) } + + @Test + def testDecomposableAggFunctions(): Unit = { + val util = batchTestUtil() + val table = util.addTable[(Long, Int, String, Long)]('rowtime, 'a, 'b, 'c) + + val windowedTable = table + .window(Tumble over 15.minutes on 'rowtime as 'w) + .groupBy('w) + .select('c.varPop, 'c.varSamp, 'c.stddevPop, 'c.stddevSamp, 'w.start, 'w.end) + + val expected = + unaryNode( + "DataSetCalc", + unaryNode( + "DataSetWindowAggregate", + unaryNode( + "DataSetCalc", + batchTableNode(0), + term("select", "c", "rowtime", + "*(c, c) AS $f2", "*(c, c) AS $f3", "*(c, c) AS $f4", "*(c, c) AS $f5") + ), + term("window", TumblingGroupWindow('w, 'rowtime, 900000.millis)), + term("select", + "SUM($f2) AS $f0", + "SUM(c) AS $f1", + "COUNT(c) AS $f2", + "SUM($f3) AS $f3", + "SUM($f4) AS $f4", + "SUM($f5) AS $f5", + "start('w) AS TMP_4", + "end('w) AS TMP_5") + ), + term("select", + "CAST(/(-($f0, /(*($f1, $f1), $f2)), $f2)) AS TMP_0", + "CAST(/(-($f3, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1)))) AS TMP_1", + "CAST(POWER(/(-($f4, /(*($f1, $f1), $f2)), $f2), 0.5)) AS TMP_2", + "CAST(POWER(/(-($f5, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1))), 0.5)) " + + "AS TMP_3", + "TMP_4", + "TMP_5") + ) + + util.verifyTable(windowedTable, expected) + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala index d7d5f1e07b9474..d29283456d9b5f 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/GroupWindowTest.scala @@ -260,4 +260,50 @@ class GroupWindowTest extends TableTestBase { streamUtil.verifySql(sql, expected) } + + @Test + def testDecomposableAggFunctions() = { + + val sql = + "SELECT " + + " VAR_POP(c), VAR_SAMP(c), STDDEV_POP(c), STDDEV_SAMP(c), " + + " TUMBLE_START(rowtime, INTERVAL '15' MINUTE), " + + " TUMBLE_END(rowtime, INTERVAL '15' MINUTE)" + + "FROM MyTable " + + "GROUP BY TUMBLE(rowtime, INTERVAL '15' MINUTE)" + val expected = + unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamGroupWindowAggregate", + unaryNode( + "DataStreamCalc", + streamTableNode(0), + term("select", "rowtime", "c", + "*(c, c) AS $f2", "*(c, c) AS $f3", "*(c, c) AS $f4", "*(c, c) AS $f5") + ), + term("window", TumblingGroupWindow('w$, 'rowtime, 900000.millis)), + term("select", + "SUM($f2) AS $f0", + "SUM(c) AS $f1", + "COUNT(c) AS $f2", + "SUM($f3) AS $f3", + "SUM($f4) AS $f4", + "SUM($f5) AS $f5", + "start('w$) AS w$start", + "end('w$) AS w$end", + "rowtime('w$) AS w$rowtime", + "proctime('w$) AS w$proctime") + ), + term("select", + "CAST(/(-($f0, /(*($f1, $f1), $f2)), $f2)) AS EXPR$0", + "CAST(/(-($f3, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1)))) AS EXPR$1", + "CAST(POWER(/(-($f4, /(*($f1, $f1), $f2)), $f2), 0.5)) AS EXPR$2", + "CAST(POWER(/(-($f5, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1))), 0.5)) " + + "AS EXPR$3", + "w$start AS EXPR$4", + "w$end AS EXPR$5") + ) + streamUtil.verifySql(sql, expected) + } } diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala index 260726ba495b99..a59ad8382a0edf 100644 --- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/GroupWindowTest.scala @@ -782,4 +782,49 @@ class GroupWindowTest extends TableTestBase { util.verifyTable(windowedTable, expected) } + + @Test + def testDecomposableAggFunctions(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Long, Int, String, Long)]('rowtime.rowtime, 'a, 'b, 'c) + + val windowedTable = table + .window(Tumble over 15.minutes on 'rowtime as 'w) + .groupBy('w) + .select('c.varPop, 'c.varSamp, 'c.stddevPop, 'c.stddevSamp, 'w.start, 'w.end) + + val expected = + unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamGroupWindowAggregate", + unaryNode( + "DataStreamCalc", + streamTableNode(0), + term("select", "c", "rowtime", + "*(c, c) AS $f2", "*(c, c) AS $f3", "*(c, c) AS $f4", "*(c, c) AS $f5") + ), + term("window", TumblingGroupWindow('w, 'rowtime, 900000.millis)), + term("select", + "SUM($f2) AS $f0", + "SUM(c) AS $f1", + "COUNT(c) AS $f2", + "SUM($f3) AS $f3", + "SUM($f4) AS $f4", + "SUM($f5) AS $f5", + "start('w) AS TMP_4", + "end('w) AS TMP_5") + ), + term("select", + "CAST(/(-($f0, /(*($f1, $f1), $f2)), $f2)) AS TMP_0", + "CAST(/(-($f3, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1)))) AS TMP_1", + "CAST(POWER(/(-($f4, /(*($f1, $f1), $f2)), $f2), 0.5)) AS TMP_2", + "CAST(POWER(/(-($f5, /(*($f1, $f1), $f2)), CASE(=($f2, 1), null, -($f2, 1))), 0.5)) " + + "AS TMP_3", + "TMP_4", + "TMP_5") + ) + + util.verifyTable(windowedTable, expected) + } } From d766988b28414c6e282c785a01b2fb6f4a4c21f6 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 15:29:12 +0100 Subject: [PATCH 0221/2294] [hotfix] [core] Add missing serialVersionUID to MapStateDescriptor --- .../org/apache/flink/api/common/state/MapStateDescriptor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java index 2e7ac98778f322..087cb5410bc598 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java @@ -42,6 +42,8 @@ @PublicEvolving public class MapStateDescriptor extends StateDescriptor, Map> { + private static final long serialVersionUID = 1L; + /** * Create a new {@code MapStateDescriptor} with the given name and the given type serializers. * From 13ef4e4406d749fbfe41f5d30d0849d0c70661d1 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 15:36:19 +0100 Subject: [PATCH 0222/2294] [hotfix] [core] Demockitofy state descriptor tests --- .../state/AggregatingStateDescriptorTest.java | 16 ++++--------- .../common/state/ListStateDescriptorTest.java | 14 +++-------- .../common/state/MapStateDescriptorTest.java | 23 ++++--------------- .../state/ReducingStateDescriptorTest.java | 22 +++++++----------- .../state/ValueStateDescriptorTest.java | 14 +++-------- 5 files changed, 23 insertions(+), 66 deletions(-) diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java index 155f23a9c637e7..f62acc8996fd52 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java @@ -18,17 +18,16 @@ package org.apache.flink.api.common.state; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.AggregateFunction; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.util.TestLogger; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import static org.junit.Assert.assertNotSame; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Tests for the {@link AggregatingStateDescriptor}. @@ -41,16 +40,11 @@ public class AggregatingStateDescriptorTest extends TestLogger { *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ - @SuppressWarnings("unchecked") @Test public void testSerializerDuplication() { - TypeSerializer serializer = mock(TypeSerializer.class); - when(serializer.duplicate()).thenAnswer(new Answer>() { - @Override - public TypeSerializer answer(InvocationOnMock invocation) throws Throwable { - return mock(TypeSerializer.class); - } - }); + // we need a serializer that actually duplicates for testing (a stateful one) + // we use Kryo here, because it meets these conditions + TypeSerializer serializer = new KryoSerializer<>(Long.class, new ExecutionConfig()); AggregateFunction aggregatingFunction = mock(AggregateFunction.class); diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java index c6d086e695c560..f45d2965ea1ec6 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java @@ -28,8 +28,6 @@ import org.apache.flink.core.testutils.CommonTestUtils; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import java.util.List; @@ -38,8 +36,6 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Tests for the {@link ListStateDescriptor}. @@ -121,13 +117,9 @@ public void testValueStateDescriptorAutoSerializer() throws Exception { @SuppressWarnings("unchecked") @Test public void testSerializerDuplication() { - TypeSerializer statefulSerializer = mock(TypeSerializer.class); - when(statefulSerializer.duplicate()).thenAnswer(new Answer>() { - @Override - public TypeSerializer answer(InvocationOnMock invocation) throws Throwable { - return mock(TypeSerializer.class); - } - }); + // we need a serializer that actually duplicates for testing (a stateful one) + // we use Kryo here, because it meets these conditions + TypeSerializer statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); ListStateDescriptor descr = new ListStateDescriptor<>("foobar", statefulSerializer); diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java index e2aa940351d2a4..21518347d69b0d 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java @@ -29,8 +29,6 @@ import org.apache.flink.core.testutils.CommonTestUtils; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import java.util.Map; @@ -39,8 +37,6 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Tests for the {@link MapStateDescriptor}. @@ -129,23 +125,12 @@ public void testMapStateDescriptorAutoSerializer() throws Exception { *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ - @SuppressWarnings("unchecked") @Test public void testSerializerDuplication() { - TypeSerializer keySerializer = mock(TypeSerializer.class); - TypeSerializer valueSerializer = mock(TypeSerializer.class); - when(keySerializer.duplicate()).thenAnswer(new Answer>() { - @Override - public TypeSerializer answer(InvocationOnMock invocation) throws Throwable { - return mock(TypeSerializer.class); - } - }); - when(valueSerializer.duplicate()).thenAnswer(new Answer>() { - @Override - public TypeSerializer answer(InvocationOnMock invocation) throws Throwable { - return mock(TypeSerializer.class); - } - }); + // we need a serializer that actually duplicates for testing (a stateful one) + // we use Kryo here, because it meets these conditions + TypeSerializer keySerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); + TypeSerializer valueSerializer = new KryoSerializer<>(Long.class, new ExecutionConfig()); MapStateDescriptor descr = new MapStateDescriptor<>("foobar", keySerializer, valueSerializer); diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java index ef39f1496c722a..1e21a78d19da87 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java @@ -29,8 +29,6 @@ import org.apache.flink.util.TestLogger; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -38,7 +36,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Tests for the {@link ReducingStateDescriptor}. @@ -118,17 +115,14 @@ public void testValueStateDescriptorAutoSerializer() throws Exception { @SuppressWarnings("unchecked") @Test public void testSerializerDuplication() { - TypeSerializer statefulSerializer = mock(TypeSerializer.class); - when(statefulSerializer.duplicate()).thenAnswer(new Answer>() { - @Override - public TypeSerializer answer(InvocationOnMock invocation) throws Throwable { - return mock(TypeSerializer.class); - } - }); - - ReduceFunction reducer = mock(ReduceFunction.class); - - ReducingStateDescriptor descr = new ReducingStateDescriptor<>("foobar", reducer, statefulSerializer); + // we need a serializer that actually duplicates for testing (a stateful one) + // we use Kryo here, because it meets these conditions + TypeSerializer statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); + + ReducingStateDescriptor descr = new ReducingStateDescriptor<>( + "foobar", + (a, b) -> a, + statefulSerializer); TypeSerializer serializerA = descr.getSerializer(); TypeSerializer serializerB = descr.getSerializer(); diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java index b43e5ad16345f5..f3b9eee93f2250 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java @@ -29,8 +29,6 @@ import org.apache.flink.util.TestLogger; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import java.io.File; @@ -39,8 +37,6 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * Tests for the {@link ValueStateDescriptor}. @@ -149,13 +145,9 @@ public void testVeryLargeDefaultValue() throws Exception { @SuppressWarnings("unchecked") @Test public void testSerializerDuplication() { - TypeSerializer statefulSerializer = mock(TypeSerializer.class); - when(statefulSerializer.duplicate()).thenAnswer(new Answer>() { - @Override - public TypeSerializer answer(InvocationOnMock invocation) throws Throwable { - return mock(TypeSerializer.class); - } - }); + // we need a serializer that actually duplicates for testing (a stateful one) + // we use Kryo here, because it meets these conditions + TypeSerializer statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); ValueStateDescriptor descr = new ValueStateDescriptor<>("foobar", statefulSerializer); From 87dcc890ffd1e52872129a2ec6c40668fd7e7184 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 15:44:27 +0100 Subject: [PATCH 0223/2294] [hotfix] [core] Make State Descriptors consistently use Preconditions instead of Objects. --- .../api/common/state/ReducingStateDescriptor.java | 8 ++++---- .../flink/api/common/state/StateDescriptor.java | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java index a14b4bd1815575..ef483e2af04225 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java @@ -24,7 +24,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; -import static java.util.Objects.requireNonNull; +import static org.apache.flink.util.Preconditions.checkNotNull; /** * {@link StateDescriptor} for {@link ReducingState}. This can be used to create partitioned @@ -52,7 +52,7 @@ public class ReducingStateDescriptor extends StateDescriptor */ public ReducingStateDescriptor(String name, ReduceFunction reduceFunction, Class typeClass) { super(name, typeClass, null); - this.reduceFunction = requireNonNull(reduceFunction); + this.reduceFunction = checkNotNull(reduceFunction); if (reduceFunction instanceof RichFunction) { throw new UnsupportedOperationException("ReduceFunction of ReducingState can not be a RichFunction."); @@ -68,7 +68,7 @@ public ReducingStateDescriptor(String name, ReduceFunction reduceFunction, Cl */ public ReducingStateDescriptor(String name, ReduceFunction reduceFunction, TypeInformation typeInfo) { super(name, typeInfo, null); - this.reduceFunction = requireNonNull(reduceFunction); + this.reduceFunction = checkNotNull(reduceFunction); } /** @@ -80,7 +80,7 @@ public ReducingStateDescriptor(String name, ReduceFunction reduceFunction, Ty */ public ReducingStateDescriptor(String name, ReduceFunction reduceFunction, TypeSerializer typeSerializer) { super(name, typeSerializer, null); - this.reduceFunction = requireNonNull(reduceFunction); + this.reduceFunction = checkNotNull(reduceFunction); } // ------------------------------------------------------------------------ diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java index 841f710db59176..5ec59e426c640d 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java @@ -34,7 +34,7 @@ import java.io.ObjectOutputStream; import java.io.Serializable; -import static java.util.Objects.requireNonNull; +import static org.apache.flink.util.Preconditions.checkNotNull; /** * Base class for state descriptors. A {@code StateDescriptor} is used for creating partitioned @@ -100,8 +100,8 @@ public enum Type { * a value before. */ protected StateDescriptor(String name, TypeSerializer serializer, T defaultValue) { - this.name = requireNonNull(name, "name must not be null"); - this.serializer = requireNonNull(serializer, "serializer must not be null"); + this.name = checkNotNull(name, "name must not be null"); + this.serializer = checkNotNull(serializer, "serializer must not be null"); this.defaultValue = defaultValue; } @@ -114,8 +114,8 @@ protected StateDescriptor(String name, TypeSerializer serializer, T defaultVa * a value before. */ protected StateDescriptor(String name, TypeInformation typeInfo, T defaultValue) { - this.name = requireNonNull(name, "name must not be null"); - this.typeInfo = requireNonNull(typeInfo, "type information must not be null"); + this.name = checkNotNull(name, "name must not be null"); + this.typeInfo = checkNotNull(typeInfo, "type information must not be null"); this.defaultValue = defaultValue; } @@ -131,8 +131,8 @@ protected StateDescriptor(String name, TypeInformation typeInfo, T defaultVal * a value before. */ protected StateDescriptor(String name, Class type, T defaultValue) { - this.name = requireNonNull(name, "name must not be null"); - requireNonNull(type, "type class must not be null"); + this.name = checkNotNull(name, "name must not be null"); + checkNotNull(type, "type class must not be null"); try { this.typeInfo = TypeExtractor.createTypeInfo(type); From 87d31f5cf76fa796b89201ed8c55890e7d36fc81 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 16:22:12 +0100 Subject: [PATCH 0224/2294] [FLINK-9034] [core] StateDescriptor does not throw away TypeInformation upon serialization. Throwing away TypeInformation upon serialization was previously done because the type information was not serializable. Now that it is serializable, we can (and should) keep it to provide consistent user experience, where all serializers respect the ExecutionConfig. --- .../api/common/state/StateDescriptor.java | 41 +++-- .../common/state/ListStateDescriptorTest.java | 48 +---- .../common/state/MapStateDescriptorTest.java | 54 +----- .../state/ReducingStateDescriptorTest.java | 54 +----- .../api/common/state/StateDescriptorTest.java | 171 ++++++++++++++++++ .../state/ValueStateDescriptorTest.java | 71 -------- 6 files changed, 200 insertions(+), 239 deletions(-) create mode 100644 flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java index 5ec59e426c640d..574c83603ec166 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java @@ -27,6 +27,8 @@ import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.util.Preconditions; +import javax.annotation.Nullable; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -35,6 +37,7 @@ import java.io.Serializable; import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; /** * Base class for state descriptors. A {@code StateDescriptor} is used for creating partitioned @@ -76,19 +79,24 @@ public enum Type { protected final String name; /** The serializer for the type. May be eagerly initialized in the constructor, - * or lazily once the type is serialized or an ExecutionConfig is provided. */ + * or lazily once the {@link #initializeSerializerUnlessSet(ExecutionConfig)} method + * is called. */ + @Nullable protected TypeSerializer serializer; + /** The type information describing the value type. Only used to if the serializer + * is created lazily. */ + @Nullable + private TypeInformation typeInfo; + /** Name for queries against state created from this StateDescriptor. */ + @Nullable private String queryableStateName; /** The default value returned by the state when no other value is bound to a key. */ + @Nullable protected transient T defaultValue; - /** The type information describing the value type. Only used to lazily create the serializer - * and dropped during serialization */ - private transient TypeInformation typeInfo; - // ------------------------------------------------------------------------ /** @@ -99,7 +107,7 @@ public enum Type { * @param defaultValue The default value that will be set when requesting state without setting * a value before. */ - protected StateDescriptor(String name, TypeSerializer serializer, T defaultValue) { + protected StateDescriptor(String name, TypeSerializer serializer, @Nullable T defaultValue) { this.name = checkNotNull(name, "name must not be null"); this.serializer = checkNotNull(serializer, "serializer must not be null"); this.defaultValue = defaultValue; @@ -113,7 +121,7 @@ protected StateDescriptor(String name, TypeSerializer serializer, T defaultVa * @param defaultValue The default value that will be set when requesting state without setting * a value before. */ - protected StateDescriptor(String name, TypeInformation typeInfo, T defaultValue) { + protected StateDescriptor(String name, TypeInformation typeInfo, @Nullable T defaultValue) { this.name = checkNotNull(name, "name must not be null"); this.typeInfo = checkNotNull(typeInfo, "type information must not be null"); this.defaultValue = defaultValue; @@ -130,7 +138,7 @@ protected StateDescriptor(String name, TypeInformation typeInfo, T defaultVal * @param defaultValue The default value that will be set when requesting state without setting * a value before. */ - protected StateDescriptor(String name, Class type, T defaultValue) { + protected StateDescriptor(String name, Class type, @Nullable T defaultValue) { this.name = checkNotNull(name, "name must not be null"); checkNotNull(type, "type class must not be null"); @@ -208,6 +216,7 @@ public void setQueryable(String queryableStateName) { * * @return Queryable state name or null if not set. */ + @Nullable public String getQueryableStateName() { return queryableStateName; } @@ -249,12 +258,13 @@ public boolean isSerializerInitialized() { */ public void initializeSerializerUnlessSet(ExecutionConfig executionConfig) { if (serializer == null) { - if (typeInfo != null) { - serializer = typeInfo.createSerializer(executionConfig); - } else { - throw new IllegalStateException( - "Cannot initialize serializer after TypeInformation was dropped during serialization"); - } + checkState(typeInfo != null, "no serializer and no type info"); + + // instantiate the serializer + serializer = typeInfo.createSerializer(executionConfig); + + // we can drop the type info now, no longer needed + typeInfo = null; } } @@ -285,9 +295,6 @@ public String toString() { // ------------------------------------------------------------------------ private void writeObject(final ObjectOutputStream out) throws IOException { - // make sure we have a serializer before the type information gets lost - initializeSerializerUnlessSet(new ExecutionConfig()); - // write all the non-transient fields out.defaultWriteObject(); diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java index f45d2965ea1ec6..e7e33e79ca5783 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java @@ -19,12 +19,9 @@ package org.apache.flink.api.common.state; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.base.ListSerializer; -import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; -import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; import org.junit.Test; @@ -35,7 +32,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Tests for the {@link ListStateDescriptor}. @@ -43,7 +39,7 @@ public class ListStateDescriptorTest { @Test - public void testValueStateDescriptorEagerSerializer() throws Exception { + public void testListStateDescriptor() throws Exception { TypeSerializer serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); @@ -66,48 +62,6 @@ public void testValueStateDescriptorEagerSerializer() throws Exception { assertEquals(serializer, copy.getElementSerializer()); } - @Test - public void testValueStateDescriptorLazySerializer() throws Exception { - // some different registered value - ExecutionConfig cfg = new ExecutionConfig(); - cfg.registerKryoType(TaskInfo.class); - - ListStateDescriptor descr = - new ListStateDescriptor<>("testName", Path.class); - - try { - descr.getSerializer(); - fail("should cause an exception"); - } catch (IllegalStateException ignored) {} - - descr.initializeSerializerUnlessSet(cfg); - - assertNotNull(descr.getSerializer()); - assertTrue(descr.getSerializer() instanceof ListSerializer); - - assertNotNull(descr.getElementSerializer()); - assertTrue(descr.getElementSerializer() instanceof KryoSerializer); - - assertTrue(((KryoSerializer) descr.getElementSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0); - } - - @Test - public void testValueStateDescriptorAutoSerializer() throws Exception { - - ListStateDescriptor descr = - new ListStateDescriptor<>("testName", String.class); - - ListStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); - - assertEquals("testName", copy.getName()); - - assertNotNull(copy.getSerializer()); - assertTrue(copy.getSerializer() instanceof ListSerializer); - - assertNotNull(copy.getElementSerializer()); - assertEquals(StringSerializer.INSTANCE, copy.getElementSerializer()); - } - /** * FLINK-6775. * diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java index 21518347d69b0d..4e64c0f436a017 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java @@ -19,13 +19,9 @@ package org.apache.flink.api.common.state; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.base.LongSerializer; import org.apache.flink.api.common.typeutils.base.MapSerializer; -import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; -import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; import org.junit.Test; @@ -36,7 +32,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Tests for the {@link MapStateDescriptor}. @@ -44,7 +39,7 @@ public class MapStateDescriptorTest { @Test - public void testMapStateDescriptorEagerSerializer() throws Exception { + public void testMapStateDescriptor() throws Exception { TypeSerializer keySerializer = new KryoSerializer<>(Integer.class, new ExecutionConfig()); TypeSerializer valueSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); @@ -72,53 +67,6 @@ public void testMapStateDescriptorEagerSerializer() throws Exception { assertEquals(valueSerializer, copy.getValueSerializer()); } - @Test - public void testMapStateDescriptorLazySerializer() throws Exception { - // some different registered value - ExecutionConfig cfg = new ExecutionConfig(); - cfg.registerKryoType(TaskInfo.class); - - MapStateDescriptor descr = - new MapStateDescriptor<>("testName", Path.class, String.class); - - try { - descr.getSerializer(); - fail("should cause an exception"); - } catch (IllegalStateException ignored) {} - - descr.initializeSerializerUnlessSet(cfg); - - assertNotNull(descr.getSerializer()); - assertTrue(descr.getSerializer() instanceof MapSerializer); - - assertNotNull(descr.getKeySerializer()); - assertTrue(descr.getKeySerializer() instanceof KryoSerializer); - - assertTrue(((KryoSerializer) descr.getKeySerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0); - - assertNotNull(descr.getValueSerializer()); - assertTrue(descr.getValueSerializer() instanceof StringSerializer); - } - - @Test - public void testMapStateDescriptorAutoSerializer() throws Exception { - - MapStateDescriptor descr = - new MapStateDescriptor<>("testName", String.class, Long.class); - - MapStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); - - assertEquals("testName", copy.getName()); - - assertNotNull(copy.getSerializer()); - assertTrue(copy.getSerializer() instanceof MapSerializer); - - assertNotNull(copy.getKeySerializer()); - assertEquals(StringSerializer.INSTANCE, copy.getKeySerializer()); - assertNotNull(copy.getValueSerializer()); - assertEquals(LongSerializer.INSTANCE, copy.getValueSerializer()); - } - /** * FLINK-6775. * diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java index 1e21a78d19da87..81b7c38cf79650 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java @@ -19,12 +19,9 @@ package org.apache.flink.api.common.state; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; -import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; import org.apache.flink.util.TestLogger; @@ -33,9 +30,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; /** * Tests for the {@link ReducingStateDescriptor}. @@ -43,10 +37,9 @@ public class ReducingStateDescriptorTest extends TestLogger { @Test - public void testValueStateDescriptorEagerSerializer() throws Exception { + public void testReducingStateDescriptor() throws Exception { - @SuppressWarnings("unchecked") - ReduceFunction reducer = mock(ReduceFunction.class); + ReduceFunction reducer = (a, b) -> a; TypeSerializer serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); @@ -56,6 +49,7 @@ public void testValueStateDescriptorEagerSerializer() throws Exception { assertEquals("testName", descr.getName()); assertNotNull(descr.getSerializer()); assertEquals(serializer, descr.getSerializer()); + assertEquals(reducer, descr.getReduceFunction()); ReducingStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); @@ -64,48 +58,6 @@ public void testValueStateDescriptorEagerSerializer() throws Exception { assertEquals(serializer, copy.getSerializer()); } - @Test - public void testValueStateDescriptorLazySerializer() throws Exception { - - @SuppressWarnings("unchecked") - ReduceFunction reducer = mock(ReduceFunction.class); - - // some different registered value - ExecutionConfig cfg = new ExecutionConfig(); - cfg.registerKryoType(TaskInfo.class); - - ReducingStateDescriptor descr = - new ReducingStateDescriptor<>("testName", reducer, Path.class); - - try { - descr.getSerializer(); - fail("should cause an exception"); - } catch (IllegalStateException ignored) {} - - descr.initializeSerializerUnlessSet(cfg); - - assertNotNull(descr.getSerializer()); - assertTrue(descr.getSerializer() instanceof KryoSerializer); - - assertTrue(((KryoSerializer) descr.getSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0); - } - - @Test - public void testValueStateDescriptorAutoSerializer() throws Exception { - - @SuppressWarnings("unchecked") - ReduceFunction reducer = mock(ReduceFunction.class); - - ReducingStateDescriptor descr = - new ReducingStateDescriptor<>("testName", reducer, String.class); - - ReducingStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); - - assertEquals("testName", copy.getName()); - assertNotNull(copy.getSerializer()); - assertEquals(StringSerializer.INSTANCE, copy.getSerializer()); - } - /** * FLINK-6775. * diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java new file mode 100644 index 00000000000000..59293f410605f4 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.state; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; +import org.apache.flink.core.fs.Path; +import org.apache.flink.core.testutils.CommonTestUtils; + +import org.junit.Test; + +import java.io.File; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests for the common/shared functionality of {@link StateDescriptor}. + */ +public class StateDescriptorTest { + + @Test + public void testInitializeWithSerializer() throws Exception { + final TypeSerializer serializer = StringSerializer.INSTANCE; + final TestStateDescriptor descr = new TestStateDescriptor<>("test", serializer); + + assertTrue(descr.isSerializerInitialized()); + assertNotNull(descr.getSerializer()); + assertTrue(descr.getSerializer() instanceof StringSerializer); + + // this should not have any effect + descr.initializeSerializerUnlessSet(new ExecutionConfig()); + assertTrue(descr.isSerializerInitialized()); + assertNotNull(descr.getSerializer()); + assertTrue(descr.getSerializer() instanceof StringSerializer); + + TestStateDescriptor clone = CommonTestUtils.createCopySerializable(descr); + assertTrue(clone.isSerializerInitialized()); + assertNotNull(clone.getSerializer()); + assertTrue(clone.getSerializer() instanceof StringSerializer); + } + + @Test + public void testInitializeSerializerBeforeSerialization() throws Exception { + final TestStateDescriptor descr = new TestStateDescriptor<>("test", String.class); + + assertFalse(descr.isSerializerInitialized()); + try { + descr.getSerializer(); + fail("should fail with an exception"); + } catch (IllegalStateException ignored) {} + + descr.initializeSerializerUnlessSet(new ExecutionConfig()); + + assertTrue(descr.isSerializerInitialized()); + assertNotNull(descr.getSerializer()); + assertTrue(descr.getSerializer() instanceof StringSerializer); + + TestStateDescriptor clone = CommonTestUtils.createCopySerializable(descr); + + assertTrue(clone.isSerializerInitialized()); + assertNotNull(clone.getSerializer()); + assertTrue(clone.getSerializer() instanceof StringSerializer); + } + + @Test + public void testInitializeSerializerAfterSerialization() throws Exception { + final TestStateDescriptor descr = new TestStateDescriptor<>("test", String.class); + + assertFalse(descr.isSerializerInitialized()); + try { + descr.getSerializer(); + fail("should fail with an exception"); + } catch (IllegalStateException ignored) {} + + TestStateDescriptor clone = CommonTestUtils.createCopySerializable(descr); + + assertFalse(clone.isSerializerInitialized()); + try { + clone.getSerializer(); + fail("should fail with an exception"); + } catch (IllegalStateException ignored) {} + + clone.initializeSerializerUnlessSet(new ExecutionConfig()); + + assertTrue(clone.isSerializerInitialized()); + assertNotNull(clone.getSerializer()); + assertTrue(clone.getSerializer() instanceof StringSerializer); + } + + @Test + public void testInitializeSerializerAfterSerializationWithCustomConfig() throws Exception { + // guard our test assumptions. + assertEquals("broken test assumption", -1, + new KryoSerializer<>(String.class, new ExecutionConfig()).getKryo() + .getRegistration(File.class).getId()); + + final ExecutionConfig config = new ExecutionConfig(); + config.registerKryoType(File.class); + + final TestStateDescriptor original = new TestStateDescriptor<>("test", Path.class); + TestStateDescriptor clone = CommonTestUtils.createCopySerializable(original); + + clone.initializeSerializerUnlessSet(config); + + // serialized one (later initialized) carries the registration + assertTrue(((KryoSerializer) clone.getSerializer()).getKryo() + .getRegistration(File.class).getId() > 0); + } + + // ------------------------------------------------------------------------ + + private static class TestStateDescriptor extends StateDescriptor { + + private static final long serialVersionUID = 1L; + + TestStateDescriptor(String name, TypeSerializer serializer) { + super(name, serializer, null); + } + + TestStateDescriptor(String name, TypeInformation typeInfo) { + super(name, typeInfo, null); + } + + TestStateDescriptor(String name, Class type) { + super(name, type, null); + } + + @Override + public State bind(StateBinder stateBinder) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public Type getType() { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + return 584523; + } + + @Override + public boolean equals(Object o) { + return o != null && o.getClass() == TestStateDescriptor.class; + } + } +} diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java index f3b9eee93f2250..7ee58fedb98644 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java @@ -19,94 +19,23 @@ package org.apache.flink.api.common.state; import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.TaskInfo; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.core.fs.Path; import org.apache.flink.core.testutils.CommonTestUtils; import org.apache.flink.util.TestLogger; import org.junit.Test; -import java.io.File; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Tests for the {@link ValueStateDescriptor}. */ public class ValueStateDescriptorTest extends TestLogger { - @Test - public void testValueStateDescriptorEagerSerializer() throws Exception { - - TypeSerializer serializer = new KryoSerializer<>(String.class, new ExecutionConfig()); - String defaultValue = "le-value-default"; - - ValueStateDescriptor descr = - new ValueStateDescriptor<>("testName", serializer, defaultValue); - - assertEquals("testName", descr.getName()); - assertEquals(defaultValue, descr.getDefaultValue()); - assertNotNull(descr.getSerializer()); - assertEquals(serializer, descr.getSerializer()); - - ValueStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); - - assertEquals("testName", copy.getName()); - assertEquals(defaultValue, copy.getDefaultValue()); - assertNotNull(copy.getSerializer()); - assertEquals(serializer, copy.getSerializer()); - } - - @Test - public void testValueStateDescriptorLazySerializer() throws Exception { - - // some default value that goes to the generic serializer - Path defaultValue = new Path(new File(ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH).toURI()); - - // some different registered value - ExecutionConfig cfg = new ExecutionConfig(); - cfg.registerKryoType(TaskInfo.class); - - ValueStateDescriptor descr = - new ValueStateDescriptor<>("testName", Path.class, defaultValue); - - try { - descr.getSerializer(); - fail("should cause an exception"); - } catch (IllegalStateException ignored) {} - - descr.initializeSerializerUnlessSet(cfg); - - assertNotNull(descr.getSerializer()); - assertTrue(descr.getSerializer() instanceof KryoSerializer); - - assertTrue(((KryoSerializer) descr.getSerializer()).getKryo().getRegistration(TaskInfo.class).getId() > 0); - } - - @Test - public void testValueStateDescriptorAutoSerializer() throws Exception { - - String defaultValue = "le-value-default"; - - ValueStateDescriptor descr = - new ValueStateDescriptor<>("testName", String.class, defaultValue); - - ValueStateDescriptor copy = CommonTestUtils.createCopySerializable(descr); - - assertEquals("testName", copy.getName()); - assertEquals(defaultValue, copy.getDefaultValue()); - assertNotNull(copy.getSerializer()); - assertEquals(StringSerializer.INSTANCE, copy.getSerializer()); - } - @Test public void testVeryLargeDefaultValue() throws Exception { // ensure that we correctly read very large data when deserializing the default value From 7667ddcfb9c2cfa96ffdf7594affeb74719e8ccf Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 16:46:13 +0100 Subject: [PATCH 0225/2294] [hotfix] [core] Consolidate serializer duplication tests in StateDescriptorTest where possible --- .../state/AggregatingStateDescriptorTest.java | 62 ------------------- .../common/state/ListStateDescriptorTest.java | 1 - .../state/ReducingStateDescriptorTest.java | 27 -------- .../api/common/state/StateDescriptorTest.java | 30 +++++++++ .../state/ValueStateDescriptorTest.java | 23 ------- 5 files changed, 30 insertions(+), 113 deletions(-) delete mode 100644 flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java deleted file mode 100644 index f62acc8996fd52..00000000000000 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/AggregatingStateDescriptorTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.api.common.state; - -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.functions.AggregateFunction; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; -import org.apache.flink.util.TestLogger; - -import org.junit.Test; - -import static org.junit.Assert.assertNotSame; -import static org.mockito.Mockito.mock; - -/** - * Tests for the {@link AggregatingStateDescriptor}. - */ -public class AggregatingStateDescriptorTest extends TestLogger { - - /** - * FLINK-6775. - * - *

    Tests that the returned serializer is duplicated. This allows to - * share the state descriptor. - */ - @Test - public void testSerializerDuplication() { - // we need a serializer that actually duplicates for testing (a stateful one) - // we use Kryo here, because it meets these conditions - TypeSerializer serializer = new KryoSerializer<>(Long.class, new ExecutionConfig()); - - AggregateFunction aggregatingFunction = mock(AggregateFunction.class); - - AggregatingStateDescriptor descr = new AggregatingStateDescriptor<>( - "foobar", - aggregatingFunction, - serializer); - - TypeSerializer serializerA = descr.getSerializer(); - TypeSerializer serializerB = descr.getSerializer(); - - // check that the retrieved serializers are not the same - assertNotSame(serializerA, serializerB); - } -} diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java index e7e33e79ca5783..b934ee09d8ed24 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java @@ -68,7 +68,6 @@ public void testListStateDescriptor() throws Exception { *

    Tests that the returned serializer is duplicated. This allows to * share the state descriptor. */ - @SuppressWarnings("unchecked") @Test public void testSerializerDuplication() { // we need a serializer that actually duplicates for testing (a stateful one) diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java index 81b7c38cf79650..5d9eba52291776 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java @@ -29,7 +29,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; /** * Tests for the {@link ReducingStateDescriptor}. @@ -57,30 +56,4 @@ public void testReducingStateDescriptor() throws Exception { assertNotNull(copy.getSerializer()); assertEquals(serializer, copy.getSerializer()); } - - /** - * FLINK-6775. - * - *

    Tests that the returned serializer is duplicated. This allows to - * share the state descriptor. - */ - @SuppressWarnings("unchecked") - @Test - public void testSerializerDuplication() { - // we need a serializer that actually duplicates for testing (a stateful one) - // we use Kryo here, because it meets these conditions - TypeSerializer statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); - - ReducingStateDescriptor descr = new ReducingStateDescriptor<>( - "foobar", - (a, b) -> a, - statefulSerializer); - - TypeSerializer serializerA = descr.getSerializer(); - TypeSerializer serializerB = descr.getSerializer(); - - // check that the retrieved serializers are not the same - assertNotSame(serializerA, serializerB); - } - } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java index 59293f410605f4..cf5327e49dda42 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -41,6 +42,10 @@ */ public class StateDescriptorTest { + // ------------------------------------------------------------------------ + // Tests for serializer initialization + // ------------------------------------------------------------------------ + @Test public void testInitializeWithSerializer() throws Exception { final TypeSerializer serializer = StringSerializer.INSTANCE; @@ -130,6 +135,31 @@ public void testInitializeSerializerAfterSerializationWithCustomConfig() throws .getRegistration(File.class).getId() > 0); } + // ------------------------------------------------------------------------ + // Tests for serializer initialization + // ------------------------------------------------------------------------ + + /** + * FLINK-6775, tests that the returned serializer is duplicated. + * This allows to share the state descriptor across threads. + */ + @Test + public void testSerializerDuplication() throws Exception { + // we need a serializer that actually duplicates for testing (a stateful one) + // we use Kryo here, because it meets these conditions + TypeSerializer statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); + + TestStateDescriptor descr = new TestStateDescriptor<>("foobar", statefulSerializer); + + TypeSerializer serializerA = descr.getSerializer(); + TypeSerializer serializerB = descr.getSerializer(); + + // check that the retrieved serializers are not the same + assertNotSame(serializerA, serializerB); + } + + // ------------------------------------------------------------------------ + // Mock implementations and test types // ------------------------------------------------------------------------ private static class TestStateDescriptor extends StateDescriptor { diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java index 7ee58fedb98644..67114e50af81b1 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java @@ -29,7 +29,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; /** * Tests for the {@link ValueStateDescriptor}. @@ -64,26 +63,4 @@ public void testVeryLargeDefaultValue() throws Exception { assertNotNull(copy.getSerializer()); assertEquals(serializer, copy.getSerializer()); } - - /** - * FLINK-6775. - * - *

    Tests that the returned serializer is duplicated. This allows to - * share the state descriptor. - */ - @SuppressWarnings("unchecked") - @Test - public void testSerializerDuplication() { - // we need a serializer that actually duplicates for testing (a stateful one) - // we use Kryo here, because it meets these conditions - TypeSerializer statefulSerializer = new KryoSerializer<>(String.class, new ExecutionConfig()); - - ValueStateDescriptor descr = new ValueStateDescriptor<>("foobar", statefulSerializer); - - TypeSerializer serializerA = descr.getSerializer(); - TypeSerializer serializerB = descr.getSerializer(); - - // check that the retrieved serializers are not the same - assertNotSame(serializerA, serializerB); - } } From f3a519712fb31f7b71181e876c3c3d5fff08eb71 Mon Sep 17 00:00:00 2001 From: Stephan Ewen Date: Tue, 20 Mar 2018 17:16:06 +0100 Subject: [PATCH 0226/2294] [FLINK-9035] [core] Fix state descriptor equals() and hashCode() handling --- .../state/AggregatingStateDescriptor.java | 31 --------- .../common/state/FoldingStateDescriptor.java | 31 --------- .../api/common/state/ListStateDescriptor.java | 30 -------- .../api/common/state/MapStateDescriptor.java | 29 -------- .../common/state/ReducingStateDescriptor.java | 30 -------- .../api/common/state/StateDescriptor.java | 17 ++++- .../common/state/ValueStateDescriptor.java | 31 --------- .../common/state/ListStateDescriptorTest.java | 28 ++++++++ .../common/state/MapStateDescriptorTest.java | 29 ++++++++ .../state/ReducingStateDescriptorTest.java | 29 ++++++++ .../api/common/state/StateDescriptorTest.java | 69 +++++++++++++++++-- .../state/ValueStateDescriptorTest.java | 28 ++++++++ 12 files changed, 193 insertions(+), 189 deletions(-) diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java index 6f6d2f9790e086..8c7fed621da71a 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/AggregatingStateDescriptor.java @@ -111,35 +111,4 @@ public AggregateFunction getAggregateFunction() { public Type getType() { return Type.AGGREGATING; } - - // ------------------------------------------------------------------------ - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - else if (o != null && getClass() == o.getClass()) { - AggregatingStateDescriptor that = (AggregatingStateDescriptor) o; - return serializer.equals(that.serializer) && name.equals(that.name); - } - else { - return false; - } - } - - @Override - public int hashCode() { - int result = serializer.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } - - @Override - public String toString() { - return "AggregatingStateDescriptor{" + - "serializer=" + serializer + - ", aggFunction=" + aggFunction + - '}'; - } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java index 261d1fe47219da..c14e4bfc183618 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/FoldingStateDescriptor.java @@ -111,37 +111,6 @@ public FoldFunction getFoldFunction() { return foldFunction; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - FoldingStateDescriptor that = (FoldingStateDescriptor) o; - - return serializer.equals(that.serializer) && name.equals(that.name); - - } - - @Override - public int hashCode() { - int result = serializer.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } - - @Override - public String toString() { - return "FoldingStateDescriptor{" + - "serializer=" + serializer + - ", initialValue=" + defaultValue + - ", foldFunction=" + foldFunction + - '}'; - } - @Override public Type getType() { return Type.FOLDING; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java index 38e56803330e5a..aa5e64b019deb7 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ListStateDescriptor.java @@ -102,34 +102,4 @@ public TypeSerializer getElementSerializer() { public Type getType() { return Type.LIST; } - - // ------------------------------------------------------------------------ - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - final ListStateDescriptor that = (ListStateDescriptor) o; - return serializer.equals(that.serializer) && name.equals(that.name); - - } - - @Override - public int hashCode() { - int result = serializer.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } - - @Override - public String toString() { - return "ListStateDescriptor{" + - "serializer=" + serializer + - '}'; - } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java index 087cb5410bc598..42b016adc9b149 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/MapStateDescriptor.java @@ -117,33 +117,4 @@ public TypeSerializer getValueSerializer() { return ((MapSerializer) rawSerializer).getValueSerializer(); } - - @Override - public int hashCode() { - int result = serializer.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - MapStateDescriptor that = (MapStateDescriptor) o; - return serializer.equals(that.serializer) && name.equals(that.name); - } - - @Override - public String toString() { - return "MapStateDescriptor{" + - "name=" + name + - ", serializer=" + serializer + - '}'; - } } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java index ef483e2af04225..0df1c2c862b062 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ReducingStateDescriptor.java @@ -97,36 +97,6 @@ public ReduceFunction getReduceFunction() { return reduceFunction; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - ReducingStateDescriptor that = (ReducingStateDescriptor) o; - - return serializer.equals(that.serializer) && name.equals(that.name); - - } - - @Override - public int hashCode() { - int result = serializer.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } - - @Override - public String toString() { - return "ReducingStateDescriptor{" + - "serializer=" + serializer + - ", reduceFunction=" + reduceFunction + - '}'; - } - @Override public Type getType() { return Type.REDUCING; diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java index 574c83603ec166..9b6b51dc8ced99 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/StateDescriptor.java @@ -273,10 +273,23 @@ public void initializeSerializerUnlessSet(ExecutionConfig executionConfig) { // ------------------------------------------------------------------------ @Override - public abstract int hashCode(); + public final int hashCode() { + return name.hashCode() + 31 * getClass().hashCode(); + } @Override - public abstract boolean equals(Object o); + public final boolean equals(Object o) { + if (o == this) { + return true; + } + else if (o != null && o.getClass() == this.getClass()) { + final StateDescriptor that = (StateDescriptor) o; + return this.name.equals(that.name); + } + else { + return false; + } + } @Override public String toString() { diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java index ef18d741209c31..4d69d811e4a599 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/state/ValueStateDescriptor.java @@ -129,37 +129,6 @@ public ValueState bind(StateBinder stateBinder) throws Exception { return stateBinder.createValueState(this); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - ValueStateDescriptor that = (ValueStateDescriptor) o; - - return serializer.equals(that.serializer) && name.equals(that.name); - - } - - @Override - public int hashCode() { - int result = serializer.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } - - @Override - public String toString() { - return "ValueStateDescriptor{" + - "name=" + name + - ", defaultValue=" + defaultValue + - ", serializer=" + serializer + - '}'; - } - @Override public Type getType() { return Type.VALUE; diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java index b934ee09d8ed24..cb6f6083d70250 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ListStateDescriptorTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.base.ListSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.core.testutils.CommonTestUtils; @@ -62,6 +63,33 @@ public void testListStateDescriptor() throws Exception { assertEquals(serializer, copy.getElementSerializer()); } + @Test + public void testHashCodeEquals() throws Exception { + final String name = "testName"; + + ListStateDescriptor original = new ListStateDescriptor<>(name, String.class); + ListStateDescriptor same = new ListStateDescriptor<>(name, String.class); + ListStateDescriptor sameBySerializer = new ListStateDescriptor<>(name, StringSerializer.INSTANCE); + + // test that hashCode() works on state descriptors with initialized and uninitialized serializers + assertEquals(original.hashCode(), same.hashCode()); + assertEquals(original.hashCode(), sameBySerializer.hashCode()); + + assertEquals(original, same); + assertEquals(original, sameBySerializer); + + // equality with a clone + ListStateDescriptor clone = CommonTestUtils.createCopySerializable(original); + assertEquals(original, clone); + + // equality with an initialized + clone.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, clone); + + original.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, same); + } + /** * FLINK-6775. * diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java index 4e64c0f436a017..069d6c2ae07d7b 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/MapStateDescriptorTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.common.typeutils.base.MapSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.core.testutils.CommonTestUtils; @@ -67,6 +68,34 @@ public void testMapStateDescriptor() throws Exception { assertEquals(valueSerializer, copy.getValueSerializer()); } + @Test + public void testHashCodeEquals() throws Exception { + final String name = "testName"; + + MapStateDescriptor original = new MapStateDescriptor<>(name, String.class, String.class); + MapStateDescriptor same = new MapStateDescriptor<>(name, String.class, String.class); + MapStateDescriptor sameBySerializer = + new MapStateDescriptor<>(name, StringSerializer.INSTANCE, StringSerializer.INSTANCE); + + // test that hashCode() works on state descriptors with initialized and uninitialized serializers + assertEquals(original.hashCode(), same.hashCode()); + assertEquals(original.hashCode(), sameBySerializer.hashCode()); + + assertEquals(original, same); + assertEquals(original, sameBySerializer); + + // equality with a clone + MapStateDescriptor clone = CommonTestUtils.createCopySerializable(original); + assertEquals(original, clone); + + // equality with an initialized + clone.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, clone); + + original.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, same); + } + /** * FLINK-6775. * diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java index 5d9eba52291776..89aa1e688242b7 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ReducingStateDescriptorTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.ReduceFunction; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.core.testutils.CommonTestUtils; import org.apache.flink.util.TestLogger; @@ -56,4 +57,32 @@ public void testReducingStateDescriptor() throws Exception { assertNotNull(copy.getSerializer()); assertEquals(serializer, copy.getSerializer()); } + + @Test + public void testHashCodeEquals() throws Exception { + final String name = "testName"; + final ReduceFunction reducer = (a, b) -> a; + + ReducingStateDescriptor original = new ReducingStateDescriptor<>(name, reducer, String.class); + ReducingStateDescriptor same = new ReducingStateDescriptor<>(name, reducer, String.class); + ReducingStateDescriptor sameBySerializer = new ReducingStateDescriptor<>(name, reducer, StringSerializer.INSTANCE); + + // test that hashCode() works on state descriptors with initialized and uninitialized serializers + assertEquals(original.hashCode(), same.hashCode()); + assertEquals(original.hashCode(), sameBySerializer.hashCode()); + + assertEquals(original, same); + assertEquals(original, sameBySerializer); + + // equality with a clone + ReducingStateDescriptor clone = CommonTestUtils.createCopySerializable(original); + assertEquals(original, clone); + + // equality with an initialized + clone.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, clone); + + original.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, same); + } } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java index cf5327e49dda42..3958baa120a128 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/StateDescriptorTest.java @@ -32,6 +32,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; @@ -158,6 +159,47 @@ public void testSerializerDuplication() throws Exception { assertNotSame(serializerA, serializerB); } + // ------------------------------------------------------------------------ + // Test hashCode() and equals() + // ------------------------------------------------------------------------ + + @Test + public void testHashCodeAndEquals() throws Exception { + final String name = "testName"; + + TestStateDescriptor original = new TestStateDescriptor<>(name, String.class); + TestStateDescriptor same = new TestStateDescriptor<>(name, String.class); + TestStateDescriptor sameBySerializer = new TestStateDescriptor<>(name, StringSerializer.INSTANCE); + + // test that hashCode() works on state descriptors with initialized and uninitialized serializers + assertEquals(original.hashCode(), same.hashCode()); + assertEquals(original.hashCode(), sameBySerializer.hashCode()); + + assertEquals(original, same); + assertEquals(original, sameBySerializer); + + // equality with a clone + TestStateDescriptor clone = CommonTestUtils.createCopySerializable(original); + assertEquals(original, clone); + + // equality with an initialized + clone.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, clone); + + original.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, same); + } + + @Test + public void testEqualsSameNameAndTypeDifferentClass() throws Exception { + final String name = "test name"; + + final TestStateDescriptor descr1 = new TestStateDescriptor<>(name, String.class); + final OtherTestStateDescriptor descr2 = new OtherTestStateDescriptor<>(name, String.class); + + assertNotEquals(descr1, descr2); + } + // ------------------------------------------------------------------------ // Mock implementations and test types // ------------------------------------------------------------------------ @@ -185,17 +227,34 @@ public State bind(StateBinder stateBinder) throws Exception { @Override public Type getType() { - throw new UnsupportedOperationException(); + return Type.VALUE; + } + } + + private static class OtherTestStateDescriptor extends StateDescriptor { + + private static final long serialVersionUID = 1L; + + OtherTestStateDescriptor(String name, TypeSerializer serializer) { + super(name, serializer, null); + } + + OtherTestStateDescriptor(String name, TypeInformation typeInfo) { + super(name, typeInfo, null); + } + + OtherTestStateDescriptor(String name, Class type) { + super(name, type, null); } @Override - public int hashCode() { - return 584523; + public State bind(StateBinder stateBinder) throws Exception { + throw new UnsupportedOperationException(); } @Override - public boolean equals(Object o) { - return o != null && o.getClass() == TestStateDescriptor.class; + public Type getType() { + return Type.VALUE; } } } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java index 67114e50af81b1..3870da05da2428 100644 --- a/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java +++ b/flink-core/src/test/java/org/apache/flink/api/common/state/ValueStateDescriptorTest.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.core.testutils.CommonTestUtils; @@ -35,6 +36,33 @@ */ public class ValueStateDescriptorTest extends TestLogger { + @Test + public void testHashCodeEquals() throws Exception { + final String name = "testName"; + + ValueStateDescriptor original = new ValueStateDescriptor<>(name, String.class); + ValueStateDescriptor same = new ValueStateDescriptor<>(name, String.class); + ValueStateDescriptor sameBySerializer = new ValueStateDescriptor<>(name, StringSerializer.INSTANCE); + + // test that hashCode() works on state descriptors with initialized and uninitialized serializers + assertEquals(original.hashCode(), same.hashCode()); + assertEquals(original.hashCode(), sameBySerializer.hashCode()); + + assertEquals(original, same); + assertEquals(original, sameBySerializer); + + // equality with a clone + ValueStateDescriptor clone = CommonTestUtils.createCopySerializable(original); + assertEquals(original, clone); + + // equality with an initialized + clone.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, clone); + + original.initializeSerializerUnlessSet(new ExecutionConfig()); + assertEquals(original, same); + } + @Test public void testVeryLargeDefaultValue() throws Exception { // ensure that we correctly read very large data when deserializing the default value From 37a114875afb9352e6f7b10e2729a94d0eeb72ee Mon Sep 17 00:00:00 2001 From: Shuyi Chen Date: Tue, 6 Feb 2018 00:50:21 -0800 Subject: [PATCH 0227/2294] [FLINK-8562] [tests] Fix YARNSessionFIFOSecuredITCase Before the YARNSessionFIFOSecuredITCase also passed without Kerberos being active. This closes #5416. --- .../flink/yarn/YARNSessionFIFOITCase.java | 13 ++- .../yarn/YARNSessionFIFOSecuredITCase.java | 13 +++ .../flink/yarn/YarnConfigurationITCase.java | 2 +- .../org/apache/flink/yarn/YarnTestBase.java | 79 ++++++++++++++----- 4 files changed, 86 insertions(+), 21 deletions(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java index b3dcaca1459656..464e73c873e4e0 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.apache.flink.yarn.UtilsTest.addTestAppender; import static org.apache.flink.yarn.UtilsTest.checkForLogString; @@ -106,8 +107,16 @@ public void testDetachedMode() throws InterruptedException, IOException { } } - //additional sleep for the JM/TM to start and establish connection - sleep(2000); + // additional sleep for the JM/TM to start and establish connection + long startTime = System.nanoTime(); + while (System.nanoTime() - startTime < TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS) && + !(verifyStringsInNamedLogFiles( + new String[]{"YARN Application Master started"}, "jobmanager.log") && + verifyStringsInNamedLogFiles( + new String[]{"Starting TaskManager actor"}, "taskmanager.log"))) { + LOG.info("Still waiting for JM/TM to initialize..."); + sleep(500); + } LOG.info("Two containers are running. Killing the application"); // kill application "externally". diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java index 3954f8ab73e52d..18e1c3abff0db4 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java @@ -30,10 +30,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.Collections; import java.util.concurrent.Callable; @@ -97,6 +99,17 @@ public static void teardownSecureCluster() throws Exception { SecureTestEnvironment.cleanup(); } + @Override + public void testDetachedMode() throws InterruptedException, IOException { + super.testDetachedMode(); + if (!verifyStringsInNamedLogFiles( + new String[]{"Login successful for user", "using keytab file"}, "jobmanager.log") || + !verifyStringsInNamedLogFiles( + new String[]{"Login successful for user", "using keytab file"}, "taskmanager.log")) { + Assert.fail("Can not find expected strings in log files."); + } + } + /* For secure cluster testing, it is enough to run only one test and override below test methods * to keep the overall build time minimal */ diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java index 2a1b099399ac0b..635fdf3d0959aa 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java @@ -78,7 +78,7 @@ public class YarnConfigurationITCase extends YarnTestBase { @Test(timeout = 60000) public void testFlinkContainerMemory() throws Exception { final YarnClient yarnClient = getYarnClient(); - final Configuration configuration = new Configuration(flinkConfiguration); + final Configuration configuration = new Configuration(flinkConfiguration.clone()); final int masterMemory = 64; final int taskManagerMemory = 128; diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index 3ec805e5058c5c..803f89cd931b59 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -21,7 +21,9 @@ import org.apache.flink.client.cli.CliFrontend; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.configuration.SecurityOptions; +import org.apache.flink.runtime.clusterframework.BootstrapTools; import org.apache.flink.test.util.TestBaseUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.util.TestLogger; @@ -57,7 +59,6 @@ import javax.annotation.Nullable; -import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; @@ -68,17 +69,20 @@ import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.PrintStream; -import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Scanner; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; +import static org.apache.flink.configuration.CoreOptions.OLD_MODE; + /** * This base class allows to use the MiniYARNCluster. * The cluster is re-used for all tests. @@ -145,7 +149,7 @@ public abstract class YarnTestBase extends TestLogger { private YarnClient yarnClient = null; - protected org.apache.flink.configuration.Configuration flinkConfiguration; + protected static org.apache.flink.configuration.Configuration flinkConfiguration; protected boolean flip6; @@ -213,8 +217,6 @@ public void checkClusterEmpty() throws IOException, YarnException { } } - flinkConfiguration = new org.apache.flink.configuration.Configuration(); - flip6 = CoreOptions.FLIP6_MODE.equalsIgnoreCase(flinkConfiguration.getString(CoreOptions.MODE)); } @@ -397,6 +399,51 @@ public boolean accept(File dir, String name) { } } + public static boolean verifyStringsInNamedLogFiles( + final String[] mustHave, final String fileName) { + List mustHaveList = Arrays.asList(mustHave); + File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY)); + if (!cwd.exists() || !cwd.isDirectory()) { + return false; + } + + File foundFile = findFile(cwd.getAbsolutePath(), new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (fileName != null && !name.equals(fileName)) { + return false; + } + File f = new File(dir.getAbsolutePath() + "/" + name); + LOG.info("Searching in {}", f.getAbsolutePath()); + try { + Set foundSet = new HashSet<>(mustHave.length); + Scanner scanner = new Scanner(f); + while (scanner.hasNextLine()) { + final String lineFromFile = scanner.nextLine(); + for (String str : mustHave) { + if (lineFromFile.contains(str)) { + foundSet.add(str); + } + } + if (foundSet.containsAll(mustHaveList)) { + return true; + } + } + } catch (FileNotFoundException e) { + LOG.warn("Unable to locate file: " + e.getMessage() + " file: " + f.getAbsolutePath()); + } + return false; + } + }); + + if (foundFile != null) { + LOG.info("Found string {} in {}.", Arrays.toString(mustHave), foundFile.getAbsolutePath()); + return true; + } else { + return false; + } + } + public static void sleep(int time) { try { Thread.sleep(time); @@ -465,27 +512,23 @@ private static void start(YarnConfiguration conf, String principal, String keyta File flinkConfDirPath = findFile(flinkDistRootDir, new ContainsName(new String[]{"flink-conf.yaml"})); Assert.assertNotNull(flinkConfDirPath); + flinkConfiguration = + GlobalConfiguration.loadConfiguration(); if (!StringUtils.isBlank(principal) && !StringUtils.isBlank(keytab)) { + //copy conf dir to test temporary workspace location tempConfPathForSecureRun = tmp.newFolder("conf"); String confDirPath = flinkConfDirPath.getParentFile().getAbsolutePath(); FileUtils.copyDirectory(new File(confDirPath), tempConfPathForSecureRun); - try (FileWriter fw = new FileWriter(new File(tempConfPathForSecureRun, "flink-conf.yaml"), true); - BufferedWriter bw = new BufferedWriter(fw); - PrintWriter out = new PrintWriter(bw)) { - - LOG.info("writing keytab: " + keytab + " and principal: " + principal + " to config file"); - out.println(""); - out.println("#Security Configurations Auto Populated "); - out.println(SecurityOptions.KERBEROS_LOGIN_KEYTAB.key() + ": " + keytab); - out.println(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key() + ": " + principal); - out.println(""); - } catch (IOException e) { - throw new RuntimeException("Exception occured while trying to append the security configurations.", e); - } + flinkConfiguration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB.key(), keytab); + flinkConfiguration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key(), principal); + flinkConfiguration.setString(CoreOptions.MODE.key(), OLD_MODE); + + BootstrapTools.writeConfiguration(flinkConfiguration, + new File(tempConfPathForSecureRun, "flink-conf.yaml")); String configDir = tempConfPathForSecureRun.getAbsolutePath(); From b550ac67fbf525863d5812d9d2a1010672a0169b Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 22 Mar 2018 13:16:10 +0100 Subject: [PATCH 0228/2294] [FLINK-8562] [tests] Introduce private global configuration to YarnTestBase --- .../yarn/YARNSessionFIFOSecuredITCase.java | 18 ++++++++++----- .../flink/yarn/YarnConfigurationITCase.java | 2 +- .../org/apache/flink/yarn/YarnTestBase.java | 22 +++++++++++-------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java index 18e1c3abff0db4..46a37a0f8fdf82 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.hamcrest.Matchers; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -102,12 +103,17 @@ public static void teardownSecureCluster() throws Exception { @Override public void testDetachedMode() throws InterruptedException, IOException { super.testDetachedMode(); - if (!verifyStringsInNamedLogFiles( - new String[]{"Login successful for user", "using keytab file"}, "jobmanager.log") || - !verifyStringsInNamedLogFiles( - new String[]{"Login successful for user", "using keytab file"}, "taskmanager.log")) { - Assert.fail("Can not find expected strings in log files."); - } + final String[] mustHave = {"Login successful for user", "using keytab file"}; + final boolean jobManagerRunsWithKerberos = verifyStringsInNamedLogFiles( + mustHave, + "jobmanager.log"); + final boolean taskManagerRunsWithKerberos = verifyStringsInNamedLogFiles( + mustHave, "taskmanager.log"); + + Assert.assertThat( + "The JobManager and the TaskManager should both run with Kerberos.", + jobManagerRunsWithKerberos && taskManagerRunsWithKerberos, + Matchers.is(true)); } /* For secure cluster testing, it is enough to run only one test and override below test methods diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java index 635fdf3d0959aa..2a1b099399ac0b 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java @@ -78,7 +78,7 @@ public class YarnConfigurationITCase extends YarnTestBase { @Test(timeout = 60000) public void testFlinkContainerMemory() throws Exception { final YarnClient yarnClient = getYarnClient(); - final Configuration configuration = new Configuration(flinkConfiguration.clone()); + final Configuration configuration = new Configuration(flinkConfiguration); final int masterMemory = 64; final int taskManagerMemory = 128; diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index 803f89cd931b59..73abc874b915fb 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -149,7 +149,9 @@ public abstract class YarnTestBase extends TestLogger { private YarnClient yarnClient = null; - protected static org.apache.flink.configuration.Configuration flinkConfiguration; + private static org.apache.flink.configuration.Configuration globalConfiguration; + + protected org.apache.flink.configuration.Configuration flinkConfiguration; protected boolean flip6; @@ -217,6 +219,7 @@ public void checkClusterEmpty() throws IOException, YarnException { } } + flinkConfiguration = new org.apache.flink.configuration.Configuration(globalConfiguration); flip6 = CoreOptions.FLIP6_MODE.equalsIgnoreCase(flinkConfiguration.getString(CoreOptions.MODE)); } @@ -512,23 +515,24 @@ private static void start(YarnConfiguration conf, String principal, String keyta File flinkConfDirPath = findFile(flinkDistRootDir, new ContainsName(new String[]{"flink-conf.yaml"})); Assert.assertNotNull(flinkConfDirPath); - flinkConfiguration = - GlobalConfiguration.loadConfiguration(); + + final String confDirPath = flinkConfDirPath.getParentFile().getAbsolutePath(); + globalConfiguration = GlobalConfiguration.loadConfiguration(confDirPath); if (!StringUtils.isBlank(principal) && !StringUtils.isBlank(keytab)) { //copy conf dir to test temporary workspace location tempConfPathForSecureRun = tmp.newFolder("conf"); - String confDirPath = flinkConfDirPath.getParentFile().getAbsolutePath(); FileUtils.copyDirectory(new File(confDirPath), tempConfPathForSecureRun); - flinkConfiguration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB.key(), keytab); - flinkConfiguration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key(), principal); - flinkConfiguration.setString(CoreOptions.MODE.key(), OLD_MODE); + globalConfiguration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB.key(), keytab); + globalConfiguration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key(), principal); + globalConfiguration.setString(CoreOptions.MODE.key(), OLD_MODE); - BootstrapTools.writeConfiguration(flinkConfiguration, - new File(tempConfPathForSecureRun, "flink-conf.yaml")); + BootstrapTools.writeConfiguration( + globalConfiguration, + new File(tempConfPathForSecureRun, "flink-conf.yaml")); String configDir = tempConfPathForSecureRun.getAbsolutePath(); From 93d99fdb39d6c2bd7715d7b6d4352ce3d895d9f8 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 22 Mar 2018 17:15:36 +0100 Subject: [PATCH 0229/2294] [hotfix] Remove unnecessary transient modifiers in CheckpointStatsTracker --- .../flink/runtime/checkpoint/CheckpointStatsTracker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java index e6386ad74d6246..9be1f695814035 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTracker.java @@ -59,7 +59,7 @@ public class CheckpointStatsTracker { * from a single Thread at a time and there can be multiple concurrent read * accesses to the latest stats snapshot. * - * Currently, writes are executed by whatever Thread executes the coordinator + *

    Currently, writes are executed by whatever Thread executes the coordinator * actions (which already happens in locked scope). Reads can come from * multiple concurrent Netty event loop Threads of the web runtime monitor. */ @@ -81,7 +81,7 @@ public class CheckpointStatsTracker { private final CheckpointStatsHistory history; /** The job vertices taking part in the checkpoints. */ - private final transient List jobVertices; + private final List jobVertices; /** The latest restored checkpoint. */ @Nullable @@ -99,7 +99,7 @@ public class CheckpointStatsTracker { /** The latest completed checkpoint. Used by the latest completed checkpoint metrics. */ @Nullable - private volatile transient CompletedCheckpointStats latestCompletedCheckpoint; + private volatile CompletedCheckpointStats latestCompletedCheckpoint; /** * Creates a new checkpoint stats tracker. From 9198c93e59d7b5bd916cabcf6bb8c52fd0bdfae2 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 22 Mar 2018 18:38:56 +0100 Subject: [PATCH 0230/2294] [hotfix] Add generics to FutureUtils.toJava calls in ClusterClient --- .../org/apache/flink/client/program/ClusterClient.java | 10 +++++----- .../apache/flink/runtime/concurrent/FutureUtils.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index b0c50e59d77052..166d9770b48bb1 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -602,7 +602,7 @@ public CompletableFuture getJobStatus(JobID jobId) { Future response = jobManager.ask(JobManagerMessages.getRequestJobStatus(jobId), timeout); - CompletableFuture javaFuture = FutureUtils.toJava(response); + CompletableFuture javaFuture = FutureUtils.toJava(response); return javaFuture.thenApply((responseMessage) -> { if (responseMessage instanceof JobManagerMessages.CurrentJobStatus) { @@ -707,9 +707,9 @@ public void stop(final JobID jobId) throws Exception { public CompletableFuture triggerSavepoint(JobID jobId, @Nullable String savepointDirectory) throws FlinkException { final ActorGateway jobManager = getJobManagerGateway(); - Future response = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobId, Option.apply(savepointDirectory)), + Future response = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobId, Option.apply(savepointDirectory)), new FiniteDuration(1, TimeUnit.HOURS)); - CompletableFuture responseFuture = FutureUtils.toJava(response); + CompletableFuture responseFuture = FutureUtils.toJava(response); return responseFuture.thenApply((responseMessage) -> { if (responseMessage instanceof JobManagerMessages.TriggerSavepointSuccess) { @@ -729,7 +729,7 @@ public CompletableFuture disposeSavepoint(String savepointPath, Tim final ActorGateway jobManager = getJobManagerGateway(); Object msg = new JobManagerMessages.DisposeSavepoint(savepointPath); - CompletableFuture responseFuture = FutureUtils.toJava( + CompletableFuture responseFuture = FutureUtils.toJava( jobManager.ask( msg, FutureUtils.toFiniteDuration(timeout))); @@ -768,7 +768,7 @@ public CompletableFuture> listJobs() throws Excepti final ActorGateway jobManager = getJobManagerGateway(); Future response = jobManager.ask(new RequestJobDetails(true, false), timeout); - CompletableFuture responseFuture = FutureUtils.toJava(response); + CompletableFuture responseFuture = FutureUtils.toJava(response); return responseFuture.thenApply((responseMessage) -> { if (responseMessage instanceof MultipleJobsDetails) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java index a2d0710e87977d..e0164a92f6a992 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java @@ -750,7 +750,7 @@ public static CompletableFuture toJava(Future scalaFuture) { scalaFuture.onComplete(new OnComplete() { @Override - public void onComplete(Throwable failure, T success) throws Throwable { + public void onComplete(Throwable failure, T success) { if (failure != null) { result.completeExceptionally(failure); } else { From cc94090ea29b5f33a0a110d7af46dfc2c7e8610a Mon Sep 17 00:00:00 2001 From: liurenjie1024 Date: Mon, 12 Mar 2018 15:43:26 +0800 Subject: [PATCH 0231/2294] [FLINK-8919] [table] Add KeyedProcessFunctionWithCleanupState. This closes #5680. --- ...KeyedProcessFunctionWithCleanupState.scala | 85 ++++++++++++ ...dProcessFunctionWithCleanupStateTest.scala | 126 +++++++++++++++++ .../ProcessFunctionWithCleanupStateTest.scala | 131 ++++++++++++++++++ 3 files changed, 342 insertions(+) create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala new file mode 100644 index 00000000000000..4d6840a3f43e09 --- /dev/null +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.aggregate + +import java.lang.{Long => JLong} +import org.apache.flink.api.common.state.{State, ValueState, ValueStateDescriptor} +import org.apache.flink.streaming.api.TimeDomain +import org.apache.flink.streaming.api.functions.{KeyedProcessFunction, ProcessFunction} +import org.apache.flink.table.api.{StreamQueryConfig, Types} + +abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: StreamQueryConfig) + extends KeyedProcessFunction[K, I, O] { + protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime + protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime + protected val stateCleaningEnabled: Boolean = minRetentionTime > 1 + + // holds the latest registered cleanup timer + private var cleanupTimeState: ValueState[JLong] = _ + + protected def initCleanupTimeState(stateName: String) { + if (stateCleaningEnabled) { + val inputCntDescriptor: ValueStateDescriptor[JLong] = + new ValueStateDescriptor[JLong](stateName, Types.LONG) + cleanupTimeState = getRuntimeContext.getState(inputCntDescriptor) + } + } + + protected def registerProcessingCleanupTimer( + ctx: KeyedProcessFunction[K, I, O]#Context, + currentTime: Long): Unit = { + if (stateCleaningEnabled) { + + // last registered timer + val curCleanupTime = cleanupTimeState.value() + + // check if a cleanup timer is registered and + // that the current cleanup timer won't delete state we need to keep + if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) { + // we need to register a new (later) timer + val cleanupTime = currentTime + maxRetentionTime + // register timer and remember clean-up time + ctx.timerService().registerProcessingTimeTimer(cleanupTime) + cleanupTimeState.update(cleanupTime) + } + } + } + + protected def isProcessingTimeTimer(ctx: OnTimerContext): Boolean = { + ctx.timeDomain() == TimeDomain.PROCESSING_TIME + } + + protected def needToCleanupState(timestamp: Long): Boolean = { + if (stateCleaningEnabled) { + val cleanupTime = cleanupTimeState.value() + // check that the triggered timer is the last registered processing time timer. + null != cleanupTime && timestamp == cleanupTime + } else { + false + } + } + + protected def cleanupState(states: State*): Unit = { + // clear all state + states.foreach(_.clear()) + if (stateCleaningEnabled) { + this.cleanupTimeState.clear() + } + } +} diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala new file mode 100644 index 00000000000000..c896666b3f758b --- /dev/null +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators + +import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} +import org.apache.flink.api.common.time.Time +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.KeyedProcessFunction +import org.apache.flink.streaming.api.operators.KeyedProcessOperator +import org.apache.flink.table.api.StreamQueryConfig +import org.apache.flink.table.runtime.aggregate.KeyedProcessFunctionWithCleanupState +import org.apache.flink.table.runtime.harness.HarnessTestBase +import org.apache.flink.util.Collector + +import org.junit.Test +import org.junit.Assert.assertEquals + +class KeyedProcessFunctionWithCleanupStateTest extends HarnessTestBase { + + @Test + def testStateCleaning(): Unit = { + val queryConfig = new StreamQueryConfig() + .withIdleStateRetentionTime(Time.milliseconds(5), Time.milliseconds(10)) + + val func = new MockedKeyedProcessFunction(queryConfig) + val operator = new KeyedProcessOperator(func) + + val testHarness = createHarnessTester(operator, + new FirstFieldSelector, + TypeInformation.of(classOf[String])) + + testHarness.open() + + testHarness.setProcessingTime(1) + // add state for key "a" + testHarness.processElement(("a", "payload"), 1) + // add state for key "b" + testHarness.processElement(("b", "payload"), 1) + + // check that we have two states (a, b) + // we check for the double number of states, because KeyedProcessFunctionWithCleanupState + // adds one more state per key to hold the cleanup timestamp. + assertEquals(4, testHarness.numKeyedStateEntries()) + + // advance time and add state for key "c" + testHarness.setProcessingTime(5) + testHarness.processElement(("c", "payload"), 1) + // add state for key "a". Timer is not reset, because it is still within minRetentionTime + testHarness.processElement(("a", "payload"), 1) + + // check that we have three states (a, b, c) + assertEquals(6, testHarness.numKeyedStateEntries()) + + // advance time and update key "b". Timer for "b" is reset to 18 + testHarness.setProcessingTime(8) + testHarness.processElement(("b", "payload"), 1) + // check that we have three states (a, b, c) + assertEquals(6, testHarness.numKeyedStateEntries()) + + // advance time to clear state for key "a" + testHarness.setProcessingTime(11) + // check that we have two states (b, c) + assertEquals(4, testHarness.numKeyedStateEntries()) + + // advance time to clear state for key "c" + testHarness.setProcessingTime(15) + // check that we have one state (b) + assertEquals(2, testHarness.numKeyedStateEntries()) + + // advance time to clear state for key "c" + testHarness.setProcessingTime(18) + // check that we have no states + assertEquals(0, testHarness.numKeyedStateEntries()) + + testHarness.close() + } +} + +private class MockedKeyedProcessFunction(queryConfig: StreamQueryConfig) + extends KeyedProcessFunctionWithCleanupState[String, (String, String), String](queryConfig) { + + var state: ValueState[String] = _ + + override def open(parameters: Configuration): Unit = { + initCleanupTimeState("CleanUpState") + val stateDesc = new ValueStateDescriptor[String]("testState", classOf[String]) + state = getRuntimeContext.getState(stateDesc) + } + + override def processElement( + value: (String, String), + ctx: KeyedProcessFunction[String, (String, String), String]#Context, + out: Collector[String]): Unit = { + + val curTime = ctx.timerService().currentProcessingTime() + registerProcessingCleanupTimer(ctx, curTime) + state.update(value._2) + } + + override def onTimer( + timestamp: Long, + ctx: KeyedProcessFunction[String, (String, String), String]#OnTimerContext, + out: Collector[String]): Unit = { + + if (needToCleanupState(timestamp)) { + cleanupState(state) + } + } +} diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala new file mode 100644 index 00000000000000..e773f4bee9be1a --- /dev/null +++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators + +import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} +import org.apache.flink.api.common.time.Time +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.functions.KeySelector +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator +import org.apache.flink.table.api.StreamQueryConfig +import org.apache.flink.table.runtime.aggregate.ProcessFunctionWithCleanupState +import org.apache.flink.table.runtime.harness.HarnessTestBase +import org.apache.flink.util.Collector +import org.junit.Assert.assertEquals +import org.junit.Test + +class ProcessFunctionWithCleanupStateTest extends HarnessTestBase { + + @Test + def testStateCleaning(): Unit = { + val queryConfig = new StreamQueryConfig() + .withIdleStateRetentionTime(Time.milliseconds(5), Time.milliseconds(10)) + + val func = new MockedProcessFunction(queryConfig) + val operator = new LegacyKeyedProcessOperator(func) + + val testHarness = createHarnessTester(operator, + new FirstFieldSelector, + TypeInformation.of(classOf[String])) + + testHarness.open() + + testHarness.setProcessingTime(1) + // add state for key "a" + testHarness.processElement(("a", "payload"), 1) + // add state for key "b" + testHarness.processElement(("b", "payload"), 1) + + // check that we have two states (a, b) + // we check for the double number of states, because KeyedProcessFunctionWithCleanupState + // adds one more state per key to hold the cleanup timestamp. + assertEquals(4, testHarness.numKeyedStateEntries()) + + // advance time and add state for key "c" + testHarness.setProcessingTime(5) + testHarness.processElement(("c", "payload"), 1) + // add state for key "a". Timer is not reset, because it is still within minRetentionTime + testHarness.processElement(("a", "payload"), 1) + + // check that we have three states (a, b, c) + assertEquals(6, testHarness.numKeyedStateEntries()) + + // advance time and update key "b". Timer for "b" is reset to 18 + testHarness.setProcessingTime(8) + testHarness.processElement(("b", "payload"), 1) + // check that we have three states (a, b, c) + assertEquals(6, testHarness.numKeyedStateEntries()) + + // advance time to clear state for key "a" + testHarness.setProcessingTime(11) + // check that we have two states (b, c) + assertEquals(4, testHarness.numKeyedStateEntries()) + + // advance time to clear state for key "c" + testHarness.setProcessingTime(15) + // check that we have one state (b) + assertEquals(2, testHarness.numKeyedStateEntries()) + + // advance time to clear state for key "c" + testHarness.setProcessingTime(18) + // check that we have no states + assertEquals(0, testHarness.numKeyedStateEntries()) + + testHarness.close() + } +} + +private class MockedProcessFunction(queryConfig: StreamQueryConfig) + extends ProcessFunctionWithCleanupState[(String, String), String](queryConfig) { + + var state: ValueState[String] = _ + + override def open(parameters: Configuration): Unit = { + initCleanupTimeState("CleanUpState") + val stateDesc = new ValueStateDescriptor[String]("testState", classOf[String]) + state = getRuntimeContext.getState(stateDesc) + } + + override def processElement( + value: (String, String), + ctx: ProcessFunction[(String, String), String]#Context, + out: Collector[String]): Unit = { + + val curTime = ctx.timerService().currentProcessingTime() + registerProcessingCleanupTimer(ctx, curTime) + state.update(value._2) + } + + override def onTimer( + timestamp: Long, + ctx: ProcessFunction[(String, String), String]#OnTimerContext, + out: Collector[String]): Unit = { + + if (needToCleanupState(timestamp)) { + cleanupState(state) + } + } +} + +private class FirstFieldSelector extends KeySelector[(String, String), String] { + override def getKey(value: (String, String)): String = value._1 +} + From 6384aa7a3b2a752ebf41475748288ab1489fafa7 Mon Sep 17 00:00:00 2001 From: yanghua Date: Thu, 22 Mar 2018 09:58:08 +0800 Subject: [PATCH 0232/2294] [FLINK-8931] TASK_KILLING is not covered by match in TaskMonitor#whenUnhandled This closes #5744. --- .../scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala index 7840fd479bd246..76a2a90e1fc072 100644 --- a/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala +++ b/flink-mesos/src/main/scala/org/apache/flink/mesos/scheduler/TaskMonitor.scala @@ -163,6 +163,7 @@ class TaskMonitor( LOG.warn(s"Mesos task ${goal.taskID.getValue} failed unexpectedly.") context.parent ! TaskTerminated(goal.taskID, msg.status()) stop() + case TASK_KILLING => stay() } case Event(msg: StatusUpdate, StateData(goal: Released)) => @@ -175,6 +176,7 @@ class TaskMonitor( LOG.info(s"Mesos task ${goal.taskID.getValue} exited as planned.") context.parent ! TaskTerminated(goal.taskID, msg.status()) stop() + case TASK_KILLING => stay() } } From 0f87968e8e8989fe9ab575dbaa1623c781db5551 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 15:09:46 +0100 Subject: [PATCH 0233/2294] [hotfix] Improve Flip-6 component logging --- .../flink/runtime/jobmaster/JobMaster.java | 4 ++-- .../runtime/jobmaster/slotpool/SlotPool.java | 5 +++-- .../resourcemanager/ResourceManager.java | 17 ++++++++--------- .../slotmanager/ResourceActions.java | 3 ++- .../slotmanager/SlotManager.java | 8 ++++---- .../runtime/taskexecutor/TaskExecutor.java | 2 ++ .../slotmanager/SlotManagerTest.java | 8 ++++---- 7 files changed, 25 insertions(+), 22 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index 6878032f0e17f6..bc18fab7e39adf 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -689,6 +689,8 @@ public CompletableFuture scheduleOrUpdateConsumers( @Override public CompletableFuture disconnectTaskManager(final ResourceID resourceID, final Exception cause) { + log.debug("Disconnect TaskExecutor {} because: {}", resourceID, cause.getMessage()); + taskManagerHeartbeatManager.unmonitorTarget(resourceID); CompletableFuture releaseFuture = slotPoolGateway.releaseTaskManager(resourceID); @@ -1516,8 +1518,6 @@ private TaskManagerHeartbeatListener(JobMasterGateway jobMasterGateway) { @Override public void notifyHeartbeatTimeout(ResourceID resourceID) { - log.info("Heartbeat of TaskManager with id {} timed out.", resourceID); - jobMasterGateway.disconnectTaskManager( resourceID, new TimeoutException("Heartbeat of TaskManager with id " + resourceID + " timed out.")); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java index 42264b53219898..3e8b7882395473 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java @@ -322,7 +322,6 @@ private CompletableFuture internalAllocateSlot( SlotProfile slotProfile, boolean allowQueuedScheduling, Time allocationTimeout) { - final SlotSharingGroupId slotSharingGroupId = task.getSlotSharingGroupId(); if (slotSharingGroupId != null) { @@ -955,7 +954,7 @@ public CompletableFuture offerSlot( } final AllocatedSlot allocatedSlot = new AllocatedSlot( - slotOffer.getAllocationId(), + allocationID, taskManagerLocation, slotOffer.getSlotIndex(), slotOffer.getResourceProfile(), @@ -971,6 +970,8 @@ public CompletableFuture offerSlot( // we could not complete the pending slot future --> try to fulfill another pending request allocatedSlots.remove(pendingRequest.getSlotRequestId()); tryFulfillSlotRequestOrMakeAvailable(allocatedSlot); + } else { + log.debug("Fulfilled slot request {} with allocated slot {}.", pendingRequest.getSlotRequestId(), allocationID); } } else { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java index 0ae4ab6af3a08e..cae9c6cdb383c9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java @@ -682,7 +682,7 @@ private RegistrationResponse registerTaskExecutorInternal( WorkerRegistration oldRegistration = taskExecutors.remove(taskExecutorResourceId); if (oldRegistration != null) { // TODO :: suggest old taskExecutor to stop itself - log.info("Replacing old instance of worker for ResourceID {}", taskExecutorResourceId); + log.info("Replacing old registration of TaskExecutor {}.", taskExecutorResourceId); // remove old task manager registration from slot manager slotManager.unregisterTaskManager(oldRegistration.getInstanceID()); @@ -779,14 +779,14 @@ protected void closeTaskManagerConnection(final ResourceID resourceID, final Exc WorkerRegistration workerRegistration = taskExecutors.remove(resourceID); if (workerRegistration != null) { - log.info("Task manager {} failed because {}.", resourceID, cause.getMessage()); + log.info("Closing TaskExecutor connection {} because: {}", resourceID, cause.getMessage()); // TODO :: suggest failed task executor to stop itself slotManager.unregisterTaskManager(workerRegistration.getInstanceID()); workerRegistration.getTaskExecutorGateway().disconnectResourceManager(cause); } else { - log.debug("Could not find a registered task manager with the process id {}.", resourceID); + log.debug("No open TaskExecutor connection {}. Ignoring close TaskExecutor connection.", resourceID); } } @@ -816,7 +816,7 @@ protected void jobLeaderLostLeadership(JobID jobId, JobMasterId oldJobMasterId) } } - protected void releaseResource(InstanceID instanceId) { + protected void releaseResource(InstanceID instanceId, Exception cause) { WorkerType worker = null; // TODO: Improve performance by having an index on the instanceId @@ -829,10 +829,9 @@ protected void releaseResource(InstanceID instanceId) { if (worker != null) { if (stopWorker(worker)) { - closeTaskManagerConnection(worker.getResourceID(), - new FlinkException("Worker was stopped.")); + closeTaskManagerConnection(worker.getResourceID(), cause); } else { - log.debug("Worker {} was not stopped.", worker.getResourceID()); + log.debug("Worker {} could not be stopped.", worker.getResourceID()); } } else { // unregister in order to clean up potential left over state @@ -990,10 +989,10 @@ protected abstract void shutDownApplication( private class ResourceActionsImpl implements ResourceActions { @Override - public void releaseResource(InstanceID instanceId) { + public void releaseResource(InstanceID instanceId, Exception cause) { validateRunsInMainThread(); - ResourceManager.this.releaseResource(instanceId); + ResourceManager.this.releaseResource(instanceId, cause); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/ResourceActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/ResourceActions.java index 753e5e2a09218b..84e7c4e785d48e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/ResourceActions.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/ResourceActions.java @@ -33,8 +33,9 @@ public interface ResourceActions { * Releases the resource with the given instance id. * * @param instanceId identifying which resource to release + * @param cause why the resource is released */ - void releaseResource(InstanceID instanceId); + void releaseResource(InstanceID instanceId, Exception cause); /** * Requests to allocate a resource with the given {@link ResourceProfile}. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java index 120e1aa1a5ac3e..6cdd997c38d036 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManager.java @@ -36,6 +36,7 @@ import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway; import org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException; import org.apache.flink.runtime.taskexecutor.exceptions.SlotOccupiedException; +import org.apache.flink.util.FlinkException; import org.apache.flink.util.Preconditions; import org.slf4j.Logger; @@ -879,8 +880,6 @@ private void checkTaskManagerTimeouts() { // first retrieve the timed out TaskManagers for (TaskManagerRegistration taskManagerRegistration : taskManagerRegistrations.values()) { - LOG.debug("Evaluating TaskManager {} for idleness.", taskManagerRegistration.getInstanceId()); - if (currentTime - taskManagerRegistration.getIdleSince() >= taskManagerTimeout.toMilliseconds()) { // we collect the instance ids first in order to avoid concurrent modifications by the // ResourceActions.releaseResource call @@ -890,7 +889,8 @@ private void checkTaskManagerTimeouts() { // second we trigger the release resource callback which can decide upon the resource release for (InstanceID timedOutTaskManagerId : timedOutTaskManagerIds) { - resourceActions.releaseResource(timedOutTaskManagerId); + LOG.debug("Release TaskExecutor {} because it exceeded the idle timeout.", timedOutTaskManagerId); + resourceActions.releaseResource(timedOutTaskManagerId, new FlinkException("TaskExecutor exceeded the idle timeout.")); } } } @@ -976,7 +976,7 @@ public void unregisterTaskManagersAndReleaseResources() { internalUnregisterTaskManager(taskManagerRegistration); - resourceActions.releaseResource(taskManagerRegistration.getInstanceId()); + resourceActions.releaseResource(taskManagerRegistration.getInstanceId(), new FlinkException("Triggering of SlotManager#unregisterTaskManagersAndReleaseResources.")); } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index f25601e534b0c0..3523992e2d405d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -1262,6 +1262,8 @@ private void unregisterTaskAndNotifyFinalState( private void freeSlotInternal(AllocationID allocationId, Throwable cause) { checkNotNull(allocationId); + log.debug("Free slot with allocation id {} because: {}", allocationId, cause.getMessage()); + try { final JobID jobId = taskSlotTable.getOwningJob(allocationId); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java index 4907756e910d34..90ed1648ad1811 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/slotmanager/SlotManagerTest.java @@ -685,7 +685,7 @@ public void run() { }); verify(resourceManagerActions, timeout(100L * tmTimeout).times(1)) - .releaseResource(eq(taskManagerConnection.getInstanceID())); + .releaseResource(eq(taskManagerConnection.getInstanceID()), any(Exception.class)); } } @@ -1027,13 +1027,13 @@ public void testTimeoutForUnusedTaskManager() throws Exception { assertTrue(idleFuture2.get()); - verify(resourceManagerActions, timeout(verifyTimeout).times(1)).releaseResource(eq(taskManagerConnection.getInstanceID())); + verify(resourceManagerActions, timeout(verifyTimeout).times(1)).releaseResource(eq(taskManagerConnection.getInstanceID()), any(Exception.class)); } } /** * Tests that a task manager timeout does not remove the slots from the SlotManager. - * A timeout should only trigger the {@link ResourceActions#releaseResource(InstanceID)} + * A timeout should only trigger the {@link ResourceActions#releaseResource(InstanceID, Exception)} * callback. The receiver of the callback can then decide what to do with the TaskManager. * * FLINK-7793 @@ -1064,7 +1064,7 @@ public void testTaskManagerTimeoutDoesNotRemoveSlots() throws Exception { assertEquals(1, slotManager.getNumberRegisteredSlots()); // wait for the timeout call to happen - verify(resourceActions, timeout(taskManagerTimeout.toMilliseconds() * 20L).atLeast(1)).releaseResource(eq(taskExecutorConnection.getInstanceID())); + verify(resourceActions, timeout(taskManagerTimeout.toMilliseconds() * 20L).atLeast(1)).releaseResource(eq(taskExecutorConnection.getInstanceID()), any(Exception.class)); assertEquals(1, slotManager.getNumberRegisteredSlots()); From 6c70b7bd8e15684358fddc802f2eb9e62c9a16ac Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 16:48:52 +0100 Subject: [PATCH 0234/2294] [FLINK-9047] Fix slot recycling in case of failed release In case that a slot cannot be released it will only recycled/reused if the owning TaskExecutor is still registered at the SlotPool. If this is not the case then we drop the slot from the SlotPool. This closes #5739. --- .../jobmaster/slotpool/SingleLogicalSlot.java | 10 +- .../runtime/jobmaster/slotpool/SlotPool.java | 12 +- .../utils/SimpleAckingTaskManagerGateway.java | 18 +-- .../jobmaster/slotpool/SlotPoolTest.java | 120 +++++++++++++++++- 4 files changed, 139 insertions(+), 21 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SingleLogicalSlot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SingleLogicalSlot.java index 9bd559bc166ef3..0736b5684ab5bb 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SingleLogicalSlot.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SingleLogicalSlot.java @@ -19,13 +19,13 @@ package org.apache.flink.runtime.jobmaster.slotpool; import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.instance.SlotSharingGroupId; +import org.apache.flink.runtime.jobmanager.scheduler.Locality; +import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; import org.apache.flink.runtime.jobmaster.LogicalSlot; import org.apache.flink.runtime.jobmaster.SlotContext; import org.apache.flink.runtime.jobmaster.SlotOwner; import org.apache.flink.runtime.jobmaster.SlotRequestId; -import org.apache.flink.runtime.instance.SlotSharingGroupId; -import org.apache.flink.runtime.jobmanager.scheduler.Locality; -import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import org.apache.flink.util.Preconditions; @@ -33,7 +33,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.function.Function; /** * Implementation of the {@link LogicalSlot} which is used by the {@link SlotPool}. @@ -127,8 +126,7 @@ public CompletableFuture releaseSlot(@Nullable Throwable cause) { // Wait until the payload has been terminated. Only then, we return the slot to its rightful owner return payload.getTerminalStateFuture() - .handle((Object ignored, Throwable throwable) -> slotOwner.returnAllocatedSlot(this)) - .thenApply(Function.identity()); + .whenComplete((Object ignored, Throwable throwable) -> slotOwner.returnAllocatedSlot(this)); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java index 3e8b7882395473..6040b418336957 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java @@ -1041,6 +1041,7 @@ else if (availableSlots.tryRemove(allocationID)) { */ @Override public CompletableFuture registerTaskManager(final ResourceID resourceID) { + log.debug("Register new TaskExecutor {}.", resourceID); registeredTaskManagers.add(resourceID); return CompletableFuture.completedFuture(Acknowledge.get()); @@ -1119,8 +1120,15 @@ private void checkIdleSlot() { freeSlotFuture.whenCompleteAsync( (Acknowledge ignored, Throwable throwable) -> { if (throwable != null) { - log.info("Releasing idle slot {} failed.", allocationID, throwable); - tryFulfillSlotRequestOrMakeAvailable(expiredSlot); + if (registeredTaskManagers.contains(expiredSlot.getTaskManagerId())) { + log.debug("Releasing slot {} of registered TaskExecutor {} failed. " + + "Trying to fulfill a different slot request.", allocationID, expiredSlot.getTaskManagerId(), + throwable); + tryFulfillSlotRequestOrMakeAvailable(expiredSlot); + } else { + log.debug("Releasing slot {} failed and owning TaskExecutor {} is no " + + "longer registered. Discarding slot.", allocationID, expiredSlot.getTaskManagerId()); + } } }, getMainThreadExecutor()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java index 628f0041371e53..5c62a7370967c7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java @@ -20,7 +20,6 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.time.Time; -import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.blob.TransientBlobKey; import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.clusterframework.ApplicationStatus; @@ -38,6 +37,7 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.function.BiFunction; import java.util.function.Consumer; /** @@ -52,7 +52,7 @@ public class SimpleAckingTaskManagerGateway implements TaskManagerGateway { private Optional> optCancelConsumer; - private volatile Consumer> freeSlotConsumer; + private volatile BiFunction> freeSlotFunction; public SimpleAckingTaskManagerGateway() { optSubmitConsumer = Optional.empty(); @@ -67,8 +67,8 @@ public void setCancelConsumer(Consumer predicate) { optCancelConsumer = Optional.of(predicate); } - public void setFreeSlotConsumer(Consumer> consumer) { - freeSlotConsumer = consumer; + public void setFreeSlotFunction(BiFunction> freeSlotFunction) { + this.freeSlotFunction = freeSlotFunction; } @Override @@ -150,12 +150,12 @@ public CompletableFuture requestTaskManagerStdout(Time timeout @Override public CompletableFuture freeSlot(AllocationID allocationId, Throwable cause, Time timeout) { - final Consumer> currentFreeSlotConsumer = freeSlotConsumer; + final BiFunction> currentFreeSlotFunction = freeSlotFunction; - if (currentFreeSlotConsumer != null) { - currentFreeSlotConsumer.accept(Tuple2.of(allocationId, cause)); + if (currentFreeSlotFunction != null) { + return currentFreeSlotFunction.apply(allocationId, cause); + } else { + return CompletableFuture.completedFuture(Acknowledge.get()); } - - return CompletableFuture.completedFuture(Acknowledge.get()); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java index c3819747595b8c..502b076e5714ae 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPoolTest.java @@ -23,6 +23,7 @@ import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; import org.apache.flink.runtime.clusterframework.types.SlotProfile; +import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobgraph.JobVertexID; @@ -38,6 +39,7 @@ import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.rpc.TestingRpcService; +import org.apache.flink.runtime.taskexecutor.TaskExecutor; import org.apache.flink.runtime.taskexecutor.slot.SlotOffer; import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; @@ -58,6 +60,7 @@ import javax.annotation.Nullable; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -67,6 +70,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static org.apache.flink.runtime.jobmaster.slotpool.AvailableSlotsTest.DEFAULT_TESTING_PROFILE; import static org.hamcrest.MatcherAssert.assertThat; @@ -559,9 +563,15 @@ public void testShutdownReleasesAllSlots() throws Exception { final ArrayBlockingQueue freedSlotQueue = new ArrayBlockingQueue<>(numSlotOffers); - taskManagerGateway.setFreeSlotConsumer(tuple -> { - while(!freedSlotQueue.offer(tuple.f0)) {} - }); + taskManagerGateway.setFreeSlotFunction( + (AllocationID allocationID, Throwable cause) -> { + try { + freedSlotQueue.put(allocationID); + return CompletableFuture.completedFuture(Acknowledge.get()); + } catch (InterruptedException e) { + return FutureUtils.completedExceptionally(e); + } + }); final CompletableFuture> acceptedSlotOffersFuture = slotPoolGateway.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers); @@ -598,7 +608,16 @@ public void testCheckIdleSlot() throws Exception { try { final BlockingQueue freedSlots = new ArrayBlockingQueue<>(1); - taskManagerGateway.setFreeSlotConsumer((tuple) -> freedSlots.offer(tuple.f0)); + taskManagerGateway.setFreeSlotFunction( + (AllocationID allocationId, Throwable cause) -> + { + try { + freedSlots.put(allocationId); + return CompletableFuture.completedFuture(Acknowledge.get()); + } catch (InterruptedException e) { + return FutureUtils.completedExceptionally(e); + } + }); final SlotPoolGateway slotPoolGateway = setupSlotPool(slotPool, resourceManagerGateway); @@ -634,6 +653,99 @@ public void testCheckIdleSlot() throws Exception { } } + /** + * Tests that idle slots which cannot be released are only recycled if the owning {@link TaskExecutor} + * is still registered at the {@link SlotPool}. See FLINK-9047. + */ + @Test + public void testReleasingIdleSlotFailed() throws Exception { + final ManualClock clock = new ManualClock(); + final SlotPool slotPool = new SlotPool( + rpcService, + jobId, + clock, + TestingUtils.infiniteTime(), + timeout); + + try { + final SlotPoolGateway slotPoolGateway = setupSlotPool(slotPool, resourceManagerGateway); + + final AllocationID expiredAllocationId = new AllocationID(); + final SlotOffer slotToExpire = new SlotOffer(expiredAllocationId, 0, ResourceProfile.UNKNOWN); + + final ArrayDeque> responseQueue = new ArrayDeque<>(2); + taskManagerGateway.setFreeSlotFunction((AllocationID allocationId, Throwable cause) -> { + if (responseQueue.isEmpty()) { + return CompletableFuture.completedFuture(Acknowledge.get()); + } else { + return responseQueue.pop(); + } + }); + + responseQueue.add(FutureUtils.completedExceptionally(new FlinkException("Test failure"))); + + final CompletableFuture responseFuture = new CompletableFuture<>(); + responseQueue.add(responseFuture); + + assertThat( + slotPoolGateway.registerTaskManager(taskManagerLocation.getResourceID()).get(), + Matchers.is(Acknowledge.get())); + + assertThat( + slotPoolGateway.offerSlot(taskManagerLocation, taskManagerGateway, slotToExpire).get(), + Matchers.is(true)); + + clock.advanceTime(timeout.toMilliseconds(), TimeUnit.MILLISECONDS); + + slotPool.triggerCheckIdleSlot(); + + CompletableFuture allocatedSlotFuture = slotPoolGateway.allocateSlot( + new SlotRequestId(), + new DummyScheduledUnit(), + SlotProfile.noRequirements(), + true, + timeout); + + // wait until the slot has been fulfilled with the previously idling slot + final LogicalSlot logicalSlot = allocatedSlotFuture.get(); + assertThat(logicalSlot.getAllocationId(), Matchers.is(expiredAllocationId)); + + // return the slot + slotPool.getSlotOwner().returnAllocatedSlot(logicalSlot).get(); + + // advance the time so that the returned slot is now idling + clock.advanceTime(timeout.toMilliseconds(), TimeUnit.MILLISECONDS); + + slotPool.triggerCheckIdleSlot(); + + // request a new slot after the idling slot has been released + allocatedSlotFuture = slotPoolGateway.allocateSlot( + new SlotRequestId(), + new DummyScheduledUnit(), + SlotProfile.noRequirements(), + true, + timeout); + + // release the TaskExecutor before we get a response from the slot releasing + slotPoolGateway.releaseTaskManager(taskManagerLocation.getResourceID()).get(); + + // let the slot releasing fail --> since the owning TaskExecutor is no longer registered + // the slot should be discarded + responseFuture.completeExceptionally(new FlinkException("Second test exception")); + + try { + // since the slot must have been discarded, we cannot fulfill the slot request + allocatedSlotFuture.get(10L, TimeUnit.MILLISECONDS); + fail("Expected to fail with a timeout."); + } catch (TimeoutException ignored) { + // expected + } + + } finally { + RpcUtils.terminateRpcEndpoint(slotPool, timeout); + } + } + private static SlotPoolGateway setupSlotPool( SlotPool slotPool, ResourceManagerGateway resourceManagerGateway) throws Exception { From 2e40f3807b2bc0c2349b967f2fb9d3d10eddcdca Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 18:44:25 +0100 Subject: [PATCH 0235/2294] [hotfix] Remove unused method from SlotPool --- .../flink/runtime/jobmaster/slotpool/SlotPool.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java index 6040b418336957..662e71a7136c02 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/slotpool/SlotPool.java @@ -69,7 +69,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -724,15 +723,6 @@ private void slotRequestToResourceManagerFailed(SlotRequestId slotRequestID, Thr } } - private void checkTimeoutSlotAllocation(SlotRequestId slotRequestID) { - PendingRequest request = pendingRequests.removeKeyA(slotRequestID); - if (request != null) { - failPendingRequest( - request, - new TimeoutException("Slot allocation request " + slotRequestID + " timed out")); - } - } - private void stashRequestWaitingForResourceManager(final PendingRequest pendingRequest) { log.info("Cannot serve slot request, no ResourceManager connected. " + From 8809185b0ea791c641babe382db2bb7e753100bb Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 21:08:13 +0100 Subject: [PATCH 0236/2294] [hotfix] Make RestServerEndpoint#uploadDir protected --- .../flink/runtime/dispatcher/DispatcherRestEndpoint.java | 4 ---- .../org/apache/flink/runtime/rest/RestServerEndpoint.java | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java index 45185528395d31..8072cf45a71005 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherRestEndpoint.java @@ -42,7 +42,6 @@ import org.apache.flink.shaded.netty4.io.netty.channel.ChannelInboundHandler; import java.io.IOException; -import java.nio.file.Path; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -52,8 +51,6 @@ */ public class DispatcherRestEndpoint extends WebMonitorEndpoint { - private final Path uploadDir; - private WebMonitorExtension webSubmissionExtension; public DispatcherRestEndpoint( @@ -80,7 +77,6 @@ public DispatcherRestEndpoint( leaderElectionService, fatalErrorHandler); - uploadDir = endpointConfiguration.getUploadDir(); webSubmissionExtension = WebMonitorExtension.empty(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java index dfb01ca2657c9f..80d9140d041b96 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java @@ -77,8 +77,9 @@ public abstract class RestServerEndpoint { private final String restBindAddress; private final int restBindPort; private final SSLEngine sslEngine; - private final Path uploadDir; private final int maxContentLength; + + protected final Path uploadDir; protected final Map responseHeaders; private final CompletableFuture terminationFuture; From e1805583b75ab7d31726cd605241e6faee793efb Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 21:04:40 +0100 Subject: [PATCH 0237/2294] [FLINK-9027] [web] Clean up web UI resources by installing shut down hook The ClusterEntrypoint creates temp directory for the RestServerEndpoint. This directory contains the web ui files and if not differently configured the web upload directory. In case of a hard shut down, as it happens with bin/stop-cluster.sh the ClusterEntrypoint will clean up this directory by installing a shut down hook. All future directory cleanup tasks should go into this method ClusterEntrypoin#cleanupDirectories. This closes #5740. --- .../program/rest/RestClusterClientTest.java | 5 -- .../runtime/entrypoint/ClusterEntrypoint.java | 55 +++++++++++++++++-- .../runtime/minicluster/MiniCluster.java | 2 +- .../runtime/rest/RestServerEndpoint.java | 14 ++--- .../rest/RestServerEndpointConfiguration.java | 3 +- .../handler/RestHandlerConfiguration.java | 20 +++---- .../webmonitor/WebMonitorEndpoint.java | 11 ++-- .../rest/RestServerEndpointITCase.java | 2 +- 8 files changed, 74 insertions(+), 38 deletions(-) diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index e108a0b116eb3b..e98ba436abf298 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -666,11 +666,6 @@ private class TestRestServerEndpoint extends RestServerEndpoint implements AutoC @Override protected void startInternal() throws Exception {} - - @Override - public void close() throws Exception { - shutDownAsync().get(); - } } @FunctionalInterface diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 63c8072a826208..8a4db0367bfe14 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -69,7 +69,9 @@ import org.apache.flink.runtime.webmonitor.retriever.impl.AkkaQueryServiceRetriever; import org.apache.flink.runtime.webmonitor.retriever.impl.RpcGatewayRetriever; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FileUtils; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.ShutdownHookUtil; import akka.actor.ActorSystem; import org.slf4j.Logger; @@ -78,10 +80,14 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; +import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -159,9 +165,13 @@ public abstract class ClusterEntrypoint implements FatalErrorHandler { @GuardedBy("lock") private JobManagerMetricGroup jobManagerMetricGroup; + private final Thread shutDownHook; + protected ClusterEntrypoint(Configuration configuration) { - this.configuration = Preconditions.checkNotNull(configuration); + this.configuration = generateClusterConfiguration(configuration); this.terminationFuture = new CompletableFuture<>(); + + shutDownHook = ShutdownHookUtil.addShutdownHook(this::cleanupDirectories, getClass().getSimpleName(), LOG); } public CompletableFuture getTerminationFuture() { @@ -479,7 +489,7 @@ protected CompletableFuture stopClusterComponents() { } if (webMonitorEndpoint != null) { - terminationFutures.add(webMonitorEndpoint.shutDownAsync()); + terminationFutures.add(webMonitorEndpoint.closeAsync()); } if (dispatcher != null) { @@ -523,6 +533,17 @@ public void onFatalError(Throwable exception) { // Internal methods // -------------------------------------------------- + private Configuration generateClusterConfiguration(Configuration configuration) { + final Configuration resultConfiguration = new Configuration(Preconditions.checkNotNull(configuration)); + + final String webTmpDir = configuration.getString(WebOptions.TMP_DIR); + final Path uniqueWebTmpDir = Paths.get(webTmpDir, "flink-web-" + UUID.randomUUID()); + + resultConfiguration.setString(WebOptions.TMP_DIR, uniqueWebTmpDir.toAbsolutePath().toString()); + + return resultConfiguration; + } + private CompletableFuture shutDownAsync(boolean cleanupHaData) { if (isShutDown.compareAndSet(false, true)) { LOG.info("Stopping {}.", getClass().getSimpleName()); @@ -535,11 +556,22 @@ private CompletableFuture shutDownAsync(boolean cleanupHaData) { serviceShutdownFuture.whenComplete( (Void ignored2, Throwable serviceThrowable) -> { + Throwable finalException = null; + if (serviceThrowable != null) { - terminationFuture.completeExceptionally( - ExceptionUtils.firstOrSuppressed(serviceThrowable, componentThrowable)); + finalException = ExceptionUtils.firstOrSuppressed(serviceThrowable, componentThrowable); } else if (componentThrowable != null) { - terminationFuture.completeExceptionally(componentThrowable); + finalException = componentThrowable; + } + + try { + cleanupDirectories(); + } catch (IOException e) { + finalException = ExceptionUtils.firstOrSuppressed(e, finalException); + } + + if (finalException != null) { + terminationFuture.completeExceptionally(finalException); } else { terminationFuture.complete(null); } @@ -576,6 +608,19 @@ private void shutDownAndTerminate( } } + /** + * Clean up of temporary directories created by the {@link ClusterEntrypoint}. + * + * @throws IOException if the temporary directories could not be cleaned up + */ + private void cleanupDirectories() throws IOException { + ShutdownHookUtil.removeShutdownHook(shutDownHook, getClass().getSimpleName(), LOG); + + final String webTmpDir = configuration.getString(WebOptions.TMP_DIR); + + FileUtils.deleteDirectory(new File(webTmpDir)); + } + // -------------------------------------------------- // Abstract methods // -------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index bc75a547b55f56..0da6f333b6830e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -758,7 +758,7 @@ private CompletableFuture shutDownDispatcher() { } if (dispatcherRestEndpoint != null) { - terminationFutures.add(dispatcherRestEndpoint.shutDownAsync()); + terminationFutures.add(dispatcherRestEndpoint.closeAsync()); dispatcherRestEndpoint = null; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java index 80d9140d041b96..15fbbb24866ec0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpoint.java @@ -25,8 +25,8 @@ import org.apache.flink.runtime.rest.handler.PipelineErrorHandler; import org.apache.flink.runtime.rest.handler.RestHandlerSpecification; import org.apache.flink.runtime.rest.handler.RouterHandler; +import org.apache.flink.util.AutoCloseableAsync; import org.apache.flink.util.ExceptionUtils; -import org.apache.flink.util.FileUtils; import org.apache.flink.util.Preconditions; import org.apache.flink.shaded.netty4.io.netty.bootstrap.ServerBootstrap; @@ -67,7 +67,7 @@ /** * An abstract class for netty-based REST server endpoints. */ -public abstract class RestServerEndpoint { +public abstract class RestServerEndpoint implements AutoCloseableAsync { protected final Logger log = LoggerFactory.getLogger(getClass()); @@ -256,7 +256,8 @@ public String getRestBaseUrl() { } } - public final CompletableFuture shutDownAsync() { + @Override + public CompletableFuture closeAsync() { synchronized (lock) { log.info("Shutting down rest endpoint."); @@ -370,12 +371,7 @@ protected CompletableFuture shutDownInternal() { }); }); - return FutureUtils.runAfterwards( - channelTerminationFuture, - () -> { - log.info("Cleaning upload directory {}", uploadDir); - FileUtils.cleanDirectory(uploadDir.toFile()); - }); + return channelTerminationFuture; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java index 1fac08e53edc9d..8af76f5bfd5c0a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/RestServerEndpointConfiguration.java @@ -36,7 +36,6 @@ import java.nio.file.Paths; import java.util.Collections; import java.util.Map; -import java.util.UUID; import static java.util.Objects.requireNonNull; @@ -172,7 +171,7 @@ public static RestServerEndpointConfiguration fromConfiguration(Configuration co final Path uploadDir = Paths.get( config.getString(WebOptions.UPLOAD_DIR, config.getString(WebOptions.TMP_DIR)), - "flink-web-upload-" + UUID.randomUUID()); + "flink-web-upload"); final int maxContentLength = config.getInteger(RestOptions.REST_SERVER_MAX_CONTENT_LENGTH); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java index f92946bd0f5f4d..3f6516aadde72f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/RestHandlerConfiguration.java @@ -23,8 +23,8 @@ import org.apache.flink.configuration.WebOptions; import org.apache.flink.util.Preconditions; -import java.io.File; -import java.util.UUID; +import java.nio.file.Path; +import java.nio.file.Paths; /** * Configuration object containing values for the rest handler configuration. @@ -37,20 +37,20 @@ public class RestHandlerConfiguration { private final Time timeout; - private final File tmpDir; + private final Path webUiDir; public RestHandlerConfiguration( long refreshInterval, int maxCheckpointStatisticCacheEntries, Time timeout, - File tmpDir) { + Path webUiDir) { Preconditions.checkArgument(refreshInterval > 0L, "The refresh interval (ms) should be larger than 0."); this.refreshInterval = refreshInterval; this.maxCheckpointStatisticCacheEntries = maxCheckpointStatisticCacheEntries; this.timeout = Preconditions.checkNotNull(timeout); - this.tmpDir = Preconditions.checkNotNull(tmpDir); + this.webUiDir = Preconditions.checkNotNull(webUiDir); } public long getRefreshInterval() { @@ -65,8 +65,8 @@ public Time getTimeout() { return timeout; } - public File getTmpDir() { - return tmpDir; + public Path getWebUiDir() { + return webUiDir; } public static RestHandlerConfiguration fromConfiguration(Configuration configuration) { @@ -76,13 +76,13 @@ public static RestHandlerConfiguration fromConfiguration(Configuration configura final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT)); - final String rootDir = "flink-web-" + UUID.randomUUID(); - final File tmpDir = new File(configuration.getString(WebOptions.TMP_DIR), rootDir); + final String rootDir = "flink-web-ui"; + final Path webUiDir = Paths.get(configuration.getString(WebOptions.TMP_DIR), rootDir); return new RestHandlerConfiguration( refreshInterval, maxCheckpointStatisticCacheEntries, timeout, - tmpDir); + webUiDir); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java index 50ad7eb1bceeb2..d4aa94e19feaa3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java @@ -127,6 +127,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -498,7 +499,7 @@ protected List> initiali executor, metricFetcher); - final File tmpDir = restConfiguration.getTmpDir(); + final Path webUiDir = restConfiguration.getWebUiDir(); Optional> optWebContent; @@ -507,7 +508,7 @@ protected List> initiali leaderRetriever, restAddressFuture, timeout, - tmpDir); + webUiDir.toFile()); } catch (IOException e) { log.warn("Could not load web content handler.", e); optWebContent = Optional.empty(); @@ -635,15 +636,15 @@ protected CompletableFuture shutDownInternal() { final CompletableFuture shutdownFuture = super.shutDownInternal(); - final File tmpDir = restConfiguration.getTmpDir(); + final Path webUiDir = restConfiguration.getWebUiDir(); return FutureUtils.runAfterwardsAsync( shutdownFuture, () -> { Exception exception = null; try { - log.info("Removing cache directory {}", tmpDir); - FileUtils.deleteDirectory(tmpDir); + log.info("Removing cache directory {}", webUiDir); + FileUtils.deleteDirectory(webUiDir.toFile()); } catch (Exception e) { exception = e; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java index 784c14158a3b4d..e510798069e9dd 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerEndpointITCase.java @@ -154,7 +154,7 @@ public void teardown() throws Exception { } if (serverEndpoint != null) { - serverEndpoint.shutDownAsync().get(); + serverEndpoint.close(); serverEndpoint = null; } } From 2d86ddf7d762da7bcd76ac269a3d72d15bffb4fd Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 21:48:19 +0100 Subject: [PATCH 0238/2294] [hotfix] Log final status and exit code under lock --- .../flink/runtime/entrypoint/ClusterEntrypoint.java | 10 +++++----- .../flink/runtime/entrypoint/JobClusterEntrypoint.java | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 8a4db0367bfe14..50d0db335f442b 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -587,12 +587,12 @@ private void shutDownAndTerminate( ApplicationStatus applicationStatus, boolean cleanupHaData) { - LOG.info("Shut down and terminate {} with return code {} and application status {}.", - getClass().getSimpleName(), - returnCode, - applicationStatus); - if (isTerminating.compareAndSet(false, true)) { + LOG.info("Shut down and terminate {} with return code {} and application status {}.", + getClass().getSimpleName(), + returnCode, + applicationStatus); + shutDownAsync(cleanupHaData).whenComplete( (Void ignored, Throwable t) -> { if (t != null) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java index df950a343be944..95d9c742bef4a6 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/JobClusterEntrypoint.java @@ -44,7 +44,6 @@ import javax.annotation.Nullable; -import java.io.IOException; import java.util.concurrent.Executor; /** @@ -83,7 +82,7 @@ protected MiniDispatcherRestEndpoint createRestEndpoint( @Override protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore( Configuration configuration, - ScheduledExecutor scheduledExecutor) throws IOException { + ScheduledExecutor scheduledExecutor) { return new MemoryArchivedExecutionGraphStore(); } From 95cde57137f9e79c4b67396e44a2663356d83d23 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 22:14:58 +0100 Subject: [PATCH 0239/2294] [hotfix] Add FutureUtils#composeAfterwards --- .../flink/runtime/concurrent/FutureUtils.java | 34 +++++ .../runtime/concurrent/FutureUtilsTest.java | 119 ++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java index e0164a92f6a992..51740e3a1aba59 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java @@ -411,6 +411,40 @@ public static CompletableFuture runAfterwardsAsync( return resultFuture; } + /** + * Run the given asynchronous action after the completion of the given future. The given future can be + * completed normally or exceptionally. In case of an exceptional completion, the + * asynchronous action's exception will be added to the initial exception. + * + * @param future to wait for its completion + * @param composedAction asynchronous action which is triggered after the future's completion + * @return Future which is completed after the asynchronous action has completed. This future can contain + * an exception if an error occurred in the given future or asynchronous action. + */ + public static CompletableFuture composeAfterwards( + CompletableFuture future, + Supplier> composedAction) { + final CompletableFuture resultFuture = new CompletableFuture<>(); + + future.whenComplete( + (Object outerIgnored, Throwable outerThrowable) -> { + final CompletableFuture composedActionFuture = composedAction.get(); + + composedActionFuture.whenComplete( + (Object innerIgnored, Throwable innerThrowable) -> { + if (innerThrowable != null) { + resultFuture.completeExceptionally(ExceptionUtils.firstOrSuppressed(innerThrowable, outerThrowable)); + } else if (outerThrowable != null) { + resultFuture.completeExceptionally(outerThrowable); + } else { + resultFuture.complete(null); + } + }); + }); + + return resultFuture; + } + // ------------------------------------------------------------------------ // composing futures // ------------------------------------------------------------------------ diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java index 57f6bd01029ab4..df2a0c748c3438 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java @@ -46,6 +46,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -327,6 +328,124 @@ public void testRunAfterwardsExceptional() throws Exception { } } + @Test + public void testComposeAfterwards() throws ExecutionException, InterruptedException { + final CompletableFuture inputFuture = new CompletableFuture<>(); + final OneShotLatch composeLatch = new OneShotLatch(); + + final CompletableFuture composeFuture = FutureUtils.composeAfterwards( + inputFuture, + () -> { + composeLatch.trigger(); + return CompletableFuture.completedFuture(null); + }); + + assertThat(composeLatch.isTriggered(), is(false)); + assertThat(composeFuture.isDone(), is(false)); + + inputFuture.complete(null); + + assertThat(composeLatch.isTriggered(), is(true)); + assertThat(composeFuture.isDone(), is(true)); + + // check that tthis future is not exceptionally completed + composeFuture.get(); + } + + @Test + public void testComposeAfterwardsFirstExceptional() throws InterruptedException { + final CompletableFuture inputFuture = new CompletableFuture<>(); + final OneShotLatch composeLatch = new OneShotLatch(); + final FlinkException testException = new FlinkException("Test exception"); + + final CompletableFuture composeFuture = FutureUtils.composeAfterwards( + inputFuture, + () -> { + composeLatch.trigger(); + return CompletableFuture.completedFuture(null); + }); + + assertThat(composeLatch.isTriggered(), is(false)); + assertThat(composeFuture.isDone(), is(false)); + + inputFuture.completeExceptionally(testException); + + assertThat(composeLatch.isTriggered(), is(true)); + assertThat(composeFuture.isDone(), is(true)); + + // check that this future is not exceptionally completed + try { + composeFuture.get(); + fail("Expected an exceptional completion"); + } catch (ExecutionException ee) { + assertThat(ExceptionUtils.stripExecutionException(ee), is(testException)); + } + } + + @Test + public void testComposeAfterwardsSecondExceptional() throws InterruptedException { + final CompletableFuture inputFuture = new CompletableFuture<>(); + final OneShotLatch composeLatch = new OneShotLatch(); + final FlinkException testException = new FlinkException("Test exception"); + + final CompletableFuture composeFuture = FutureUtils.composeAfterwards( + inputFuture, + () -> { + composeLatch.trigger(); + return FutureUtils.completedExceptionally(testException); + }); + + assertThat(composeLatch.isTriggered(), is(false)); + assertThat(composeFuture.isDone(), is(false)); + + inputFuture.complete(null); + + assertThat(composeLatch.isTriggered(), is(true)); + assertThat(composeFuture.isDone(), is(true)); + + // check that this future is not exceptionally completed + try { + composeFuture.get(); + fail("Expected an exceptional completion"); + } catch (ExecutionException ee) { + assertThat(ExceptionUtils.stripExecutionException(ee), is(testException)); + } + } + + @Test + public void testComposeAfterwardsBothExceptional() throws InterruptedException { + final CompletableFuture inputFuture = new CompletableFuture<>(); + final FlinkException testException1 = new FlinkException("Test exception1"); + final FlinkException testException2 = new FlinkException("Test exception2"); + final OneShotLatch composeLatch = new OneShotLatch(); + + final CompletableFuture composeFuture = FutureUtils.composeAfterwards( + inputFuture, + () -> { + composeLatch.trigger(); + return FutureUtils.completedExceptionally(testException2); + }); + + assertThat(composeLatch.isTriggered(), is(false)); + assertThat(composeFuture.isDone(), is(false)); + + inputFuture.completeExceptionally(testException1); + + assertThat(composeLatch.isTriggered(), is(true)); + assertThat(composeFuture.isDone(), is(true)); + + // check that this future is not exceptionally completed + try { + composeFuture.get(); + fail("Expected an exceptional completion"); + } catch (ExecutionException ee) { + final Throwable actual = ExceptionUtils.stripExecutionException(ee); + assertThat(actual, is(testException1)); + assertThat(actual.getSuppressed(), arrayWithSize(1)); + assertThat(actual.getSuppressed()[0], is(testException2)); + } + } + @Test public void testCompleteAll() throws Exception { final CompletableFuture inputFuture1 = new CompletableFuture<>(); From 9ee02f6a3463c531400404dee791033e120f5bfc Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 21 Mar 2018 22:19:28 +0100 Subject: [PATCH 0240/2294] [FLINK-8900] [yarn] Properly unregister application from Yarn RM This closes #5741. --- .../MesosResourceManager.java | 4 +- .../MesosResourceManagerTest.java | 2 +- .../runtime/entrypoint/ClusterEntrypoint.java | 81 ++++++++++++------- .../resourcemanager/ResourceManager.java | 16 ++-- .../ResourceManagerGateway.java | 11 ++- .../StandaloneResourceManager.java | 2 +- .../TestingResourceManager.java | 2 +- .../utils/TestingResourceManagerGateway.java | 4 +- .../flink/yarn/YarnResourceManager.java | 6 +- 9 files changed, 77 insertions(+), 51 deletions(-) diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java index 1f58b119f27682..4f4a6d1942c3aa 100644 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java @@ -362,9 +362,9 @@ public CompletableFuture postStop() { } @Override - protected void shutDownApplication( + protected void internalDeregisterApplication( ApplicationStatus finalStatus, - @Nullable String optionalDiagnostics) throws ResourceManagerException { + @Nullable String diagnostics) throws ResourceManagerException { LOG.info("Shutting down and unregistering as a Mesos framework."); Exception exception = null; diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java index 412e18da65a30b..5d9a6cffd70382 100644 --- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java +++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java @@ -737,7 +737,7 @@ public void testStopWorker() throws Exception { public void testShutdownApplication() throws Exception { new Context() {{ startResourceManager(); - resourceManager.shutDownCluster(ApplicationStatus.SUCCEEDED, ""); + resourceManager.deregisterApplication(ApplicationStatus.SUCCEEDED, ""); // verify that the Mesos framework is shutdown verify(rmServices.schedulerDriver).stop(false); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 50d0db335f442b..b25729bd1e0238 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -197,6 +197,7 @@ protected void startCluster() { shutDownAndTerminate( STARTUP_FAILURE_RETURN_CODE, ApplicationStatus.FAILED, + t.getMessage(), false); } } @@ -245,6 +246,7 @@ protected void runCluster(Configuration configuration) throws Exception { shutDownAndTerminate( SUCCESS_RETURN_CODE, ApplicationStatus.SUCCEEDED, + throwable != null ? throwable.getMessage() : null, true); }); } @@ -544,38 +546,34 @@ private Configuration generateClusterConfiguration(Configuration configuration) return resultConfiguration; } - private CompletableFuture shutDownAsync(boolean cleanupHaData) { + private CompletableFuture shutDownAsync( + boolean cleanupHaData, + ApplicationStatus applicationStatus, + @Nullable String diagnostics) { if (isShutDown.compareAndSet(false, true)) { LOG.info("Stopping {}.", getClass().getSimpleName()); - final CompletableFuture componentShutdownFuture = stopClusterComponents(); - - componentShutdownFuture.whenComplete( - (Void ignored1, Throwable componentThrowable) -> { - final CompletableFuture serviceShutdownFuture = stopClusterServices(cleanupHaData); - - serviceShutdownFuture.whenComplete( - (Void ignored2, Throwable serviceThrowable) -> { - Throwable finalException = null; - - if (serviceThrowable != null) { - finalException = ExceptionUtils.firstOrSuppressed(serviceThrowable, componentThrowable); - } else if (componentThrowable != null) { - finalException = componentThrowable; - } - - try { - cleanupDirectories(); - } catch (IOException e) { - finalException = ExceptionUtils.firstOrSuppressed(e, finalException); - } - - if (finalException != null) { - terminationFuture.completeExceptionally(finalException); - } else { - terminationFuture.complete(null); - } - }); + final CompletableFuture shutDownApplicationFuture = deregisterApplication(applicationStatus, diagnostics); + + final CompletableFuture componentShutdownFuture = FutureUtils.composeAfterwards( + shutDownApplicationFuture, + this::stopClusterComponents); + + final CompletableFuture serviceShutdownFuture = FutureUtils.composeAfterwards( + componentShutdownFuture, + () -> stopClusterServices(cleanupHaData)); + + final CompletableFuture cleanupDirectoriesFuture = FutureUtils.runAfterwards( + serviceShutdownFuture, + this::cleanupDirectories); + + cleanupDirectoriesFuture.whenComplete( + (Void ignored2, Throwable serviceThrowable) -> { + if (serviceThrowable != null) { + terminationFuture.completeExceptionally(serviceThrowable); + } else { + terminationFuture.complete(null); + } }); } @@ -585,6 +583,7 @@ private CompletableFuture shutDownAsync(boolean cleanupHaData) { private void shutDownAndTerminate( int returnCode, ApplicationStatus applicationStatus, + @Nullable String diagnostics, boolean cleanupHaData) { if (isTerminating.compareAndSet(false, true)) { @@ -593,7 +592,10 @@ private void shutDownAndTerminate( returnCode, applicationStatus); - shutDownAsync(cleanupHaData).whenComplete( + shutDownAsync( + cleanupHaData, + applicationStatus, + diagnostics).whenComplete( (Void ignored, Throwable t) -> { if (t != null) { LOG.info("Could not properly shut down cluster entrypoint.", t); @@ -608,6 +610,25 @@ private void shutDownAndTerminate( } } + /** + * Deregister the Flink application from the resource management system by signalling + * the {@link ResourceManager}. + * + * @param applicationStatus to terminate the application with + * @param diagnostics additional information about the shut down, can be {@code null} + * @return Future which is completed once the shut down + */ + private CompletableFuture deregisterApplication(ApplicationStatus applicationStatus, @Nullable String diagnostics) { + synchronized (lock) { + if (resourceManager != null) { + final ResourceManagerGateway selfGateway = resourceManager.getSelfGateway(ResourceManagerGateway.class); + return selfGateway.deregisterApplication(applicationStatus, diagnostics).thenApply(ack -> null); + } else { + return CompletableFuture.completedFuture(null); + } + } + } + /** * Clean up of temporary directories created by the {@link ClusterEntrypoint}. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java index cae9c6cdb383c9..c75346900cdd8e 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManager.java @@ -486,19 +486,21 @@ public void unRegisterInfoMessageListener(final String address) { * Cleanup application and shut down cluster. * * @param finalStatus of the Flink application - * @param optionalDiagnostics diagnostics message for the Flink application or {@code null} + * @param diagnostics diagnostics message for the Flink application or {@code null} */ @Override - public void shutDownCluster( + public CompletableFuture deregisterApplication( final ApplicationStatus finalStatus, - @Nullable final String optionalDiagnostics) { - log.info("Shut down cluster because application is in {}, diagnostics {}.", finalStatus, optionalDiagnostics); + @Nullable final String diagnostics) { + log.info("Shut down cluster because application is in {}, diagnostics {}.", finalStatus, diagnostics); try { - shutDownApplication(finalStatus, optionalDiagnostics); + internalDeregisterApplication(finalStatus, diagnostics); } catch (ResourceManagerException e) { log.warn("Could not properly shutdown the application.", e); } + + return CompletableFuture.completedFuture(Acknowledge.get()); } @Override @@ -946,7 +948,7 @@ public void handleError(final Exception exception) { protected abstract void initialize() throws ResourceManagerException; /** - * The framework specific code for shutting down the application. This should report the + * The framework specific code to deregister the application. This should report the * application's final status and shut down the resource manager cleanly. * *

    This method also needs to make sure all pending containers that are not registered @@ -956,7 +958,7 @@ public void handleError(final Exception exception) { * @param optionalDiagnostics A diagnostics message or {@code null}. * @throws ResourceManagerException if the application could not be shut down. */ - protected abstract void shutDownApplication( + protected abstract void internalDeregisterApplication( ApplicationStatus finalStatus, @Nullable String optionalDiagnostics) throws ResourceManagerException; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java index 836bc0b0faf000..bd282d6cff0bc0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerGateway.java @@ -41,6 +41,8 @@ import org.apache.flink.runtime.taskexecutor.SlotReport; import org.apache.flink.runtime.taskexecutor.TaskExecutor; +import javax.annotation.Nullable; + import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -133,11 +135,12 @@ void notifySlotAvailable( void unRegisterInfoMessageListener(String infoMessageListenerAddress); /** - * shutdown cluster - * @param finalStatus - * @param optionalDiagnostics + * Deregister Flink from the underlying resource management system. + * + * @param finalStatus final status with which to deregister the Flink application + * @param diagnostics additional information for the resource management system, can be {@code null} */ - void shutDownCluster(final ApplicationStatus finalStatus, final String optionalDiagnostics); + CompletableFuture deregisterApplication(final ApplicationStatus finalStatus, @Nullable final String diagnostics); /** * Gets the currently registered number of TaskManagers. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java index 7226d296f373d7..d8e0e480a2a4d8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManager.java @@ -72,7 +72,7 @@ protected void initialize() throws ResourceManagerException { } @Override - protected void shutDownApplication(ApplicationStatus finalStatus, @Nullable String optionalDiagnostics) { + protected void internalDeregisterApplication(ApplicationStatus finalStatus, @Nullable String diagnostics) { } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java index 3db9be032e6ea4..2bd976bd9bf327 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java @@ -68,7 +68,7 @@ protected void initialize() throws ResourceManagerException { } @Override - protected void shutDownApplication(ApplicationStatus finalStatus, @Nullable String optionalDiagnostics) throws ResourceManagerException { + protected void internalDeregisterApplication(ApplicationStatus finalStatus, @Nullable String diagnostics) throws ResourceManagerException { // noop } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/TestingResourceManagerGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/TestingResourceManagerGateway.java index 33c6c08d667048..9b4041414d62e8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/TestingResourceManagerGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/TestingResourceManagerGateway.java @@ -214,8 +214,8 @@ public void unRegisterInfoMessageListener(String infoMessageListenerAddress) { } @Override - public void shutDownCluster(ApplicationStatus finalStatus, String optionalDiagnostics) { - + public CompletableFuture deregisterApplication(ApplicationStatus finalStatus, String diagnostics) { + return CompletableFuture.completedFuture(Acknowledge.get()); } @Override diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 97db2ad8a37a12..bfe7d65262ac64 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -266,16 +266,16 @@ public CompletableFuture postStop() { } @Override - protected void shutDownApplication( + protected void internalDeregisterApplication( ApplicationStatus finalStatus, - @Nullable String optionalDiagnostics) { + @Nullable String diagnostics) { // first, de-register from YARN FinalApplicationStatus yarnStatus = getYarnStatus(finalStatus); log.info("Unregister application from the YARN Resource Manager with final status {}.", yarnStatus); try { - resourceManagerClient.unregisterApplicationMaster(yarnStatus, optionalDiagnostics, ""); + resourceManagerClient.unregisterApplicationMaster(yarnStatus, diagnostics, ""); } catch (Throwable t) { log.error("Could not unregister the application master.", t); } From edb6f7fef8c5df6af43bbe28f96d8c6bb3332d00 Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 19 Mar 2018 11:36:39 +0100 Subject: [PATCH 0241/2294] [FLINK-8956][tests] Port RescalingITCase to flip6 This closes #5715. --- .../test/checkpointing/RescalingITCase.java | 282 ++++++------------ 1 file changed, 97 insertions(+), 185 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java index a23c679e65f2b3..e4f4389bb6abc5 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java @@ -25,23 +25,24 @@ import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.state.ValueState; import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.CheckpointingOptions; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.runtime.client.JobStatusMessage; +import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.executiongraph.ExecutionJobVertex; -import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; -import org.apache.flink.runtime.messages.JobManagerMessages; import org.apache.flink.runtime.state.FunctionInitializationContext; import org.apache.flink.runtime.state.FunctionSnapshotContext; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; -import org.apache.flink.runtime.testingUtils.TestingCluster; -import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages; +import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; import org.apache.flink.streaming.api.checkpoint.ListCheckpointed; import org.apache.flink.streaming.api.datastream.DataStream; @@ -50,7 +51,9 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.util.Collector; +import org.apache.flink.util.FlinkException; import org.apache.flink.util.TestLogger; import org.junit.AfterClass; @@ -62,25 +65,20 @@ import org.junit.runners.Parameterized; import java.io.File; +import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import scala.Option; -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.duration.Deadline; -import scala.concurrent.duration.FiniteDuration; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Test savepoint rescaling. @@ -106,7 +104,7 @@ enum OperatorCheckpointMethod { NON_PARTITIONED, CHECKPOINTED_FUNCTION, CHECKPOINTED_FUNCTION_BROADCAST, LIST_CHECKPOINTED } - private static TestingCluster cluster; + private static MiniClusterResource cluster; @ClassRule public static TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -120,8 +118,6 @@ public void setup() throws Exception { currentBackend = backend; Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, slotsPerTaskManager); final File checkpointDir = temporaryFolder.newFolder(); final File savepointDir = temporaryFolder.newFolder(); @@ -130,15 +126,20 @@ public void setup() throws Exception { config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString()); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString()); - cluster = new TestingCluster(config); - cluster.start(); + cluster = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + config, + numTaskManagers, + slotsPerTaskManager), + true); + cluster.before(); } } @AfterClass public static void shutDownExistingCluster() { if (cluster != null) { - cluster.stop(); + cluster.after(); cluster = null; } } @@ -175,20 +176,18 @@ public void testSavepointRescalingKeyedState(boolean scaleOut, boolean deriveMax final int parallelism2 = scaleOut ? numSlots : numSlots / 2; final int maxParallelism = 13; - FiniteDuration timeout = new FiniteDuration(3, TimeUnit.MINUTES); - Deadline deadline = timeout.fromNow(); + Duration timeout = Duration.ofMinutes(3); + Deadline deadline = Deadline.now().plus(timeout); - ActorGateway jobManager = null; - JobID jobID = null; + ClusterClient client = cluster.getClusterClient(); try { - jobManager = cluster.getLeaderGateway(deadline.timeLeft()); - JobGraph jobGraph = createJobGraphWithKeyedState(parallelism, maxParallelism, numberKeys, numberElements, false, 100); - jobID = jobGraph.getJobID(); + final JobID jobID = jobGraph.getJobID(); - cluster.submitJobDetached(jobGraph); + client.setDetached(true); + client.submitJob(jobGraph, RescalingITCase.class.getClassLoader()); // wait til the sources have emitted numberElements for each key and completed a checkpoint SubtaskIndexFlatMapper.workCompletedLatch.await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); @@ -210,22 +209,15 @@ public void testSavepointRescalingKeyedState(boolean scaleOut, boolean deriveMax // clear the CollectionSink set for the restarted job CollectionSink.clearElementsSet(); - Future savepointPathFuture = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobID, Option.empty()), deadline.timeLeft()); - - final String savepointPath = ((JobManagerMessages.TriggerSavepointSuccess) - Await.result(savepointPathFuture, deadline.timeLeft())).savepointPath(); - - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), deadline.timeLeft()); + CompletableFuture savepointPathFuture = client.triggerSavepoint(jobID, null); - Future cancellationResponseFuture = jobManager.ask(new JobManagerMessages.CancelJob(jobID), deadline.timeLeft()); + final String savepointPath = savepointPathFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - Object cancellationResponse = Await.result(cancellationResponseFuture, deadline.timeLeft()); + client.cancel(jobID); - assertTrue(cancellationResponse instanceof JobManagerMessages.CancellationSuccess); - - Await.ready(jobRemovedFuture, deadline.timeLeft()); - - jobID = null; + while (!getRunningJobs(client).isEmpty()) { + Thread.sleep(50); + } int restoreMaxParallelism = deriveMaxParallelism ? ExecutionJobVertex.VALUE_NOT_SET : maxParallelism; @@ -233,11 +225,8 @@ public void testSavepointRescalingKeyedState(boolean scaleOut, boolean deriveMax scaledJobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath)); - jobID = scaledJobGraph.getJobID(); - - cluster.submitJobAndWait(scaledJobGraph, false); - - jobID = null; + client.setDetached(false); + client.submitJob(scaledJobGraph, RescalingITCase.class.getClassLoader()); Set> actualResult2 = CollectionSink.getElementsSet(); @@ -253,17 +242,6 @@ public void testSavepointRescalingKeyedState(boolean scaleOut, boolean deriveMax } finally { // clear the CollectionSink set for the restarted job CollectionSink.clearElementsSet(); - - // clear any left overs from a possibly failed job - if (jobID != null && jobManager != null) { - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), timeout); - - try { - Await.ready(jobRemovedFuture, timeout); - } catch (TimeoutException | InterruptedException ie) { - fail("Failed while cleaning up the cluster."); - } - } } } @@ -279,57 +257,39 @@ public void testSavepointRescalingNonPartitionedStateCausesException() throws Ex final int parallelism2 = numSlots; final int maxParallelism = 13; - FiniteDuration timeout = new FiniteDuration(3, TimeUnit.MINUTES); - Deadline deadline = timeout.fromNow(); + Duration timeout = Duration.ofMinutes(3); + Deadline deadline = Deadline.now().plus(timeout); - JobID jobID = null; - ActorGateway jobManager = null; + ClusterClient client = cluster.getClusterClient(); try { - jobManager = cluster.getLeaderGateway(deadline.timeLeft()); - JobGraph jobGraph = createJobGraphWithOperatorState(parallelism, maxParallelism, OperatorCheckpointMethod.NON_PARTITIONED); - jobID = jobGraph.getJobID(); - - cluster.submitJobDetached(jobGraph); + final JobID jobID = jobGraph.getJobID(); - Object savepointResponse = null; + client.setDetached(true); + client.submitJob(jobGraph, RescalingITCase.class.getClassLoader()); // wait until the operator is started StateSourceBase.workStartedLatch.await(); - Future savepointPathFuture = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobID, Option.empty()), deadline.timeLeft()); - FiniteDuration waitingTime = new FiniteDuration(10, TimeUnit.SECONDS); - savepointResponse = Await.result(savepointPathFuture, waitingTime); + CompletableFuture savepointPathFuture = client.triggerSavepoint(jobID, null); - assertTrue(String.valueOf(savepointResponse), savepointResponse instanceof JobManagerMessages.TriggerSavepointSuccess); + final String savepointPath = savepointPathFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - final String savepointPath = ((JobManagerMessages.TriggerSavepointSuccess) savepointResponse).savepointPath(); + client.cancel(jobID); - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), deadline.timeLeft()); - - Future cancellationResponseFuture = jobManager.ask(new JobManagerMessages.CancelJob(jobID), deadline.timeLeft()); - - Object cancellationResponse = Await.result(cancellationResponseFuture, deadline.timeLeft()); - - assertTrue(cancellationResponse instanceof JobManagerMessages.CancellationSuccess); - - Await.ready(jobRemovedFuture, deadline.timeLeft()); + while (!getRunningJobs(client).isEmpty()) { + Thread.sleep(50); + } // job successfully removed - jobID = null; - JobGraph scaledJobGraph = createJobGraphWithOperatorState(parallelism2, maxParallelism, OperatorCheckpointMethod.NON_PARTITIONED); scaledJobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath)); - jobID = scaledJobGraph.getJobID(); - - cluster.submitJobAndWait(scaledJobGraph, false); - - jobID = null; - + client.setDetached(false); + client.submitJob(scaledJobGraph, RescalingITCase.class.getClassLoader()); } catch (JobExecutionException exception) { if (exception.getCause() instanceof IllegalStateException) { // we expect a IllegalStateException wrapped @@ -338,17 +298,6 @@ public void testSavepointRescalingNonPartitionedStateCausesException() throws Ex } else { throw exception; } - } finally { - // clear any left overs from a possibly failed job - if (jobID != null && jobManager != null) { - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), timeout); - - try { - Await.ready(jobRemovedFuture, timeout); - } catch (TimeoutException | InterruptedException ie) { - fail("Failed while cleaning up the cluster."); - } - } } } @@ -367,14 +316,12 @@ public void testSavepointRescalingWithKeyedAndNonPartitionedState() throws Excep int parallelism2 = numSlots; int maxParallelism = 13; - FiniteDuration timeout = new FiniteDuration(3, TimeUnit.MINUTES); - Deadline deadline = timeout.fromNow(); + Duration timeout = Duration.ofMinutes(3); + Deadline deadline = Deadline.now().plus(timeout); - ActorGateway jobManager = null; - JobID jobID = null; + ClusterClient client = cluster.getClusterClient(); try { - jobManager = cluster.getLeaderGateway(deadline.timeLeft()); JobGraph jobGraph = createJobGraphWithKeyedAndNonPartitionedOperatorState( parallelism, @@ -385,9 +332,10 @@ public void testSavepointRescalingWithKeyedAndNonPartitionedState() throws Excep false, 100); - jobID = jobGraph.getJobID(); + final JobID jobID = jobGraph.getJobID(); - cluster.submitJobDetached(jobGraph); + client.setDetached(true); + client.submitJob(jobGraph, RescalingITCase.class.getClassLoader()); // wait til the sources have emitted numberElements for each key and completed a checkpoint SubtaskIndexFlatMapper.workCompletedLatch.await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); @@ -409,22 +357,15 @@ public void testSavepointRescalingWithKeyedAndNonPartitionedState() throws Excep // clear the CollectionSink set for the restarted job CollectionSink.clearElementsSet(); - Future savepointPathFuture = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobID, Option.empty()), deadline.timeLeft()); - - final String savepointPath = ((JobManagerMessages.TriggerSavepointSuccess) - Await.result(savepointPathFuture, deadline.timeLeft())).savepointPath(); - - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), deadline.timeLeft()); - - Future cancellationResponseFuture = jobManager.ask(new JobManagerMessages.CancelJob(jobID), deadline.timeLeft()); + CompletableFuture savepointPathFuture = client.triggerSavepoint(jobID, null); - Object cancellationResponse = Await.result(cancellationResponseFuture, deadline.timeLeft()); + final String savepointPath = savepointPathFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - assertTrue(cancellationResponse instanceof JobManagerMessages.CancellationSuccess); + client.cancel(jobID); - Await.ready(jobRemovedFuture, deadline.timeLeft()); - - jobID = null; + while (!getRunningJobs(client).isEmpty()) { + Thread.sleep(50); + } JobGraph scaledJobGraph = createJobGraphWithKeyedAndNonPartitionedOperatorState( parallelism2, @@ -437,11 +378,8 @@ public void testSavepointRescalingWithKeyedAndNonPartitionedState() throws Excep scaledJobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath)); - jobID = scaledJobGraph.getJobID(); - - cluster.submitJobAndWait(scaledJobGraph, false); - - jobID = null; + client.setDetached(false); + client.submitJob(scaledJobGraph, RescalingITCase.class.getClassLoader()); Set> actualResult2 = CollectionSink.getElementsSet(); @@ -457,17 +395,6 @@ public void testSavepointRescalingWithKeyedAndNonPartitionedState() throws Excep } finally { // clear the CollectionSink set for the restarted job CollectionSink.clearElementsSet(); - - // clear any left overs from a possibly failed job - if (jobID != null && jobManager != null) { - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), timeout); - - try { - Await.ready(jobRemovedFuture, timeout); - } catch (TimeoutException | InterruptedException ie) { - fail("Failed while cleaning up the cluster."); - } - } } } @@ -510,11 +437,10 @@ public void testSavepointRescalingPartitionedOperatorState(boolean scaleOut, Ope final int parallelism2 = scaleOut ? numSlots / 2 : numSlots; final int maxParallelism = 13; - FiniteDuration timeout = new FiniteDuration(3, TimeUnit.MINUTES); - Deadline deadline = timeout.fromNow(); + Duration timeout = Duration.ofMinutes(3); + Deadline deadline = Deadline.now().plus(timeout); - JobID jobID = null; - ActorGateway jobManager = null; + ClusterClient client = cluster.getClusterClient(); int counterSize = Math.max(parallelism, parallelism2); @@ -528,54 +454,44 @@ public void testSavepointRescalingPartitionedOperatorState(boolean scaleOut, Ope } try { - jobManager = cluster.getLeaderGateway(deadline.timeLeft()); - JobGraph jobGraph = createJobGraphWithOperatorState(parallelism, maxParallelism, checkpointMethod); - jobID = jobGraph.getJobID(); - - cluster.submitJobDetached(jobGraph); + final JobID jobID = jobGraph.getJobID(); - Object savepointResponse = null; + client.setDetached(true); + client.submitJob(jobGraph, RescalingITCase.class.getClassLoader()); // wait until the operator is started StateSourceBase.workStartedLatch.await(); - while (deadline.hasTimeLeft()) { - - Future savepointPathFuture = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobID, Option.empty()), deadline.timeLeft()); - FiniteDuration waitingTime = new FiniteDuration(10, TimeUnit.SECONDS); - savepointResponse = Await.result(savepointPathFuture, waitingTime); - - if (savepointResponse instanceof JobManagerMessages.TriggerSavepointSuccess) { - break; - } - } - - assertTrue(savepointResponse instanceof JobManagerMessages.TriggerSavepointSuccess); - - final String savepointPath = ((JobManagerMessages.TriggerSavepointSuccess) savepointResponse).savepointPath(); - - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), deadline.timeLeft()); - - Future cancellationResponseFuture = jobManager.ask(new JobManagerMessages.CancelJob(jobID), deadline.timeLeft()); - - Object cancellationResponse = Await.result(cancellationResponseFuture, deadline.timeLeft()); + CompletableFuture savepointPathFuture = FutureUtils.retryWithDelay( + () -> { + try { + return client.triggerSavepoint(jobID, null); + } catch (FlinkException e) { + return FutureUtils.completedExceptionally(e); + } + }, + (int) deadline.timeLeft().getSeconds() / 10, + Time.seconds(10), + (throwable) -> true, + TestingUtils.defaultScheduledExecutor() + ); - assertTrue(cancellationResponse instanceof JobManagerMessages.CancellationSuccess); + final String savepointPath = savepointPathFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - Await.ready(jobRemovedFuture, deadline.timeLeft()); + client.cancel(jobID); - // job successfully removed - jobID = null; + while (!getRunningJobs(client).isEmpty()) { + Thread.sleep(50); + } JobGraph scaledJobGraph = createJobGraphWithOperatorState(parallelism2, maxParallelism, checkpointMethod); scaledJobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath)); - jobID = scaledJobGraph.getJobID(); - - cluster.submitJobAndWait(scaledJobGraph, false); + client.setDetached(false); + client.submitJob(scaledJobGraph, RescalingITCase.class.getClassLoader()); int sumExp = 0; int sumAct = 0; @@ -609,19 +525,7 @@ public void testSavepointRescalingPartitionedOperatorState(boolean scaleOut, Ope } assertEquals(sumExp, sumAct); - jobID = null; - } finally { - // clear any left overs from a possibly failed job - if (jobID != null && jobManager != null) { - Future jobRemovedFuture = jobManager.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobID), timeout); - - try { - Await.ready(jobRemovedFuture, timeout); - } catch (TimeoutException | InterruptedException ie) { - fail("Failed while cleaning up the cluster."); - } - } } } @@ -1028,4 +932,12 @@ public void initializeState(FunctionInitializationContext context) throws Except } } } + + private static List getRunningJobs(ClusterClient client) throws Exception { + Collection statusMessages = client.listJobs().get(); + return statusMessages.stream() + .filter(status -> !status.getJobState().isGloballyTerminalState()) + .map(JobStatusMessage::getJobId) + .collect(Collectors.toList()); + } } From 4f5488c592fe153897042d24f9bd03b50767ba9a Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 19 Mar 2018 13:59:22 +0100 Subject: [PATCH 0242/2294] [FLINK-8959][tests] Port AccumulatorLiveITCase to flip6 This closes #5719. --- .../accumulators/AccumulatorLiveITCase.java | 336 +++++---------- .../LegacyAccumulatorLiveITCase.java | 386 ++++++++++++++++++ 2 files changed, 482 insertions(+), 240 deletions(-) create mode 100644 flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java index 756b81e095c095..ff362dde50246c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java @@ -18,292 +18,199 @@ package org.apache.flink.test.accumulators; -import org.apache.flink.api.common.JobExecutionResult; -import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.Plan; -import org.apache.flink.api.common.accumulators.Accumulator; import org.apache.flink.api.common.accumulators.IntCounter; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.api.common.io.OutputFormat; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.api.java.LocalEnvironment; +import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.AkkaOptions; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.HeartbeatManagerOptions; +import org.apache.flink.core.testutils.OneShotLatch; import org.apache.flink.optimizer.DataStatistics; import org.apache.flink.optimizer.Optimizer; import org.apache.flink.optimizer.plan.OptimizedPlan; import org.apache.flink.optimizer.plantranslate.JobGraphGenerator; -import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.akka.ListeningBehaviour; -import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; -import org.apache.flink.runtime.instance.ActorGateway; -import org.apache.flink.runtime.instance.AkkaActorGateway; +import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.messages.JobManagerMessages; -import org.apache.flink.runtime.testingUtils.TestingCluster; -import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages; -import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.test.util.MiniClusterResource; +import org.apache.flink.testutils.category.Flip6; import org.apache.flink.util.Collector; import org.apache.flink.util.TestLogger; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.pattern.Patterns; -import akka.testkit.JavaTestKit; -import akka.util.Timeout; -import org.junit.After; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.duration.FiniteDuration; - -import static org.junit.Assert.fail; - /** - * Tests the availability of accumulator results during runtime. The test case tests a user-defined - * accumulator and Flink's internal accumulators for two consecutive tasks. - * - *

    CHAINED[Source -> Map] -> Sink - * - *

    Checks are performed as the elements arrive at the operators. Checks consist of a message sent by - * the task to the task manager which notifies the job manager and sends the current accumulators. - * The task blocks until the test has been notified about the current accumulator values. - * - *

    A barrier between the operators ensures that that pipelining is disabled for the streaming test. - * The batch job reads the records one at a time. The streaming code buffers the records beforehand; - * that's why exact guarantees about the number of records read are very hard to make. Thus, why we - * check for an upper bound of the elements read. + * Tests the availability of accumulator results during runtime. */ +@Category(Flip6.class) public class AccumulatorLiveITCase extends TestLogger { private static final Logger LOG = LoggerFactory.getLogger(AccumulatorLiveITCase.class); - private static ActorSystem system; - private static ActorGateway jobManagerGateway; - private static ActorRef taskManager; - - private static JobID jobID; - private static JobGraph jobGraph; - // name of user accumulator private static final String ACCUMULATOR_NAME = "test"; + private static final long HEARTBEAT_INTERVAL = 50L; + // number of heartbeat intervals to check private static final int NUM_ITERATIONS = 5; - private static List inputData = new ArrayList<>(NUM_ITERATIONS); + private static final List inputData = new ArrayList<>(NUM_ITERATIONS); - private static final FiniteDuration TIMEOUT = new FiniteDuration(10, TimeUnit.SECONDS); + static { + // generate test data + for (int i = 0; i < NUM_ITERATIONS; i++) { + inputData.add(i); + } + } - @Before - public void before() throws Exception { - system = AkkaUtils.createLocalActorSystem(new Configuration()); + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + 1, + 1), + true); + private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT()); - TestingCluster testingCluster = new TestingCluster(config, false, true); - testingCluster.start(); - - jobManagerGateway = testingCluster.getLeaderGateway(TestingUtils.TESTING_DURATION()); - taskManager = testingCluster.getTaskManagersAsJava().get(0); - - // generate test data - for (int i = 0; i < NUM_ITERATIONS; i++) { - inputData.add(i, String.valueOf(i + 1)); - } + config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL); - NotifyingMapper.finished = false; + return config; } - @After - public void after() throws Exception { - JavaTestKit.shutdownActorSystem(system); - inputData.clear(); + @Before + public void resetLatches() throws InterruptedException { + NotifyingMapper.reset(); } @Test public void testBatch() throws Exception { - - /** The program **/ - ExecutionEnvironment env = new BatchPlanExtractor(); + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); - DataSet input = env.fromCollection(inputData); + DataSet input = env.fromCollection(inputData); input .flatMap(new NotifyingMapper()) - .output(new NotifyingOutputFormat()); - - env.execute(); + .output(new DummyOutputFormat()); // Extract job graph and set job id for the task to notify of accumulator changes. - jobGraph = getOptimizedPlan(((BatchPlanExtractor) env).plan); - jobID = jobGraph.getJobID(); + JobGraph jobGraph = getJobGraph(env.createProgramPlan()); - verifyResults(); + submitJobAndVerifyResults(jobGraph); } @Test public void testStreaming() throws Exception { - StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment(); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); - DataStream input = env.fromCollection(inputData); + DataStream input = env.fromCollection(inputData); input .flatMap(new NotifyingMapper()) - .writeUsingOutputFormat(new NotifyingOutputFormat()).disableChaining(); + .writeUsingOutputFormat(new DummyOutputFormat()).disableChaining(); - jobGraph = env.getStreamGraph().getJobGraph(); - jobID = jobGraph.getJobID(); + JobGraph jobGraph = env.getStreamGraph().getJobGraph(); - verifyResults(); + submitJobAndVerifyResults(jobGraph); } - private static void verifyResults() { - new JavaTestKit(system) {{ - - ActorGateway selfGateway = new AkkaActorGateway(getRef(), jobManagerGateway.leaderSessionID()); - - // register for accumulator changes - jobManagerGateway.tell(new TestingJobManagerMessages.NotifyWhenAccumulatorChange(jobID), selfGateway); - expectMsgEquals(TIMEOUT, true); - - // submit job - - jobManagerGateway.tell( - new JobManagerMessages.SubmitJob( - jobGraph, - ListeningBehaviour.EXECUTION_RESULT), - selfGateway); - expectMsgClass(TIMEOUT, JobManagerMessages.JobSubmitSuccess.class); - - TestingJobManagerMessages.UpdatedAccumulators msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); - Map> userAccumulators = msg.userAccumulators(); - - ExecutionAttemptID mapperTaskID = null; - - ExecutionAttemptID sinkTaskID = null; - - /* Check for accumulator values */ - if (checkUserAccumulators(0, userAccumulators)) { - LOG.info("Passed initial check for map task."); - } else { - fail("Wrong accumulator results when map task begins execution."); - } - - int expectedAccVal = 0; - - /* for mapper task */ - for (int i = 1; i <= NUM_ITERATIONS; i++) { - expectedAccVal += i; - - // receive message - msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); - userAccumulators = msg.userAccumulators(); - - LOG.info("{}", userAccumulators); - - if (checkUserAccumulators(expectedAccVal, userAccumulators)) { - LOG.info("Passed round #" + i); - } else if (checkUserAccumulators(expectedAccVal, userAccumulators)) { - // we determined the wrong task id and need to switch the two here - ExecutionAttemptID temp = mapperTaskID; - mapperTaskID = sinkTaskID; - sinkTaskID = temp; - LOG.info("Passed round #" + i); - } else { - fail("Failed in round #" + i); - } - } - - msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); - userAccumulators = msg.userAccumulators(); - - if (checkUserAccumulators(expectedAccVal, userAccumulators)) { - LOG.info("Passed initial check for sink task."); - } else { - fail("Wrong accumulator results when sink task begins execution."); - } - - /* for sink task */ - for (int i = 1; i <= NUM_ITERATIONS; i++) { - - // receive message - msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); - - userAccumulators = msg.userAccumulators(); - - LOG.info("{}", userAccumulators); - - if (checkUserAccumulators(expectedAccVal, userAccumulators)) { - LOG.info("Passed round #" + i); - } else { - fail("Failed in round #" + i); - } - } - - expectMsgClass(TIMEOUT, JobManagerMessages.JobResultSuccess.class); - - }}; - } - - private static boolean checkUserAccumulators(int expected, Map> accumulatorMap) { - LOG.info("checking user accumulators"); - return accumulatorMap.containsKey(ACCUMULATOR_NAME) && expected == ((IntCounter) accumulatorMap.get(ACCUMULATOR_NAME)).getLocalValue(); + private static void submitJobAndVerifyResults(JobGraph jobGraph) throws Exception { + Deadline deadline = Deadline.now().plus(Duration.ofSeconds(30)); + + ClusterClient client = MINI_CLUSTER_RESOURCE.getClusterClient(); + + client.setDetached(true); + client.submitJob(jobGraph, AccumulatorLiveITCase.class.getClassLoader()); + + try { + NotifyingMapper.notifyLatch.await(); + + FutureUtils.retrySuccesfulWithDelay( + () -> { + try { + return CompletableFuture.completedFuture(client.getAccumulators(jobGraph.getJobID())); + } catch (Exception e) { + return FutureUtils.completedExceptionally(e); + } + }, + Time.milliseconds(20), + deadline, + accumulators -> accumulators.size() == 1 + && accumulators.containsKey(ACCUMULATOR_NAME) + && (int) accumulators.get(ACCUMULATOR_NAME) == NUM_ITERATIONS, + TestingUtils.defaultScheduledExecutor() + ).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); + + NotifyingMapper.shutdownLatch.trigger(); + } finally { + NotifyingMapper.shutdownLatch.trigger(); + } } /** * UDF that notifies when it changes the accumulator values. */ - private static class NotifyingMapper extends RichFlatMapFunction { + private static class NotifyingMapper extends RichFlatMapFunction { private static final long serialVersionUID = 1L; - private IntCounter counter = new IntCounter(); + private static final OneShotLatch notifyLatch = new OneShotLatch(); + private static final OneShotLatch shutdownLatch = new OneShotLatch(); - private static boolean finished = false; + private final IntCounter counter = new IntCounter(); @Override public void open(Configuration parameters) throws Exception { getRuntimeContext().addAccumulator(ACCUMULATOR_NAME, counter); - notifyTaskManagerOfAccumulatorUpdate(); } @Override - public void flatMap(String value, Collector out) throws Exception { - int val = Integer.valueOf(value); - counter.add(val); - out.collect(val); + public void flatMap(Integer value, Collector out) throws Exception { + counter.add(1); + out.collect(value); LOG.debug("Emitting value {}.", value); - notifyTaskManagerOfAccumulatorUpdate(); + if (counter.getLocalValuePrimitive() == 5) { + notifyLatch.trigger(); + } } @Override public void close() throws Exception { - finished = true; + shutdownLatch.await(); + } + + private static void reset() throws InterruptedException { + notifyLatch.reset(); + shutdownLatch.reset(); } } /** - * Outputs format which notifies of accumulator changes and waits for the previous mapper. + * Outputs format which waits for the previous mapper. */ - private static class NotifyingOutputFormat implements OutputFormat { + private static class DummyOutputFormat implements OutputFormat { private static final long serialVersionUID = 1L; @Override @@ -312,17 +219,10 @@ public void configure(Configuration parameters) { @Override public void open(int taskNumber, int numTasks) throws IOException { - while (!NotifyingMapper.finished) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) {} - } - notifyTaskManagerOfAccumulatorUpdate(); } @Override public void writeRecord(Integer record) throws IOException { - notifyTaskManagerOfAccumulatorUpdate(); } @Override @@ -330,57 +230,13 @@ public void close() throws IOException { } } - /** - * Notify task manager of accumulator update and wait until the Heartbeat containing the message - * has been reported. - */ - public static void notifyTaskManagerOfAccumulatorUpdate() { - new JavaTestKit(system) {{ - Timeout timeout = new Timeout(TIMEOUT); - Future ask = Patterns.ask(taskManager, new TestingTaskManagerMessages.AccumulatorsChanged(jobID), timeout); - try { - Await.result(ask, timeout.duration()); - } catch (Exception e) { - fail("Failed to notify task manager of accumulator update."); - } - }}; - } - /** * Helpers to generate the JobGraph. */ - private static JobGraph getOptimizedPlan(Plan plan) { + private static JobGraph getJobGraph(Plan plan) { Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); JobGraphGenerator jgg = new JobGraphGenerator(); OptimizedPlan op = pc.compile(plan); return jgg.compileJobGraph(op); } - - private static class BatchPlanExtractor extends LocalEnvironment { - - private Plan plan = null; - - @Override - public JobExecutionResult execute(String jobName) throws Exception { - plan = createProgramPlan(); - return new JobExecutionResult(new JobID(), -1, null); - } - } - - /** - * This is used to for creating the example topology. {@link #execute} is never called, we - * only use this to call {@link #getStreamGraph()}. - */ - private static class DummyStreamExecutionEnvironment extends StreamExecutionEnvironment { - - @Override - public JobExecutionResult execute() throws Exception { - return execute("default"); - } - - @Override - public JobExecutionResult execute(String jobName) throws Exception { - throw new RuntimeException("This should not be called."); - } - } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java new file mode 100644 index 00000000000000..6595b100c15813 --- /dev/null +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.test.accumulators; + +import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.Plan; +import org.apache.flink.api.common.accumulators.Accumulator; +import org.apache.flink.api.common.accumulators.IntCounter; +import org.apache.flink.api.common.functions.RichFlatMapFunction; +import org.apache.flink.api.common.io.OutputFormat; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.api.java.LocalEnvironment; +import org.apache.flink.configuration.AkkaOptions; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.optimizer.DataStatistics; +import org.apache.flink.optimizer.Optimizer; +import org.apache.flink.optimizer.plan.OptimizedPlan; +import org.apache.flink.optimizer.plantranslate.JobGraphGenerator; +import org.apache.flink.runtime.akka.AkkaUtils; +import org.apache.flink.runtime.akka.ListeningBehaviour; +import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; +import org.apache.flink.runtime.instance.ActorGateway; +import org.apache.flink.runtime.instance.AkkaActorGateway; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.messages.JobManagerMessages; +import org.apache.flink.runtime.testingUtils.TestingCluster; +import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages; +import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages; +import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.util.Collector; +import org.apache.flink.util.TestLogger; + +import akka.actor.ActorRef; +import akka.actor.ActorSystem; +import akka.pattern.Patterns; +import akka.testkit.JavaTestKit; +import akka.util.Timeout; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import scala.concurrent.Await; +import scala.concurrent.Future; +import scala.concurrent.duration.FiniteDuration; + +import static org.junit.Assert.fail; + +/** + * Tests the availability of accumulator results during runtime. The test case tests a user-defined + * accumulator and Flink's internal accumulators for two consecutive tasks. + * + *

    CHAINED[Source -> Map] -> Sink + * + *

    Checks are performed as the elements arrive at the operators. Checks consist of a message sent by + * the task to the task manager which notifies the job manager and sends the current accumulators. + * The task blocks until the test has been notified about the current accumulator values. + * + *

    A barrier between the operators ensures that that pipelining is disabled for the streaming test. + * The batch job reads the records one at a time. The streaming code buffers the records beforehand; + * that's why exact guarantees about the number of records read are very hard to make. Thus, why we + * check for an upper bound of the elements read. + */ +public class LegacyAccumulatorLiveITCase extends TestLogger { + + private static final Logger LOG = LoggerFactory.getLogger(LegacyAccumulatorLiveITCase.class); + + private static ActorSystem system; + private static ActorGateway jobManagerGateway; + private static ActorRef taskManager; + + private static JobID jobID; + private static JobGraph jobGraph; + + // name of user accumulator + private static final String ACCUMULATOR_NAME = "test"; + + // number of heartbeat intervals to check + private static final int NUM_ITERATIONS = 5; + + private static List inputData = new ArrayList<>(NUM_ITERATIONS); + + private static final FiniteDuration TIMEOUT = new FiniteDuration(10, TimeUnit.SECONDS); + + @Before + public void before() throws Exception { + system = AkkaUtils.createLocalActorSystem(new Configuration()); + + Configuration config = new Configuration(); + config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); + config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT()); + TestingCluster testingCluster = new TestingCluster(config, false, true); + testingCluster.start(); + + jobManagerGateway = testingCluster.getLeaderGateway(TestingUtils.TESTING_DURATION()); + taskManager = testingCluster.getTaskManagersAsJava().get(0); + + // generate test data + for (int i = 0; i < NUM_ITERATIONS; i++) { + inputData.add(i, String.valueOf(i + 1)); + } + + NotifyingMapper.finished = false; + } + + @After + public void after() throws Exception { + JavaTestKit.shutdownActorSystem(system); + inputData.clear(); + } + + @Test + public void testBatch() throws Exception { + + /** The program **/ + ExecutionEnvironment env = new BatchPlanExtractor(); + env.setParallelism(1); + + DataSet input = env.fromCollection(inputData); + input + .flatMap(new NotifyingMapper()) + .output(new NotifyingOutputFormat()); + + env.execute(); + + // Extract job graph and set job id for the task to notify of accumulator changes. + jobGraph = getOptimizedPlan(((BatchPlanExtractor) env).plan); + jobID = jobGraph.getJobID(); + + verifyResults(); + } + + @Test + public void testStreaming() throws Exception { + + StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment(); + env.setParallelism(1); + + DataStream input = env.fromCollection(inputData); + input + .flatMap(new NotifyingMapper()) + .writeUsingOutputFormat(new NotifyingOutputFormat()).disableChaining(); + + jobGraph = env.getStreamGraph().getJobGraph(); + jobID = jobGraph.getJobID(); + + verifyResults(); + } + + private static void verifyResults() { + new JavaTestKit(system) {{ + + ActorGateway selfGateway = new AkkaActorGateway(getRef(), jobManagerGateway.leaderSessionID()); + + // register for accumulator changes + jobManagerGateway.tell(new TestingJobManagerMessages.NotifyWhenAccumulatorChange(jobID), selfGateway); + expectMsgEquals(TIMEOUT, true); + + // submit job + + jobManagerGateway.tell( + new JobManagerMessages.SubmitJob( + jobGraph, + ListeningBehaviour.EXECUTION_RESULT), + selfGateway); + expectMsgClass(TIMEOUT, JobManagerMessages.JobSubmitSuccess.class); + + TestingJobManagerMessages.UpdatedAccumulators msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); + Map> userAccumulators = msg.userAccumulators(); + + ExecutionAttemptID mapperTaskID = null; + + ExecutionAttemptID sinkTaskID = null; + + /* Check for accumulator values */ + if (checkUserAccumulators(0, userAccumulators)) { + LOG.info("Passed initial check for map task."); + } else { + fail("Wrong accumulator results when map task begins execution."); + } + + int expectedAccVal = 0; + + /* for mapper task */ + for (int i = 1; i <= NUM_ITERATIONS; i++) { + expectedAccVal += i; + + // receive message + msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); + userAccumulators = msg.userAccumulators(); + + LOG.info("{}", userAccumulators); + + if (checkUserAccumulators(expectedAccVal, userAccumulators)) { + LOG.info("Passed round #" + i); + } else if (checkUserAccumulators(expectedAccVal, userAccumulators)) { + // we determined the wrong task id and need to switch the two here + ExecutionAttemptID temp = mapperTaskID; + mapperTaskID = sinkTaskID; + sinkTaskID = temp; + LOG.info("Passed round #" + i); + } else { + fail("Failed in round #" + i); + } + } + + msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); + userAccumulators = msg.userAccumulators(); + + if (checkUserAccumulators(expectedAccVal, userAccumulators)) { + LOG.info("Passed initial check for sink task."); + } else { + fail("Wrong accumulator results when sink task begins execution."); + } + + /* for sink task */ + for (int i = 1; i <= NUM_ITERATIONS; i++) { + + // receive message + msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); + + userAccumulators = msg.userAccumulators(); + + LOG.info("{}", userAccumulators); + + if (checkUserAccumulators(expectedAccVal, userAccumulators)) { + LOG.info("Passed round #" + i); + } else { + fail("Failed in round #" + i); + } + } + + expectMsgClass(TIMEOUT, JobManagerMessages.JobResultSuccess.class); + + }}; + } + + private static boolean checkUserAccumulators(int expected, Map> accumulatorMap) { + LOG.info("checking user accumulators"); + return accumulatorMap.containsKey(ACCUMULATOR_NAME) && expected == ((IntCounter) accumulatorMap.get(ACCUMULATOR_NAME)).getLocalValue(); + } + + /** + * UDF that notifies when it changes the accumulator values. + */ + private static class NotifyingMapper extends RichFlatMapFunction { + private static final long serialVersionUID = 1L; + + private IntCounter counter = new IntCounter(); + + private static boolean finished = false; + + @Override + public void open(Configuration parameters) throws Exception { + getRuntimeContext().addAccumulator(ACCUMULATOR_NAME, counter); + notifyTaskManagerOfAccumulatorUpdate(); + } + + @Override + public void flatMap(String value, Collector out) throws Exception { + int val = Integer.valueOf(value); + counter.add(val); + out.collect(val); + LOG.debug("Emitting value {}.", value); + notifyTaskManagerOfAccumulatorUpdate(); + } + + @Override + public void close() throws Exception { + finished = true; + } + } + + /** + * Outputs format which notifies of accumulator changes and waits for the previous mapper. + */ + private static class NotifyingOutputFormat implements OutputFormat { + private static final long serialVersionUID = 1L; + + @Override + public void configure(Configuration parameters) { + } + + @Override + public void open(int taskNumber, int numTasks) throws IOException { + while (!NotifyingMapper.finished) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) {} + } + notifyTaskManagerOfAccumulatorUpdate(); + } + + @Override + public void writeRecord(Integer record) throws IOException { + notifyTaskManagerOfAccumulatorUpdate(); + } + + @Override + public void close() throws IOException { + } + } + + /** + * Notify task manager of accumulator update and wait until the Heartbeat containing the message + * has been reported. + */ + public static void notifyTaskManagerOfAccumulatorUpdate() { + new JavaTestKit(system) {{ + Timeout timeout = new Timeout(TIMEOUT); + Future ask = Patterns.ask(taskManager, new TestingTaskManagerMessages.AccumulatorsChanged(jobID), timeout); + try { + Await.result(ask, timeout.duration()); + } catch (Exception e) { + fail("Failed to notify task manager of accumulator update."); + } + }}; + } + + /** + * Helpers to generate the JobGraph. + */ + private static JobGraph getOptimizedPlan(Plan plan) { + Optimizer pc = new Optimizer(new DataStatistics(), new Configuration()); + JobGraphGenerator jgg = new JobGraphGenerator(); + OptimizedPlan op = pc.compile(plan); + return jgg.compileJobGraph(op); + } + + private static class BatchPlanExtractor extends LocalEnvironment { + + private Plan plan = null; + + @Override + public JobExecutionResult execute(String jobName) throws Exception { + plan = createProgramPlan(); + return new JobExecutionResult(new JobID(), -1, null); + } + } + + /** + * This is used to for creating the example topology. {@link #execute} is never called, we + * only use this to call {@link #getStreamGraph()}. + */ + private static class DummyStreamExecutionEnvironment extends StreamExecutionEnvironment { + + @Override + public JobExecutionResult execute() throws Exception { + return execute("default"); + } + + @Override + public JobExecutionResult execute(String jobName) throws Exception { + throw new RuntimeException("This should not be called."); + } + } +} From 0623e24c8814e073426062e8b27bf88e664ee3aa Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 19 Mar 2018 14:17:34 +0100 Subject: [PATCH 0243/2294] [FLINK-8957][tests] Port JMXJobManagerMetricTest to flip6 This closes #5720. --- flink-metrics/flink-metrics-jmx/pom.xml | 6 ++ .../jobmanager/JMXJobManagerMetricTest.java | 69 ++++++++++--------- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/flink-metrics/flink-metrics-jmx/pom.xml b/flink-metrics/flink-metrics-jmx/pom.xml index d738a7e9abeab2..123f8e7140cb60 100644 --- a/flink-metrics/flink-metrics-jmx/pom.xml +++ b/flink-metrics/flink-metrics-jmx/pom.xml @@ -85,5 +85,11 @@ under the License. ${project.version} test + + org.apache.flink + flink-test-utils_${scala.binary.version} + ${project.version} + test + diff --git a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java index 6770ec326de649..c00b5d357cb293 100644 --- a/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java +++ b/flink-metrics/flink-metrics-jmx/src/test/java/org/apache/flink/runtime/jobmanager/JMXJobManagerMetricTest.java @@ -18,38 +18,40 @@ package org.apache.flink.runtime.jobmanager; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.MetricOptions; import org.apache.flink.core.testutils.OneShotLatch; import org.apache.flink.metrics.jmx.JMXReporter; import org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy; +import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings; -import org.apache.flink.runtime.testingUtils.TestingCluster; -import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages; +import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.test.util.MiniClusterResource; import org.junit.Assert; +import org.junit.ClassRule; import org.junit.Test; import javax.management.MBeanServer; import javax.management.ObjectName; import java.lang.management.ManagementFactory; +import java.time.Duration; import java.util.Collections; import java.util.Set; import java.util.concurrent.TimeUnit; -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.duration.Deadline; -import scala.concurrent.duration.FiniteDuration; - import static org.junit.Assert.assertEquals; /** @@ -57,24 +59,31 @@ */ public class JMXJobManagerMetricTest { - /** - * Tests that metrics registered on the JobManager are actually accessible via JMX. - */ - @Test - public void testJobManagerJMXMetricAccess() throws Exception { - Deadline deadline = new FiniteDuration(2, TimeUnit.MINUTES).fromNow(); + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + 1, + 1), + true); + + private static Configuration getConfiguration() { Configuration flinkConfiguration = new Configuration(); flinkConfiguration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName()); - flinkConfiguration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.port", "9060-9075"); - flinkConfiguration.setString(MetricOptions.SCOPE_NAMING_JM_JOB, "jobmanager."); - TestingCluster flink = new TestingCluster(flinkConfiguration); + return flinkConfiguration; + } - try { - flink.start(); + /** + * Tests that metrics registered on the JobManager are actually accessible via JMX. + */ + @Test + public void testJobManagerJMXMetricAccess() throws Exception { + Deadline deadline = Deadline.now().plus(Duration.ofMinutes(2)); + try { JobVertex sourceJobVertex = new JobVertex("Source"); sourceJobVertex.setInvokableClass(BlockingInvokable.class); @@ -92,28 +101,26 @@ public void testJobManagerJMXMetricAccess() throws Exception { true), null)); - flink.waitForActorsToBeAlive(); - - flink.submitJobDetached(jobGraph); + ClusterClient client = MINI_CLUSTER_RESOURCE.getClusterClient(); + client.setDetached(true); + client.submitJob(jobGraph, JMXJobManagerMetricTest.class.getClassLoader()); - Future jobRunning = flink.getLeaderGateway(deadline.timeLeft()) - .ask(new TestingJobManagerMessages.WaitForAllVerticesToBeRunning(jobGraph.getJobID()), deadline.timeLeft()); - Await.ready(jobRunning, deadline.timeLeft()); + FutureUtils.retrySuccesfulWithDelay( + () -> client.getJobStatus(jobGraph.getJobID()), + Time.milliseconds(10), + deadline, + status -> status == JobStatus.RUNNING, + TestingUtils.defaultScheduledExecutor() + ).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); Set nameSet = mBeanServer.queryNames(new ObjectName("org.apache.flink.jobmanager.job.lastCheckpointSize:job_name=TestingJob,*"), null); Assert.assertEquals(1, nameSet.size()); assertEquals(-1L, mBeanServer.getAttribute(nameSet.iterator().next(), "Value")); - Future jobFinished = flink.getLeaderGateway(deadline.timeLeft()) - .ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobGraph.getJobID()), deadline.timeLeft()); - BlockingInvokable.unblock(); - - // wait til the job has finished - Await.ready(jobFinished, deadline.timeLeft()); } finally { - flink.stop(); + BlockingInvokable.unblock(); } } From 0c56e1917aa3a563a7425ba98ff33ed9bfcd22c5 Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 19 Mar 2018 15:16:18 +0100 Subject: [PATCH 0244/2294] [FLINK-8958][tests] Port TaskCancelAsyncProducerConsumerITCase to flip6 This closes #5722. --- ...TaskCancelAsyncProducerConsumerITCase.java | 287 ++++++++++++++++++ ...TaskCancelAsyncProducerConsumerITCase.java | 82 ++--- 2 files changed, 329 insertions(+), 40 deletions(-) create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LegacyTaskCancelAsyncProducerConsumerITCase.java diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LegacyTaskCancelAsyncProducerConsumerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LegacyTaskCancelAsyncProducerConsumerITCase.java new file mode 100644 index 00000000000000..ee0bfda39671d9 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/LegacyTaskCancelAsyncProducerConsumerITCase.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.taskmanager; + +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.instance.ActorGateway; +import org.apache.flink.runtime.io.network.api.writer.RecordWriter; +import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter; +import org.apache.flink.runtime.io.network.partition.ResultPartitionType; +import org.apache.flink.runtime.io.network.partition.consumer.InputGate; +import org.apache.flink.runtime.jobgraph.DistributionPattern; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.jobgraph.JobVertex; +import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; +import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; +import org.apache.flink.runtime.messages.JobManagerMessages.CancelJob; +import org.apache.flink.runtime.testingUtils.TestingCluster; +import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobStatus; +import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.WaitForAllVerticesToBeRunning; +import org.apache.flink.types.LongValue; +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import scala.concurrent.Await; +import scala.concurrent.Future; +import scala.concurrent.duration.Deadline; +import scala.concurrent.duration.FiniteDuration; + +import static org.apache.flink.runtime.io.network.buffer.LocalBufferPoolDestroyTest.isInBlockingBufferRequest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class LegacyTaskCancelAsyncProducerConsumerITCase extends TestLogger { + + // The Exceptions thrown by the producer/consumer Threads + private static volatile Exception ASYNC_PRODUCER_EXCEPTION; + private static volatile Exception ASYNC_CONSUMER_EXCEPTION; + + // The Threads producing/consuming the intermediate stream + private static volatile Thread ASYNC_PRODUCER_THREAD; + private static volatile Thread ASYNC_CONSUMER_THREAD; + + /** + * Tests that a task waiting on an async producer/consumer that is stuck + * in a blocking buffer request can be properly cancelled. + * + *

    This is currently required for the Flink Kafka sources, which spawn + * a separate Thread consuming from Kafka and producing the intermediate + * streams in the spawned Thread instead of the main task Thread. + */ + @Test + public void testCancelAsyncProducerAndConsumer() throws Exception { + Deadline deadline = new FiniteDuration(2, TimeUnit.MINUTES).fromNow(); + TestingCluster flink = null; + + try { + // Cluster + Configuration config = new Configuration(); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); + config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); + config.setInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE, 4096); + config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 9); + + flink = new TestingCluster(config, true); + flink.start(); + + // Job with async producer and consumer + JobVertex producer = new JobVertex("AsyncProducer"); + producer.setParallelism(1); + producer.setInvokableClass(AsyncProducer.class); + + JobVertex consumer = new JobVertex("AsyncConsumer"); + consumer.setParallelism(1); + consumer.setInvokableClass(AsyncConsumer.class); + consumer.connectNewDataSetAsInput(producer, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); + + SlotSharingGroup slot = new SlotSharingGroup(producer.getID(), consumer.getID()); + producer.setSlotSharingGroup(slot); + consumer.setSlotSharingGroup(slot); + + JobGraph jobGraph = new JobGraph(producer, consumer); + + // Submit job and wait until running + ActorGateway jobManager = flink.getLeaderGateway(deadline.timeLeft()); + flink.submitJobDetached(jobGraph); + + Object msg = new WaitForAllVerticesToBeRunning(jobGraph.getJobID()); + Future runningFuture = jobManager.ask(msg, deadline.timeLeft()); + Await.ready(runningFuture, deadline.timeLeft()); + + // Wait for blocking requests, cancel and wait for cancellation + msg = new NotifyWhenJobStatus(jobGraph.getJobID(), JobStatus.CANCELED); + Future cancelledFuture = jobManager.ask(msg, deadline.timeLeft()); + + boolean producerBlocked = false; + for (int i = 0; i < 50; i++) { + Thread thread = ASYNC_PRODUCER_THREAD; + + if (thread != null && thread.isAlive()) { + StackTraceElement[] stackTrace = thread.getStackTrace(); + producerBlocked = isInBlockingBufferRequest(stackTrace); + } + + if (producerBlocked) { + break; + } else { + // Retry + Thread.sleep(500L); + } + } + + // Verify that async producer is in blocking request + assertTrue("Producer thread is not blocked: " + Arrays.toString(ASYNC_PRODUCER_THREAD.getStackTrace()), producerBlocked); + + boolean consumerWaiting = false; + for (int i = 0; i < 50; i++) { + Thread thread = ASYNC_CONSUMER_THREAD; + + if (thread != null && thread.isAlive()) { + consumerWaiting = thread.getState() == Thread.State.WAITING; + } + + if (consumerWaiting) { + break; + } else { + // Retry + Thread.sleep(500L); + } + } + + // Verify that async consumer is in blocking request + assertTrue("Consumer thread is not blocked.", consumerWaiting); + + msg = new CancelJob(jobGraph.getJobID()); + Future cancelFuture = jobManager.ask(msg, deadline.timeLeft()); + Await.ready(cancelFuture, deadline.timeLeft()); + + Await.ready(cancelledFuture, deadline.timeLeft()); + + // Verify the expected Exceptions + assertNotNull(ASYNC_PRODUCER_EXCEPTION); + assertEquals(IllegalStateException.class, ASYNC_PRODUCER_EXCEPTION.getClass()); + + assertNotNull(ASYNC_CONSUMER_EXCEPTION); + assertEquals(IllegalStateException.class, ASYNC_CONSUMER_EXCEPTION.getClass()); + } finally { + if (flink != null) { + flink.stop(); + } + } + } + + /** + * Invokable emitting records in a separate Thread (not the main Task + * thread). + */ + public static class AsyncProducer extends AbstractInvokable { + + public AsyncProducer(Environment environment) { + super(environment); + } + + @Override + public void invoke() throws Exception { + Thread producer = new ProducerThread(getEnvironment().getWriter(0)); + + // Publish the async producer for the main test Thread + ASYNC_PRODUCER_THREAD = producer; + + producer.start(); + + // Wait for the producer Thread to finish. This is executed in the + // main Task thread and will be interrupted on cancellation. + while (producer.isAlive()) { + try { + producer.join(); + } catch (InterruptedException ignored) { + } + } + } + + /** + * The Thread emitting the records. + */ + private static class ProducerThread extends Thread { + + private final RecordWriter recordWriter; + + public ProducerThread(ResultPartitionWriter partitionWriter) { + this.recordWriter = new RecordWriter<>(partitionWriter); + } + + @Override + public void run() { + LongValue current = new LongValue(0); + + try { + while (true) { + current.setValue(current.getValue() + 1); + recordWriter.emit(current); + recordWriter.flushAll(); + } + } catch (Exception e) { + ASYNC_PRODUCER_EXCEPTION = e; + } + } + } + } + + /** + * Invokable consuming buffers in a separate Thread (not the main Task + * thread). + */ + public static class AsyncConsumer extends AbstractInvokable { + + public AsyncConsumer(Environment environment) { + super(environment); + } + + @Override + public void invoke() throws Exception { + Thread consumer = new ConsumerThread(getEnvironment().getInputGate(0)); + + // Publish the async consumer for the main test Thread + ASYNC_CONSUMER_THREAD = consumer; + + consumer.start(); + + // Wait for the consumer Thread to finish. This is executed in the + // main Task thread and will be interrupted on cancellation. + while (consumer.isAlive()) { + try { + consumer.join(); + } catch (InterruptedException ignored) { + } + } + } + + /** + * The Thread consuming buffers. + */ + private static class ConsumerThread extends Thread { + + private final InputGate inputGate; + + public ConsumerThread(InputGate inputGate) { + this.inputGate = inputGate; + } + + @Override + public void run() { + try { + while (true) { + inputGate.getNextBufferOrEvent(); + } + } catch (Exception e) { + ASYNC_CONSUMER_EXCEPTION = e; + } + } + } + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java index c63af83dc20a57..4b73b0925ff507 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java @@ -18,11 +18,12 @@ package org.apache.flink.runtime.taskmanager; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.execution.Environment; -import org.apache.flink.runtime.instance.ActorGateway; import org.apache.flink.runtime.io.network.api.writer.RecordWriter; import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter; import org.apache.flink.runtime.io.network.partition.ResultPartitionType; @@ -33,28 +34,26 @@ import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; -import org.apache.flink.runtime.messages.JobManagerMessages.CancelJob; -import org.apache.flink.runtime.testingUtils.TestingCluster; -import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobStatus; -import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.WaitForAllVerticesToBeRunning; +import org.apache.flink.runtime.minicluster.MiniCluster; +import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; +import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.testutils.category.Flip6; import org.apache.flink.types.LongValue; import org.apache.flink.util.TestLogger; import org.junit.Test; +import org.junit.experimental.categories.Category; +import java.time.Duration; import java.util.Arrays; import java.util.concurrent.TimeUnit; -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.duration.Deadline; -import scala.concurrent.duration.FiniteDuration; - import static org.apache.flink.runtime.io.network.buffer.LocalBufferPoolDestroyTest.isInBlockingBufferRequest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +@Category(Flip6.class) public class TaskCancelAsyncProducerConsumerITCase extends TestLogger { // The Exceptions thrown by the producer/consumer Threads @@ -75,18 +74,20 @@ public class TaskCancelAsyncProducerConsumerITCase extends TestLogger { */ @Test public void testCancelAsyncProducerAndConsumer() throws Exception { - Deadline deadline = new FiniteDuration(2, TimeUnit.MINUTES).fromNow(); - TestingCluster flink = null; - - try { - // Cluster - Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); - config.setInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE, 4096); - config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 9); - - flink = new TestingCluster(config, true); + Deadline deadline = Deadline.now().plus(Duration.ofMinutes(2)); + + // Cluster + Configuration config = new Configuration(); + config.setInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE, 4096); + config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 9); + + MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() + .setConfiguration(config) + .setNumTaskManagers(1) + .setNumSlotsPerTaskManager(1) + .build(); + + try (MiniCluster flink = new MiniCluster(miniClusterConfiguration)) { flink.start(); // Job with async producer and consumer @@ -106,16 +107,15 @@ public void testCancelAsyncProducerAndConsumer() throws Exception { JobGraph jobGraph = new JobGraph(producer, consumer); // Submit job and wait until running - ActorGateway jobManager = flink.getLeaderGateway(deadline.timeLeft()); - flink.submitJobDetached(jobGraph); - - Object msg = new WaitForAllVerticesToBeRunning(jobGraph.getJobID()); - Future runningFuture = jobManager.ask(msg, deadline.timeLeft()); - Await.ready(runningFuture, deadline.timeLeft()); + flink.runDetached(jobGraph); - // Wait for blocking requests, cancel and wait for cancellation - msg = new NotifyWhenJobStatus(jobGraph.getJobID(), JobStatus.CANCELED); - Future cancelledFuture = jobManager.ask(msg, deadline.timeLeft()); + FutureUtils.retrySuccesfulWithDelay( + () -> flink.getJobStatus(jobGraph.getJobID()), + Time.milliseconds(10), + deadline, + status -> status == JobStatus.RUNNING, + TestingUtils.defaultScheduledExecutor() + ).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); boolean producerBlocked = false; for (int i = 0; i < 50; i++) { @@ -156,11 +156,17 @@ public void testCancelAsyncProducerAndConsumer() throws Exception { // Verify that async consumer is in blocking request assertTrue("Consumer thread is not blocked.", consumerWaiting); - msg = new CancelJob(jobGraph.getJobID()); - Future cancelFuture = jobManager.ask(msg, deadline.timeLeft()); - Await.ready(cancelFuture, deadline.timeLeft()); + flink.cancelJob(jobGraph.getJobID()) + .get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); - Await.ready(cancelledFuture, deadline.timeLeft()); + // wait until the job is canceled + FutureUtils.retrySuccesfulWithDelay( + () -> flink.getJobStatus(jobGraph.getJobID()), + Time.milliseconds(10), + deadline, + status -> status == JobStatus.CANCELED, + TestingUtils.defaultScheduledExecutor() + ).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); // Verify the expected Exceptions assertNotNull(ASYNC_PRODUCER_EXCEPTION); @@ -168,10 +174,6 @@ public void testCancelAsyncProducerAndConsumer() throws Exception { assertNotNull(ASYNC_CONSUMER_EXCEPTION); assertEquals(IllegalStateException.class, ASYNC_CONSUMER_EXCEPTION.getClass()); - } finally { - if (flink != null) { - flink.stop(); - } } } From 0ce8574603aac784da30a354f608ff939e09ef58 Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 20 Mar 2018 11:38:24 +0100 Subject: [PATCH 0245/2294] [hotfix][utils] Add ExceptionUtils#findThrowable with predicate --- .../org/apache/flink/util/ExceptionUtils.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java index 6af16fcfa4f6c1..459648fdc82659 100644 --- a/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java +++ b/flink-core/src/main/java/org/apache/flink/util/ExceptionUtils.java @@ -35,6 +35,7 @@ import java.util.Optional; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; +import java.util.function.Predicate; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -325,6 +326,30 @@ public static Optional findThrowable(Throwable throwabl return Optional.empty(); } + /** + * Checks whether a throwable chain contains an exception matching a predicate and returns it. + * + * @param throwable the throwable chain to check. + * @param predicate the predicate of the exception to search for in the chain. + * @return Optional throwable of the requested type if available, otherwise empty + */ + public static Optional findThrowable(Throwable throwable, Predicate predicate) { + if (throwable == null || predicate == null) { + return Optional.empty(); + } + + Throwable t = throwable; + while (t != null) { + if (predicate.test(t)) { + return Optional.of(t); + } else { + t = t.getCause(); + } + } + + return Optional.empty(); + } + /** * Checks whether a throwable chain contains a specific error message and returns the corresponding throwable. * From adeff9267ff23ebd14de39533341713241f25dfb Mon Sep 17 00:00:00 2001 From: zentol Date: Tue, 20 Mar 2018 11:40:48 +0100 Subject: [PATCH 0246/2294] [FLINK-8964][tests] Port JobSubmissionFailsITCase to flip6 This closes #5727. --- .../failing/JobSubmissionFailsITCase.java | 169 ++++++------------ 1 file changed, 55 insertions(+), 114 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java index a647af9c447c08..ecd16a1ff30b85 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java @@ -19,30 +19,25 @@ package org.apache.flink.test.example.failing; -import org.apache.flink.api.common.JobExecutionResult; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.runtime.client.JobExecutionException; -import org.apache.flink.runtime.client.JobSubmissionException; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobVertex; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; -import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testtasks.NoOpInvokable; +import org.apache.flink.test.util.MiniClusterResource; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.util.Arrays; import java.util.Collection; +import java.util.Optional; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -51,47 +46,32 @@ @RunWith(Parameterized.class) public class JobSubmissionFailsITCase extends TestLogger { + private static final int NUM_TM = 2; private static final int NUM_SLOTS = 20; - private static LocalFlinkMiniCluster cluster; - private static JobGraph workingJobGraph; - - @BeforeClass - public static void setup() { - try { - Configuration config = new Configuration(); - config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_SLOTS / 2); - - cluster = new LocalFlinkMiniCluster(config); - - cluster.start(); - - final JobVertex jobVertex = new JobVertex("Working job vertex."); - jobVertex.setInvokableClass(NoOpInvokable.class); - workingJobGraph = new JobGraph("Working testing job", jobVertex); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + NUM_TM, + NUM_SLOTS / NUM_TM), + true); + + private static Configuration getConfiguration() { + Configuration config = new Configuration(); + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); + return config; } - @AfterClass - public static void teardown() { - try { - cluster.stop(); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } + private static JobGraph getWorkingJobGraph() { + final JobVertex jobVertex = new JobVertex("Working job vertex."); + jobVertex.setInvokableClass(NoOpInvokable.class); + return new JobGraph("Working testing job", jobVertex); } // -------------------------------------------------------------------------------------------- - private boolean detached; + private final boolean detached; public JobSubmissionFailsITCase(boolean detached) { this.detached = detached; @@ -105,90 +85,51 @@ public static Collection executionModes(){ // -------------------------------------------------------------------------------------------- - private JobExecutionResult submitJob(JobGraph jobGraph) throws Exception { - if (detached) { - cluster.submitJobDetached(jobGraph); - return null; - } - else { - return cluster.submitJobAndWait(jobGraph, false, TestingUtils.TESTING_DURATION()); - } - } - @Test - public void testExceptionInInitializeOnMaster() { - try { - final JobVertex failingJobVertex = new FailingJobVertex("Failing job vertex"); - failingJobVertex.setInvokableClass(NoOpInvokable.class); - - final JobGraph failingJobGraph = new JobGraph("Failing testing job", failingJobVertex); + public void testExceptionInInitializeOnMaster() throws Exception { + final JobVertex failingJobVertex = new FailingJobVertex("Failing job vertex"); + failingJobVertex.setInvokableClass(NoOpInvokable.class); - try { - submitJob(failingJobGraph); - fail("Expected JobExecutionException."); - } - catch (JobExecutionException e) { - assertEquals("Test exception.", e.getCause().getMessage()); - } - catch (Throwable t) { - t.printStackTrace(); - fail("Caught wrong exception of type " + t.getClass() + "."); - } + final JobGraph failingJobGraph = new JobGraph("Failing testing job", failingJobVertex); - cluster.submitJobAndWait(workingJobGraph, false); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } + ClusterClient client = MINI_CLUSTER_RESOURCE.getClusterClient(); + client.setDetached(detached); - @Test - public void testSubmitEmptyJobGraph() { try { - final JobGraph jobGraph = new JobGraph("Testing job"); - - try { - submitJob(jobGraph); - fail("Expected JobSubmissionException."); - } - catch (JobSubmissionException e) { - assertTrue(e.getMessage() != null && e.getMessage().contains("empty")); + client.submitJob(failingJobGraph, JobSubmissionFailsITCase.class.getClassLoader()); + fail("Job submission should have thrown an exception."); + } catch (Exception e) { + Optional expectedCause = ExceptionUtils.findThrowable(e, + candidate -> "Test exception.".equals(candidate.getMessage())); + if (!expectedCause.isPresent()) { + throw e; } - catch (Throwable t) { - t.printStackTrace(); - fail("Caught wrong exception of type " + t.getClass() + "."); - } - - cluster.submitJobAndWait(workingJobGraph, false); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); } + + client.setDetached(false); + client.submitJob(getWorkingJobGraph(), JobSubmissionFailsITCase.class.getClassLoader()); } @Test - public void testSubmitNullJobGraph() { + public void testSubmitEmptyJobGraph() throws Exception { + final JobGraph jobGraph = new JobGraph("Testing job"); + + ClusterClient client = MINI_CLUSTER_RESOURCE.getClusterClient(); + client.setDetached(detached); + try { - try { - submitJob(null); - fail("Expected JobSubmissionException."); - } - catch (NullPointerException e) { - // yo! + client.submitJob(jobGraph, JobSubmissionFailsITCase.class.getClassLoader()); + fail("Job submission should have thrown an exception."); + } catch (Exception e) { + Optional expectedCause = ExceptionUtils.findThrowable(e, + throwable -> throwable.getMessage() != null && throwable.getMessage().contains("empty")); + if (!expectedCause.isPresent()) { + throw e; } - catch (Throwable t) { - t.printStackTrace(); - fail("Caught wrong exception of type " + t.getClass() + "."); - } - - cluster.submitJobAndWait(workingJobGraph, false); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); } + + client.setDetached(false); + client.submitJob(getWorkingJobGraph(), JobSubmissionFailsITCase.class.getClassLoader()); } // -------------------------------------------------------------------------------------------- From 2ac3474c4482b0366a905bce345dc9f90e64ba2f Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 26 Feb 2018 17:19:15 +0100 Subject: [PATCH 0247/2294] [FLINK-8965][tests] Port TimestampITCase to flip6 This closes #5728. --- .../streaming/runtime/TimestampITCase.java | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java index 5e08e8ae3d2cb0..3b46c8259e51e2 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java @@ -24,11 +24,11 @@ import org.apache.flink.api.common.functions.StoppableFunction; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.core.testutils.MultiShotLatch; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; +import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -45,17 +45,18 @@ import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.util.TestStreamEnvironment; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -75,34 +76,24 @@ public class TimestampITCase extends TestLogger { // this is used in some tests to synchronize static MultiShotLatch latch; - private static LocalFlinkMiniCluster cluster; + @ClassRule + public static final MiniClusterResource CLUSTER = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + NUM_TASK_MANAGERS, + NUM_TASK_SLOTS), + true); - @Before - public void setupLatch() { - // ensure that we get a fresh latch for each test - latch = new MultiShotLatch(); - } - - @BeforeClass - public static void startCluster() { + private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS); config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L); - - cluster = new LocalFlinkMiniCluster(config, false); - - cluster.start(); - - TestStreamEnvironment.setAsContext(cluster, PARALLELISM); + return config; } - @AfterClass - public static void shutdownCluster() { - cluster.stop(); - cluster = null; - - TestStreamEnvironment.unsetAsContext(); + @Before + public void setupLatch() { + // ensure that we get a fresh latch for each test + latch = new MultiShotLatch(); } /** @@ -162,7 +153,8 @@ public void testWatermarkPropagation() throws Exception { public void testWatermarkPropagationNoFinalWatermarkOnStop() throws Exception { // for this test to work, we need to be sure that no other jobs are being executed - while (!cluster.getCurrentlyRunningJobsJava().isEmpty()) { + final ClusterClient clusterClient = CLUSTER.getClusterClient(); + while (!getRunningJobs(clusterClient).isEmpty()) { Thread.sleep(100); } @@ -185,14 +177,15 @@ public void testWatermarkPropagationNoFinalWatermarkOnStop() throws Exception { .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true)) .addSink(new DiscardingSink()); - new Thread("stopper") { + Thread t = new Thread("stopper") { @Override public void run() { try { // try until we get the running jobs - List running; - while ((running = cluster.getCurrentlyRunningJobsJava()).isEmpty()) { + List running = getRunningJobs(clusterClient); + while (running.isEmpty()) { Thread.sleep(10); + running = getRunningJobs(clusterClient); } JobID id = running.get(0); @@ -200,7 +193,7 @@ public void run() { // send stop until the job is stopped do { try { - cluster.stopJob(id); + clusterClient.stop(id); } catch (Exception e) { if (e.getCause() instanceof IllegalStateException) { @@ -214,13 +207,14 @@ public void run() { } Thread.sleep(10); } - while (!cluster.getCurrentlyRunningJobsJava().isEmpty()); + while (!getRunningJobs(clusterClient).isEmpty()); } catch (Throwable t) { t.printStackTrace(); } } - }.start(); + }; + t.start(); env.execute(); @@ -246,6 +240,7 @@ public void run() { subtaskWatermarks.get(subtaskWatermarks.size() - 1)); } } + t.join(); } /** @@ -855,4 +850,12 @@ public void run(SourceContext ctx) throws Exception { @Override public void cancel() {} } + + private static List getRunningJobs(ClusterClient client) throws Exception { + Collection statusMessages = client.listJobs().get(); + return statusMessages.stream() + .filter(status -> !status.getJobState().isGloballyTerminalState()) + .map(JobStatusMessage::getJobId) + .collect(Collectors.toList()); + } } From 7b07fa5a09279a55c99d80db92ebf98a7dcd9644 Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Fri, 23 Mar 2018 22:14:46 +0100 Subject: [PATCH 0248/2294] [hotfix] [table] Add Java deprecation annotation to TableEnvironment.sql(). --- .../scala/org/apache/flink/table/api/TableEnvironment.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala index 13e56567ac038f..d6106bec69bd9c 100644 --- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala +++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala @@ -574,9 +574,11 @@ abstract class TableEnvironment(val config: TableConfig) { * tEnv.sql(s"SELECT * FROM $table") * }}} * + * @deprecated Use sqlQuery() instead. * @param query The SQL query to evaluate. * @return The result of the query as Table. */ + @Deprecated @deprecated("Please use sqlQuery() instead.") def sql(query: String): Table = { sqlQuery(query) From 448b9eddf6d2b787f65519f4f937c6ac772f8cbd Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 21 Mar 2018 16:25:37 +0800 Subject: [PATCH 0249/2294] [FLINK-8975] [test] Add Kafka events generator job for StateMachineExample --- .../statemachine/KafkaEventsGeneratorJob.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/KafkaEventsGeneratorJob.java diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/KafkaEventsGeneratorJob.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/KafkaEventsGeneratorJob.java new file mode 100644 index 00000000000000..059b2c0240a880 --- /dev/null +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/KafkaEventsGeneratorJob.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.examples.statemachine; + +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010; +import org.apache.flink.streaming.examples.statemachine.generator.EventsGeneratorSource; +import org.apache.flink.streaming.examples.statemachine.kafka.EventDeSerializer; + +/** + * Job to generate input events that are written to Kafka, for the {@link StateMachineExample} job. + */ +public class KafkaEventsGeneratorJob { + + public static void main(String[] args) throws Exception { + + final ParameterTool params = ParameterTool.fromArgs(args); + + double errorRate = params.getDouble("error-rate", 0.0); + int sleep = params.getInt("sleep", 1); + + String kafkaTopic = params.get("kafka-topic"); + String brokers = params.get("brokers", "localhost:9092"); + + System.out.printf("Generating events to Kafka with standalone source with error rate %f and sleep delay %s millis\n", errorRate, sleep); + System.out.println(); + + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + env + .addSource(new EventsGeneratorSource(errorRate, sleep)) + .addSink(new FlinkKafkaProducer010<>(brokers, kafkaTopic, new EventDeSerializer())); + + // trigger program execution + env.execute("State machine example Kafka events generator job"); + } + +} From a0c17d94072ec6e127ea3cb3c507a595283b9b87 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Wed, 21 Mar 2018 16:32:51 +0800 Subject: [PATCH 0250/2294] [FLINK-8975] [test] Add resume from savepoint end-to-end test This closes #5733. --- .../run-pre-commit-tests.sh | 8 + flink-end-to-end-tests/test-scripts/common.sh | 22 +++ .../test-scripts/test_resume_savepoint.sh | 155 ++++++++++++++++++ .../statemachine/StateMachineExample.java | 2 +- 4 files changed, 186 insertions(+), 1 deletion(-) create mode 100755 flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh index 2c1810b91c8bf8..4fd580cc26dfc9 100755 --- a/flink-end-to-end-tests/run-pre-commit-tests.sh +++ b/flink-end-to-end-tests/run-pre-commit-tests.sh @@ -53,6 +53,14 @@ if [ $EXIT_CODE == 0 ]; then EXIT_CODE=$? fi +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Resuming Savepoint end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh + EXIT_CODE=$? +fi + if [ $EXIT_CODE == 0 ]; then printf "\n==============================================================================\n" printf "Running class loading end-to-end test\n" diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index ef4856f561b542..d4b91266ac8650 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -111,6 +111,28 @@ function stop_cluster { rm $FLINK_DIR/log/* } +function wait_job_running { + for i in {1..10}; do + JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list | grep "$1") + + if [[ "$JOB_LIST_RESULT" == "" ]]; then + echo "Job ($1) is not yet running." + else + echo "Job ($1) is running." + break + fi + sleep 1 + done +} + +function take_savepoint { + "$FLINK_DIR"/bin/flink savepoint $1 $2 +} + +function cancel_job { + "$FLINK_DIR"/bin/flink cancel $1 +} + function check_result_hash { local name=$1 local outfile_prefix=$2 diff --git a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh new file mode 100755 index 00000000000000..7108d9007330a3 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh + +# get Kafka 0.10.0 +mkdir -p $TEST_DATA_DIR +if [ -z "$3" ]; then + # need to download Kafka because no Kafka was specified on the invocation + KAFKA_URL="https://archive.apache.org/dist/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz" + echo "Downloading Kafka from $KAFKA_URL" + curl "$KAFKA_URL" > $TEST_DATA_DIR/kafka.tgz +else + echo "Using specified Kafka from $3" + cp $3 $TEST_DATA_DIR/kafka.tgz +fi + +tar xzf $TEST_DATA_DIR/kafka.tgz -C $TEST_DATA_DIR/ +KAFKA_DIR=$TEST_DATA_DIR/kafka_2.11-0.10.2.0 + +# fix kafka config +sed -i -e "s+^\(dataDir\s*=\s*\).*$+\1$TEST_DATA_DIR/zookeeper+" $KAFKA_DIR/config/zookeeper.properties +sed -i -e "s+^\(log\.dirs\s*=\s*\).*$+\1$TEST_DATA_DIR/kafka+" $KAFKA_DIR/config/server.properties +$KAFKA_DIR/bin/zookeeper-server-start.sh -daemon $KAFKA_DIR/config/zookeeper.properties +$KAFKA_DIR/bin/kafka-server-start.sh -daemon $KAFKA_DIR/config/server.properties + +# modify configuration to have 2 slots +cp $FLINK_DIR/conf/flink-conf.yaml $FLINK_DIR/conf/flink-conf.yaml.bak +sed -i -e 's/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: 2/' $FLINK_DIR/conf/flink-conf.yaml + +# modify configuration to use SLF4J reporter; we will be using this to monitor the state machine progress +cp $FLINK_DIR/opt/flink-metrics-slf4j-*.jar $FLINK_DIR/lib/ +echo "metrics.reporter.slf4j.class: org.apache.flink.metrics.slf4j.Slf4jReporter" >> $FLINK_DIR/conf/flink-conf.yaml +echo "metrics.reporter.slf4j.interval: 1 SECONDS" >> $FLINK_DIR/conf/flink-conf.yaml + +start_cluster + +# make sure to stop Kafka and ZooKeeper at the end, as well as cleaning up the Flink cluster and our moodifications +function test_cleanup { + $KAFKA_DIR/bin/kafka-server-stop.sh + $KAFKA_DIR/bin/zookeeper-server-stop.sh + + # revert our modifications to the Flink distribution + rm $FLINK_DIR/conf/flink-conf.yaml + mv $FLINK_DIR/conf/flink-conf.yaml.bak $FLINK_DIR/conf/flink-conf.yaml + rm $FLINK_DIR/lib/flink-metrics-slf4j-*.jar + + # make sure to run regular cleanup as well + cleanup +} +trap test_cleanup INT +trap test_cleanup EXIT + +# zookeeper outputs the "Node does not exist" bit to stderr +while [[ $($KAFKA_DIR/bin/zookeeper-shell.sh localhost:2181 get /brokers/ids/0 2>&1) =~ .*Node\ does\ not\ exist.* ]]; do + echo "Waiting for broker..." + sleep 1 +done + +# create the required topic +$KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-input + +# run the state machine example job +STATE_MACHINE_JOB=$($FLINK_DIR/bin/flink run -d $FLINK_DIR/examples/streaming/StateMachineExample.jar \ + --kafka-topic test-input \ + | grep "Job has been submitted with JobID" | sed 's/.* //g') + +wait_job_running $STATE_MACHINE_JOB + +# then, run the events generator +EVENTS_GEN_JOB=$($FLINK_DIR/bin/flink run -d -c org.apache.flink.streaming.examples.statemachine.KafkaEventsGeneratorJob $FLINK_DIR/examples/streaming/StateMachineExample.jar \ + --kafka-topic test-input --sleep 15 \ + | grep "Job has been submitted with JobID" | sed 's/.* //g') + +wait_job_running $EVENTS_GEN_JOB + +function get_metric_state_machine_processed_records { + grep ".State machine job.Flat Map -> Sink: Print to Std. Out.0.numRecordsIn:" $FLINK_DIR/log/*taskexecutor*.log | sed 's/.* //g' | tail -1 +} + +function get_num_metric_samples { + grep ".State machine job.Flat Map -> Sink: Print to Std. Out.0.numRecordsIn:" $FLINK_DIR/log/*taskexecutor*.log | wc -l +} + +# monitor the numRecordsIn metric of the state machine operator; +# only proceed to savepoint when the operator has processed 200 records +while : ; do + NUM_RECORDS=$(get_metric_state_machine_processed_records) + + if [ -z $NUM_RECORDS ]; then + NUM_RECORDS=0 + fi + + if (( $NUM_RECORDS < 200 )); then + echo "Waiting for state machine job to process up to 200 records, current progress: $NUM_RECORDS records ..." + sleep 1 + else + break + fi +done + +# take a savepoint of the state machine job +SAVEPOINT_PATH=$(take_savepoint $STATE_MACHINE_JOB $TEST_DATA_DIR \ + | grep "Savepoint completed. Path:" | sed 's/.* //g') + +cancel_job $STATE_MACHINE_JOB + +# Since it is not possible to differentiate reporter output between the first and second execution, +# we remember the number of metrics sampled in the first execution so that they can be ignored in the following monitorings +OLD_NUM_METRICS=$(get_num_metric_samples) + +# resume state machine job with savepoint +STATE_MACHINE_JOB=$($FLINK_DIR/bin/flink run -s $SAVEPOINT_PATH -d $FLINK_DIR/examples/streaming/StateMachineExample.jar \ + --kafka-topic test-input \ + | grep "Job has been submitted with JobID" | sed 's/.* //g') + +wait_job_running $STATE_MACHINE_JOB + +# monitor the numRecordsIn metric of the state machine operator in the second execution +# we let the test finish once the second restore execution has processed 200 records +while : ; do + NUM_METRICS=$(get_num_metric_samples) + NUM_RECORDS=$(get_metric_state_machine_processed_records) + + # only account for metrics that appeared in the second execution + if (( $OLD_NUM_METRICS >= $NUM_METRICS )) ; then + NUM_RECORDS=0 + fi + + if (( $NUM_RECORDS < 200 )); then + echo "Waiting for state machine job to process up to 200 records, current progress: $NUM_RECORDS records ..." + sleep 1 + else + break + fi +done + +# if state is errorneous and the state machine job produces alerting state transitions, +# output would be non-empty and the test will not pass diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/StateMachineExample.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/StateMachineExample.java index 052e954690d64d..14757fb325e6fb 100644 --- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/StateMachineExample.java +++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/statemachine/StateMachineExample.java @@ -108,7 +108,7 @@ public static void main(String[] args) throws Exception { alerts.print(); // trigger program execution - env.execute(); + env.execute("State machine job"); } // ------------------------------------------------------------------------ From e231c5cd9b6923363255ca61837505a9618767e5 Mon Sep 17 00:00:00 2001 From: "Tzu-Li (Gordon) Tai" Date: Thu, 22 Mar 2018 17:09:53 +0800 Subject: [PATCH 0251/2294] [FLINK-8976] [test] Add end-to-end tests for resuming savepoint with differrent parallelism This closes #5745. --- .../run-pre-commit-tests.sh | 20 +++++++++++++++-- .../test-scripts/test_resume_savepoint.sh | 22 +++++++++++++++---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh index 4fd580cc26dfc9..1de66cdce35a34 100755 --- a/flink-end-to-end-tests/run-pre-commit-tests.sh +++ b/flink-end-to-end-tests/run-pre-commit-tests.sh @@ -55,9 +55,25 @@ fi if [ $EXIT_CODE == 0 ]; then printf "\n==============================================================================\n" - printf "Running Resuming Savepoint end-to-end test\n" + printf "Running Resuming Savepoint (no parallelism change) end-to-end test\n" printf "==============================================================================\n" - $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh + $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Resuming Savepoint (scale up) end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 + EXIT_CODE=$? +fi + +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Resuming Savepoint (scale down) end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 EXIT_CODE=$? fi diff --git a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh index 7108d9007330a3..83e0e5a08ce587 100755 --- a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh +++ b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh @@ -17,6 +17,11 @@ # limitations under the License. ################################################################################ +if [ -z $1 ] || [ -z $2 ]; then + echo "Usage: ./test_resume_savepoint.sh " + exit 1 +fi + source "$(dirname "$0")"/common.sh # get Kafka 0.10.0 @@ -40,9 +45,18 @@ sed -i -e "s+^\(log\.dirs\s*=\s*\).*$+\1$TEST_DATA_DIR/kafka+" $KAFKA_DIR/config $KAFKA_DIR/bin/zookeeper-server-start.sh -daemon $KAFKA_DIR/config/zookeeper.properties $KAFKA_DIR/bin/kafka-server-start.sh -daemon $KAFKA_DIR/config/server.properties -# modify configuration to have 2 slots +ORIGINAL_DOP=$1 +NEW_DOP=$2 + +if (( $ORIGINAL_DOP >= $NEW_DOP )); then + NUM_SLOTS=$(( $ORIGINAL_DOP + 1 )) +else + NUM_SLOTS=$(( $NEW_DOP + 1 )) +fi + +# modify configuration to have enough slots cp $FLINK_DIR/conf/flink-conf.yaml $FLINK_DIR/conf/flink-conf.yaml.bak -sed -i -e 's/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: 2/' $FLINK_DIR/conf/flink-conf.yaml +sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: $NUM_SLOTS/" $FLINK_DIR/conf/flink-conf.yaml # modify configuration to use SLF4J reporter; we will be using this to monitor the state machine progress cp $FLINK_DIR/opt/flink-metrics-slf4j-*.jar $FLINK_DIR/lib/ @@ -77,7 +91,7 @@ done $KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-input # run the state machine example job -STATE_MACHINE_JOB=$($FLINK_DIR/bin/flink run -d $FLINK_DIR/examples/streaming/StateMachineExample.jar \ +STATE_MACHINE_JOB=$($FLINK_DIR/bin/flink run -d -p $ORIGINAL_DOP $FLINK_DIR/examples/streaming/StateMachineExample.jar \ --kafka-topic test-input \ | grep "Job has been submitted with JobID" | sed 's/.* //g') @@ -126,7 +140,7 @@ cancel_job $STATE_MACHINE_JOB OLD_NUM_METRICS=$(get_num_metric_samples) # resume state machine job with savepoint -STATE_MACHINE_JOB=$($FLINK_DIR/bin/flink run -s $SAVEPOINT_PATH -d $FLINK_DIR/examples/streaming/StateMachineExample.jar \ +STATE_MACHINE_JOB=$($FLINK_DIR/bin/flink run -s $SAVEPOINT_PATH -p $NEW_DOP -d $FLINK_DIR/examples/streaming/StateMachineExample.jar \ --kafka-topic test-input \ | grep "Job has been submitted with JobID" | sed 's/.* //g') From 9170df55d9338b3c3d8ff52c43ced7feb4076cad Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Wed, 21 Mar 2018 10:54:49 +0100 Subject: [PATCH 0252/2294] [hotfix][runtime] Remove unused method --- .../executiongraph/ArchivedExecutionGraph.java | 5 +---- .../runtime/executiongraph/ExecutionGraph.java | 15 --------------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java index d285b20ea1fedb..d471f86ffd3c16 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java @@ -229,10 +229,7 @@ public boolean isArchived() { return true; } - public StringifiedAccumulatorResult[] getUserAccumulators() { - return archivedUserAccumulators; - } - + @Override public ArchivedExecutionConfig getArchivedExecutionConfig() { return archivedExecutionConfig; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java index ee23884d3a6f77..590fb075fe49ab 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java @@ -763,21 +763,6 @@ public Executor getFutureExecutor() { return userAccumulators; } - /** - * Gets the accumulator results. - */ - public Map getAccumulators() { - - Map> accumulatorMap = aggregateUserAccumulators(); - - Map result = new HashMap<>(); - for (Map.Entry> entry : accumulatorMap.entrySet()) { - result.put(entry.getKey(), entry.getValue().getLocalValue()); - } - - return result; - } - /** * Gets a serialized accumulator map. * @return The accumulator map with serialized accumulator values. From 9fbb461e1e1f71ac7610d9742d2d65c7c264a6de Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Thu, 22 Mar 2018 15:39:59 +0100 Subject: [PATCH 0253/2294] [hotfix][tests] Do not hide original exception in the tests --- .../SerializedJobExecutionResultTest.java | 106 ++++++++---------- 1 file changed, 47 insertions(+), 59 deletions(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java index b3bac5802b9a74..38447e2a201d3e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java @@ -36,68 +36,56 @@ public class SerializedJobExecutionResultTest { @Test - public void testSerialization() { - try { - final ClassLoader classloader = getClass().getClassLoader(); - - JobID origJobId = new JobID(); - long origTime = 65927436589267L; - - Map> origMap = new HashMap>(); - origMap.put("name1", new SerializedValue(723L)); - origMap.put("name2", new SerializedValue("peter")); - - SerializedJobExecutionResult result = new SerializedJobExecutionResult(origJobId, origTime, origMap); - - // serialize and deserialize the object - SerializedJobExecutionResult cloned = CommonTestUtils.createCopySerializable(result); - - assertEquals(origJobId, cloned.getJobId()); - assertEquals(origTime, cloned.getNetRuntime()); - assertEquals(origTime, cloned.getNetRuntime(TimeUnit.MILLISECONDS)); - assertEquals(origMap, cloned.getSerializedAccumulatorResults()); - - // convert to deserialized result - JobExecutionResult jResult = result.toJobExecutionResult(classloader); - JobExecutionResult jResultCopied = result.toJobExecutionResult(classloader); - - assertEquals(origJobId, jResult.getJobID()); - assertEquals(origJobId, jResultCopied.getJobID()); - assertEquals(origTime, jResult.getNetRuntime()); - assertEquals(origTime, jResult.getNetRuntime(TimeUnit.MILLISECONDS)); - assertEquals(origTime, jResultCopied.getNetRuntime()); - assertEquals(origTime, jResultCopied.getNetRuntime(TimeUnit.MILLISECONDS)); - - for (Map.Entry> entry : origMap.entrySet()) { - String name = entry.getKey(); - Object value = entry.getValue().deserializeValue(classloader); - assertEquals(value, jResult.getAccumulatorResult(name)); - assertEquals(value, jResultCopied.getAccumulatorResult(name)); - } - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); + public void testSerialization() throws Exception { + final ClassLoader classloader = getClass().getClassLoader(); + + JobID origJobId = new JobID(); + long origTime = 65927436589267L; + + Map> origMap = new HashMap>(); + origMap.put("name1", new SerializedValue(723L)); + origMap.put("name2", new SerializedValue("peter")); + + SerializedJobExecutionResult result = new SerializedJobExecutionResult(origJobId, origTime, origMap); + + // serialize and deserialize the object + SerializedJobExecutionResult cloned = CommonTestUtils.createCopySerializable(result); + + assertEquals(origJobId, cloned.getJobId()); + assertEquals(origTime, cloned.getNetRuntime()); + assertEquals(origTime, cloned.getNetRuntime(TimeUnit.MILLISECONDS)); + assertEquals(origMap, cloned.getSerializedAccumulatorResults()); + + // convert to deserialized result + JobExecutionResult jResult = result.toJobExecutionResult(classloader); + JobExecutionResult jResultCopied = result.toJobExecutionResult(classloader); + + assertEquals(origJobId, jResult.getJobID()); + assertEquals(origJobId, jResultCopied.getJobID()); + assertEquals(origTime, jResult.getNetRuntime()); + assertEquals(origTime, jResult.getNetRuntime(TimeUnit.MILLISECONDS)); + assertEquals(origTime, jResultCopied.getNetRuntime()); + assertEquals(origTime, jResultCopied.getNetRuntime(TimeUnit.MILLISECONDS)); + + for (Map.Entry> entry : origMap.entrySet()) { + String name = entry.getKey(); + Object value = entry.getValue().deserializeValue(classloader); + assertEquals(value, jResult.getAccumulatorResult(name)); + assertEquals(value, jResultCopied.getAccumulatorResult(name)); } } @Test - public void testSerializationWithNullValues() { - try { - SerializedJobExecutionResult result = new SerializedJobExecutionResult(null, 0L, null); - SerializedJobExecutionResult cloned = CommonTestUtils.createCopySerializable(result); - - assertNull(cloned.getJobId()); - assertEquals(0L, cloned.getNetRuntime()); - assertNull(cloned.getSerializedAccumulatorResults()); - - JobExecutionResult jResult = result.toJobExecutionResult(getClass().getClassLoader()); - assertNull(jResult.getJobID()); - assertTrue(jResult.getAllAccumulatorResults().isEmpty()); - } - catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } + public void testSerializationWithNullValues() throws Exception { + SerializedJobExecutionResult result = new SerializedJobExecutionResult(null, 0L, null); + SerializedJobExecutionResult cloned = CommonTestUtils.createCopySerializable(result); + + assertNull(cloned.getJobId()); + assertEquals(0L, cloned.getNetRuntime()); + assertNull(cloned.getSerializedAccumulatorResults()); + + JobExecutionResult jResult = result.toJobExecutionResult(getClass().getClassLoader()); + assertNull(jResult.getJobID()); + assertTrue(jResult.getAllAccumulatorResults().isEmpty()); } } From 1132cd7dbc541a948975ef4b69e2a85cd79c81a5 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Thu, 22 Mar 2018 17:55:49 +0100 Subject: [PATCH 0254/2294] [hotfix][tests] Allow to run SpillableSubpartitionTests in the loop --- .../io/network/partition/SpillableSubpartitionTest.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java index 840669e7c3fccc..15acd7c70b0b93 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SpillableSubpartitionTest.java @@ -34,6 +34,7 @@ import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -74,7 +75,12 @@ public class SpillableSubpartitionTest extends SubpartitionTestBase { private static final ExecutorService executorService = Executors.newCachedThreadPool(); /** Asynchronous I/O manager. */ - private static final IOManager ioManager = new IOManagerAsync(); + private static IOManager ioManager; + + @BeforeClass + public static void setup() { + ioManager = new IOManagerAsync(); + } @AfterClass public static void shutdown() { From a6a7623dd8a549f4a31bbc361b891c65b5feef07 Mon Sep 17 00:00:00 2001 From: Piotr Nowojski Date: Wed, 21 Mar 2018 13:08:36 +0100 Subject: [PATCH 0255/2294] [FLINK-8721][flip6] Handle archiving failures for accumulators During archivization, wrap errors thrown by users' Accumulators into a OptionalFailure and do not fail the job because of that. This closes #5737. --- .../flink/client/program/ClusterClient.java | 7 +- .../client/program/MiniClusterClient.java | 11 +- .../program/rest/RestClusterClient.java | 5 +- .../program/rest/RestClusterClientTest.java | 11 +- .../flink/api/common/JobExecutionResult.java | 14 +- .../accumulators/AccumulatorHelper.java | 63 ++++++-- .../FailedAccumulatorSerialization.java | 73 --------- .../common/operators/CollectionExecutor.java | 3 +- .../apache/flink/util/OptionalFailure.java | 135 ++++++++++++++++ .../flink/util/function/CheckedSupplier.java | 12 ++ .../FailedAccumulatorSerializationTest.java | 89 ----------- .../accumulators/AccumulatorSnapshot.java | 2 +- .../StringifiedAccumulatorResult.java | 54 +++++-- .../client/SerializedJobExecutionResult.java | 13 +- .../executiongraph/AccessExecutionGraph.java | 3 +- .../ArchivedExecutionGraph.java | 10 +- .../runtime/executiongraph/Execution.java | 10 +- .../executiongraph/ExecutionGraph.java | 46 +++--- .../executiongraph/ExecutionJobVertex.java | 3 +- .../flink/runtime/jobmaster/JobResult.java | 11 +- .../handler/job/JobAccumulatorsHandler.java | 3 +- .../rest/messages/JobAccumulatorsInfo.java | 7 +- .../messages/json/JobResultDeserializer.java | 9 +- .../messages/json/JobResultSerializer.java | 7 +- .../accumulators/AccumulatorMessages.scala | 7 +- .../StringifiedAccumulatorResultTest.java | 30 +++- .../SerializedJobExecutionResultTest.java | 46 ++++-- .../ArchivedExecutionGraphTest.java | 12 +- ...ecutionAttemptAccumulatorsHandlerTest.java | 9 +- .../utils/ArchivedExecutionGraphBuilder.java | 7 +- .../JobExecutionResultResponseBodyTest.java | 3 +- .../TestingJobManagerMessages.scala | 3 +- .../accumulators/AccumulatorErrorITCase.java | 149 +++++++++++------- .../test/accumulators/AccumulatorITCase.java | 1 + .../accumulators/AccumulatorLiveITCase.java | 2 +- .../LegacyAccumulatorLiveITCase.java | 7 +- .../utils/SavepointMigrationTestBase.java | 11 +- 37 files changed, 524 insertions(+), 364 deletions(-) delete mode 100644 flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java create mode 100644 flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java delete mode 100644 flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index 166d9770b48bb1..79a53831095819 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -65,6 +65,7 @@ import org.apache.flink.runtime.util.LeaderRetrievalUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedValue; @@ -792,7 +793,7 @@ public CompletableFuture> listJobs() throws Excepti * @param jobID The job identifier of a job. * @return A Map containing the accumulator's name and its value. */ - public Map getAccumulators(JobID jobID) throws Exception { + public Map> getAccumulators(JobID jobID) throws Exception { return getAccumulators(jobID, ClassLoader.getSystemClassLoader()); } @@ -803,7 +804,7 @@ public Map getAccumulators(JobID jobID) throws Exception { * @param loader The class loader for deserializing the accumulator results. * @return A Map containing the accumulator's name and its value. */ - public Map getAccumulators(JobID jobID, ClassLoader loader) throws Exception { + public Map> getAccumulators(JobID jobID, ClassLoader loader) throws Exception { ActorGateway jobManagerGateway = getJobManagerGateway(); Future response; @@ -816,7 +817,7 @@ public Map getAccumulators(JobID jobID, ClassLoader loader) thro Object result = Await.result(response, timeout); if (result instanceof AccumulatorResultsFound) { - Map> serializedAccumulators = + Map>> serializedAccumulators = ((AccumulatorResultsFound) result).result(); return AccumulatorHelper.deserializeAccumulators(serializedAccumulators, loader); diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 276df62f7233d8..44f6ef630d26a7 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -41,6 +41,7 @@ import org.apache.flink.runtime.util.LeaderRetrievalUtils; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedValue; import javax.annotation.Nonnull; @@ -132,16 +133,16 @@ public CompletableFuture> listJobs() throws Excepti } @Override - public Map getAccumulators(JobID jobID) throws Exception { + public Map> getAccumulators(JobID jobID) throws Exception { return getAccumulators(jobID, ClassLoader.getSystemClassLoader()); } @Override - public Map getAccumulators(JobID jobID, ClassLoader loader) throws Exception { + public Map> getAccumulators(JobID jobID, ClassLoader loader) throws Exception { AccessExecutionGraph executionGraph = guardWithSingleRetry(() -> miniCluster.getExecutionGraph(jobID), scheduledExecutor).get(); - Map> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); - Map result = new HashMap<>(accumulatorsSerialized.size()); - for (Map.Entry> acc : accumulatorsSerialized.entrySet()) { + Map>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); + Map> result = new HashMap<>(accumulatorsSerialized.size()); + for (Map.Entry>> acc : accumulatorsSerialized.entrySet()) { result.put(acc.getKey(), acc.getValue().deserializeValue(loader)); } return result; diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index f3f196182011e0..2e1ffb02298183 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -89,6 +89,7 @@ import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.ExecutorUtils; import org.apache.flink.util.FlinkException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.CheckedSupplier; @@ -409,7 +410,7 @@ private CompletableFuture triggerSavepoint( } @Override - public Map getAccumulators(final JobID jobID, ClassLoader loader) throws Exception { + public Map> getAccumulators(final JobID jobID, ClassLoader loader) throws Exception { final JobAccumulatorsHeaders accumulatorsHeaders = JobAccumulatorsHeaders.getInstance(); final JobAccumulatorsMessageParameters accMsgParams = accumulatorsHeaders.getUnresolvedMessageParameters(); accMsgParams.jobPathParameter.resolve(jobID); @@ -420,7 +421,7 @@ public Map getAccumulators(final JobID jobID, ClassLoader loader accMsgParams ); - Map result = Collections.emptyMap(); + Map> result = Collections.emptyMap(); try { result = responseFuture.thenApply((JobAccumulatorsInfo accumulatorsInfo) -> { diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index e98ba436abf298..77a4113f59ed43 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -84,6 +84,7 @@ import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.testutils.category.Flip6; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; import org.apache.flink.util.TestLogger; @@ -362,7 +363,7 @@ public void testSubmitJobAndWaitForExecutionResult() throws Exception { JobExecutionResultResponseBody.created(new JobResult.Builder() .jobId(jobId) .netRuntime(Long.MAX_VALUE) - .accumulatorResults(Collections.singletonMap("testName", new SerializedValue<>(1.0))) + .accumulatorResults(Collections.singletonMap("testName", new SerializedValue<>(OptionalFailure.of(1.0)))) .build()), JobExecutionResultResponseBody.created(new JobResult.Builder() .jobId(jobId) @@ -558,12 +559,12 @@ public void testGetAccumulators() throws Exception { JobID id = new JobID(); { - Map accumulators = restClusterClient.getAccumulators(id); + Map> accumulators = restClusterClient.getAccumulators(id); assertNotNull(accumulators); assertEquals(1, accumulators.size()); assertEquals(true, accumulators.containsKey("testKey")); - assertEquals("testValue", accumulators.get("testKey").toString()); + assertEquals("testValue", accumulators.get("testKey").get().toString()); } } } @@ -594,9 +595,9 @@ protected CompletableFuture handleRequest( userTaskAccumulators.add(new JobAccumulatorsInfo.UserTaskAccumulator("testName", "testType", "testValue")); if (includeSerializedValue) { - Map> serializedUserTaskAccumulators = new HashMap<>(1); + Map>> serializedUserTaskAccumulators = new HashMap<>(1); try { - serializedUserTaskAccumulators.put("testKey", new SerializedValue<>("testValue")); + serializedUserTaskAccumulators.put("testKey", new SerializedValue<>(OptionalFailure.of("testValue"))); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java index a200d123713c2e..9e1a3a552b6641 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java @@ -20,10 +20,12 @@ import org.apache.flink.annotation.Public; import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.util.OptionalFailure; import java.util.Collections; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * The result of a job execution. Gives access to the execution time of the job, @@ -34,7 +36,7 @@ public class JobExecutionResult extends JobSubmissionResult { private final long netRuntime; - private final Map accumulatorResults; + private final Map> accumulatorResults; /** * Creates a new JobExecutionResult. @@ -43,7 +45,7 @@ public class JobExecutionResult extends JobSubmissionResult { * @param netRuntime The net runtime of the job (excluding pre-flight phase like the optimizer) in milliseconds * @param accumulators A map of all accumulators produced by the job. */ - public JobExecutionResult(JobID jobID, long netRuntime, Map accumulators) { + public JobExecutionResult(JobID jobID, long netRuntime, Map> accumulators) { super(jobID); this.netRuntime = netRuntime; @@ -85,7 +87,7 @@ public long getNetRuntime(TimeUnit desiredUnit) { */ @SuppressWarnings("unchecked") public T getAccumulatorResult(String accumulatorName) { - return (T) this.accumulatorResults.get(accumulatorName); + return (T) this.accumulatorResults.get(accumulatorName).getUnchecked(); } /** @@ -95,7 +97,9 @@ public T getAccumulatorResult(String accumulatorName) { * @return A map containing all accumulators produced by the job. */ public Map getAllAccumulatorResults() { - return this.accumulatorResults; + return accumulatorResults.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getUnchecked())); } /** @@ -109,7 +113,7 @@ public Map getAllAccumulatorResults() { @Deprecated @PublicEvolving public Integer getIntCounterResult(String accumulatorName) { - Object result = this.accumulatorResults.get(accumulatorName); + Object result = this.accumulatorResults.get(accumulatorName).getUnchecked(); if (result == null) { return null; } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java index 78fb68bf6fa2e1..9bc129929ca929 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java @@ -19,20 +19,27 @@ package org.apache.flink.api.common.accumulators; import org.apache.flink.annotation.Internal; +import org.apache.flink.util.FlinkException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.io.Serializable; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; /** * Helper functions for the interaction with {@link Accumulator}. */ @Internal public class AccumulatorHelper { + private static final Logger LOG = LoggerFactory.getLogger(AccumulatorHelper.class); /** * Merge two collections of accumulators. The second will be merged into the @@ -44,19 +51,28 @@ public class AccumulatorHelper { * The collection of accumulators that will be merged into the * other */ - public static void mergeInto(Map> target, Map> toMerge) { + public static void mergeInto(Map>> target, Map> toMerge) { for (Map.Entry> otherEntry : toMerge.entrySet()) { - Accumulator ownAccumulator = target.get(otherEntry.getKey()); + OptionalFailure> ownAccumulator = target.get(otherEntry.getKey()); if (ownAccumulator == null) { // Create initial counter (copy!) - target.put(otherEntry.getKey(), otherEntry.getValue().clone()); + target.put( + otherEntry.getKey(), + wrapUnchecked(otherEntry.getKey(), () -> otherEntry.getValue().clone())); + } + else if (ownAccumulator.isFailure()) { + continue; } else { + Accumulator accumulator = ownAccumulator.getUnchecked(); // Both should have the same type - AccumulatorHelper.compareAccumulatorTypes(otherEntry.getKey(), - ownAccumulator.getClass(), otherEntry.getValue().getClass()); + compareAccumulatorTypes(otherEntry.getKey(), + accumulator.getClass(), otherEntry.getValue().getClass()); // Merge target counter with other counter - mergeSingle(ownAccumulator, otherEntry.getValue()); + + target.put( + otherEntry.getKey(), + wrapUnchecked(otherEntry.getKey(), () -> mergeSingle(accumulator, otherEntry.getValue().clone()))); } } } @@ -64,8 +80,8 @@ public static void mergeInto(Map> target, Map void mergeSingle(Accumulator target, - Accumulator toMerge) { + private static Accumulator mergeSingle(Accumulator target, + Accumulator toMerge) { @SuppressWarnings("unchecked") Accumulator typedTarget = (Accumulator) target; @@ -73,6 +89,8 @@ private static void mergeSingle(Accumulator ta Accumulator typedToMerge = (Accumulator) toMerge; typedTarget.merge(typedToMerge); + + return typedTarget; } /** @@ -106,14 +124,25 @@ public static void compareAccumulatorTypes( * Transform the Map with accumulators into a Map containing only the * results. */ - public static Map toResultMap(Map> accumulators) { - Map resultMap = new HashMap(); + public static Map> toResultMap(Map> accumulators) { + Map> resultMap = new HashMap<>(); for (Map.Entry> entry : accumulators.entrySet()) { - resultMap.put(entry.getKey(), entry.getValue().getLocalValue()); + resultMap.put(entry.getKey(), wrapUnchecked(entry.getKey(), () -> entry.getValue().getLocalValue())); } return resultMap; } + private static OptionalFailure wrapUnchecked(String name, Supplier supplier) { + return OptionalFailure.createFrom(() -> { + try { + return supplier.get(); + } catch (RuntimeException ex) { + LOG.error("Unexpected error while handling accumulator [" + name + "]", ex); + throw new FlinkException(ex); + } + }); + } + public static String getResultsFormatted(Map map) { StringBuilder builder = new StringBuilder(); for (Map.Entry entry : map.entrySet()) { @@ -152,19 +181,19 @@ public static String getResultsFormatted(Map map) { * @throws IOException * @throws ClassNotFoundException */ - public static Map deserializeAccumulators( - Map> serializedAccumulators, ClassLoader loader) - throws IOException, ClassNotFoundException { + public static Map> deserializeAccumulators( + Map>> serializedAccumulators, + ClassLoader loader) throws IOException, ClassNotFoundException { if (serializedAccumulators == null || serializedAccumulators.isEmpty()) { return Collections.emptyMap(); } - Map accumulators = new HashMap<>(serializedAccumulators.size()); + Map> accumulators = new HashMap<>(serializedAccumulators.size()); - for (Map.Entry> entry : serializedAccumulators.entrySet()) { + for (Map.Entry>> entry : serializedAccumulators.entrySet()) { - Object value = null; + OptionalFailure value = null; if (entry.getValue() != null) { value = entry.getValue().deserializeValue(loader); } diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java deleted file mode 100644 index b208b9ee268521..00000000000000 --- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.api.common.accumulators; - -import org.apache.flink.util.ExceptionUtils; -import org.apache.flink.util.Preconditions; - -import java.io.Serializable; - -/** - * {@link Accumulator} implementation which indicates a serialization problem with the original - * accumulator. Accessing any of the {@link Accumulator} method will result in throwing the - * serialization exception. - * - * @param type of the value - * @param type of the accumulator result - */ -public class FailedAccumulatorSerialization implements Accumulator { - private static final long serialVersionUID = 6965908827065879760L; - - private final Throwable throwable; - - public FailedAccumulatorSerialization(Throwable throwable) { - this.throwable = Preconditions.checkNotNull(throwable); - } - - public Throwable getThrowable() { - return throwable; - } - - @Override - public void add(V value) { - ExceptionUtils.rethrow(throwable); - } - - @Override - public R getLocalValue() { - ExceptionUtils.rethrow(throwable); - return null; - } - - @Override - public void resetLocal() { - ExceptionUtils.rethrow(throwable); - } - - @Override - public void merge(Accumulator other) { - ExceptionUtils.rethrow(throwable); - } - - @Override - public Accumulator clone() { - ExceptionUtils.rethrow(throwable); - return null; - } -} diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java index 07f48fc1675fe0..55f3df7d31c0e3 100644 --- a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java +++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java @@ -50,6 +50,7 @@ import org.apache.flink.metrics.MetricGroup; import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; import org.apache.flink.types.Value; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Visitor; import java.util.ArrayList; @@ -115,7 +116,7 @@ public JobExecutionResult execute(Plan program) throws Exception { } long endTime = System.currentTimeMillis(); - Map accumulatorResults = AccumulatorHelper.toResultMap(accumulators); + Map> accumulatorResults = AccumulatorHelper.toResultMap(accumulators); return new JobExecutionResult(null, endTime - startTime, accumulatorResults); } diff --git a/flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java b/flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java new file mode 100644 index 00000000000000..ace3cad93e4024 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.util; + +import org.apache.flink.util.function.CheckedSupplier; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.Objects; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Wrapper around an object representing either a success (with a given value) or a failure cause. + */ +public class OptionalFailure implements Serializable { + private static final long serialVersionUID = 1L; + + @Nullable + private transient T value; + + @Nullable + private Throwable failureCause; + + private OptionalFailure(@Nullable T value, @Nullable Throwable failureCause) { + this.value = value; + this.failureCause = failureCause; + } + + public static OptionalFailure of(T value) { + return new OptionalFailure<>(value, null); + } + + public static OptionalFailure ofFailure(Throwable failureCause) { + return new OptionalFailure<>(null, failureCause); + } + + /** + * @return wrapped {@link OptionalFailure} returned by {@code valueSupplier} or wrapped failure if + * {@code valueSupplier} has thrown an {@link Exception}. + */ + public static OptionalFailure createFrom(CheckedSupplier valueSupplier) { + try { + return of(valueSupplier.get()); + } catch (Exception ex) { + return ofFailure(ex); + } + } + + /** + * @return stored value or throw a {@link FlinkException} with {@code failureCause}. + */ + public T get() throws FlinkException { + if (value != null) { + return value; + } + checkNotNull(failureCause); + throw new FlinkException(failureCause); + } + + /** + * @return same as {@link #get()} but throws a {@link FlinkRuntimeException}. + */ + public T getUnchecked() throws FlinkRuntimeException { + if (value != null) { + return value; + } + checkNotNull(failureCause); + throw new FlinkRuntimeException(failureCause); + } + + public Throwable getFailureCause() { + return checkNotNull(failureCause); + } + + public boolean isFailure() { + return failureCause != null; + } + + @Override + public int hashCode() { + return Objects.hash(value, failureCause); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj == this) { + return true; + } + if (!(obj instanceof OptionalFailure)) { + return false; + } + OptionalFailure other = (OptionalFailure) obj; + return Objects.equals(value, other.value) && + Objects.equals(failureCause, other.failureCause); + } + + private void writeObject(ObjectOutputStream stream) throws IOException { + stream.defaultWriteObject(); + stream.writeObject(value); + } + + private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { + stream.defaultReadObject(); + value = (T) stream.readObject(); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{value=" + value + ", failureCause=" + failureCause + "}"; + } +} diff --git a/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java b/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java index a0bcc1311856bc..585d705f59676b 100644 --- a/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java +++ b/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java @@ -18,6 +18,8 @@ package org.apache.flink.util.function; +import org.apache.flink.util.FlinkException; + import java.util.function.Supplier; /** @@ -36,4 +38,14 @@ static Supplier unchecked(CheckedSupplier checkedSupplier) { }; } + static CheckedSupplier checked(Supplier supplier) { + return () -> { + try { + return supplier.get(); + } + catch (RuntimeException e) { + throw new FlinkException(e); + } + }; + } } diff --git a/flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java b/flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java deleted file mode 100644 index 7335d30dfc8c2e..00000000000000 --- a/flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.api.common.accumulators; - -import org.apache.flink.util.ExceptionUtils; -import org.apache.flink.util.InstantiationUtil; -import org.apache.flink.util.TestLogger; - -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -/** - * Tests for the {@link FailedAccumulatorSerialization}. - */ -public class FailedAccumulatorSerializationTest extends TestLogger { - - private static final IOException TEST_EXCEPTION = new IOException("Test exception"); - - /** - * Tests that any method call will throw the contained throwable (wrapped in an - * unchecked exception if it is checked). - */ - @Test - public void testMethodCallThrowsException() { - final FailedAccumulatorSerialization accumulator = new FailedAccumulatorSerialization<>(TEST_EXCEPTION); - - try { - accumulator.getLocalValue(); - } catch (RuntimeException re) { - assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true)); - } - - try { - accumulator.resetLocal(); - } catch (RuntimeException re) { - assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true)); - } - - try { - accumulator.add(1); - } catch (RuntimeException re) { - assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true)); - } - - try { - accumulator.merge(new IntMinimum()); - } catch (RuntimeException re) { - assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true)); - } - } - - /** - * Tests that the class can be serialized and deserialized using Java serialization. - */ - @Test - public void testSerialization() throws Exception { - final FailedAccumulatorSerialization accumulator = new FailedAccumulatorSerialization<>(TEST_EXCEPTION); - - final byte[] serializedAccumulator = InstantiationUtil.serializeObject(accumulator); - - final FailedAccumulatorSerialization deserializedAccumulator = InstantiationUtil.deserializeObject(serializedAccumulator, ClassLoader.getSystemClassLoader()); - - assertThat(deserializedAccumulator.getThrowable(), is(instanceOf(TEST_EXCEPTION.getClass()))); - assertThat(deserializedAccumulator.getThrowable().getMessage(), is(equalTo(TEST_EXCEPTION.getMessage()))); - } - -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java index 0bfb1acf274377..d01cd322fd42a5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java @@ -48,7 +48,7 @@ public AccumulatorSnapshot(JobID jobID, ExecutionAttemptID executionAttemptID, Map> userAccumulators) throws IOException { this.jobID = jobID; this.executionAttemptID = executionAttemptID; - this.userAccumulators = new SerializedValue>>(userAccumulators); + this.userAccumulators = new SerializedValue<>(userAccumulators); } public JobID getJobID() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java index b55159c00c55b9..f283bcdcd72e46 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java @@ -19,6 +19,13 @@ package org.apache.flink.runtime.accumulators; import org.apache.flink.api.common.accumulators.Accumulator; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.OptionalFailure; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; import java.util.Map; @@ -26,6 +33,7 @@ * Container class that transports the result of an accumulator as set of strings. */ public class StringifiedAccumulatorResult implements java.io.Serializable{ + private static final Logger LOG = LoggerFactory.getLogger(StringifiedAccumulatorResult.class); private static final long serialVersionUID = -4642311296836822611L; @@ -58,7 +66,7 @@ public String getValue() { /** * Flatten a map of accumulator names to Accumulator instances into an array of StringifiedAccumulatorResult values. */ - public static StringifiedAccumulatorResult[] stringifyAccumulatorResults(Map> accs) { + public static StringifiedAccumulatorResult[] stringifyAccumulatorResults(Map>> accs) { if (accs == null || accs.isEmpty()) { return new StringifiedAccumulatorResult[0]; } @@ -66,23 +74,37 @@ public static StringifiedAccumulatorResult[] stringifyAccumulatorResults(Map> entry : accs.entrySet()) { - StringifiedAccumulatorResult result; - Accumulator accumulator = entry.getValue(); - if (accumulator != null) { - Object localValue = accumulator.getLocalValue(); - if (localValue != null) { - result = new StringifiedAccumulatorResult(entry.getKey(), accumulator.getClass().getSimpleName(), localValue.toString()); - } else { - result = new StringifiedAccumulatorResult(entry.getKey(), accumulator.getClass().getSimpleName(), "null"); - } - } else { - result = new StringifiedAccumulatorResult(entry.getKey(), "null", "null"); - } - - results[i++] = result; + for (Map.Entry>> entry : accs.entrySet()) { + results[i++] = stringifyAccumulatorResult(entry.getKey(), entry.getValue()); } return results; } } + + private static StringifiedAccumulatorResult stringifyAccumulatorResult( + String name, + @Nullable OptionalFailure> accumulator) { + if (accumulator == null) { + return new StringifiedAccumulatorResult(name, "null", "null"); + } + else if (accumulator.isFailure()) { + return new StringifiedAccumulatorResult( + name, + "null", + ExceptionUtils.stringifyException(accumulator.getFailureCause())); + } + else { + Object localValue; + String simpleName = "null"; + try { + simpleName = accumulator.getUnchecked().getClass().getSimpleName(); + localValue = accumulator.getUnchecked().getLocalValue(); + } + catch (RuntimeException exception) { + LOG.error("Failed to stringify accumulator [" + name + "]", exception); + localValue = ExceptionUtils.stringifyException(exception); + } + return new StringifiedAccumulatorResult(name, simpleName, localValue != null ? localValue.toString() : "null"); + } + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java index ec2312f36b8780..0ca7a4145da544 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.accumulators.AccumulatorHelper; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedValue; import java.io.IOException; @@ -37,7 +38,7 @@ public class SerializedJobExecutionResult implements java.io.Serializable { private final JobID jobId; - private final Map> accumulatorResults; + private final Map>> accumulatorResults; private final long netRuntime; @@ -48,8 +49,10 @@ public class SerializedJobExecutionResult implements java.io.Serializable { * @param netRuntime The net runtime of the job (excluding pre-flight phase like the optimizer) in milliseconds * @param accumulators A map of all accumulator results produced by the job, in serialized form */ - public SerializedJobExecutionResult(JobID jobID, long netRuntime, - Map> accumulators) { + public SerializedJobExecutionResult( + JobID jobID, + long netRuntime, + Map>> accumulators) { this.jobId = jobID; this.netRuntime = netRuntime; this.accumulatorResults = accumulators; @@ -74,12 +77,12 @@ public long getNetRuntime(TimeUnit desiredUnit) { return desiredUnit.convert(getNetRuntime(), TimeUnit.MILLISECONDS); } - public Map> getSerializedAccumulatorResults() { + public Map>> getSerializedAccumulatorResults() { return this.accumulatorResults; } public JobExecutionResult toJobExecutionResult(ClassLoader loader) throws IOException, ClassNotFoundException { - Map accumulators = + Map> accumulators = AccumulatorHelper.deserializeAccumulators(accumulatorResults, loader); return new JobExecutionResult(jobId, netRuntime, accumulators); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java index cc56209e3de09d..de578bdb9053bd 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java @@ -25,6 +25,7 @@ import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedValue; import javax.annotation.Nullable; @@ -155,7 +156,7 @@ public interface AccessExecutionGraph { * * @return map containing serialized values of user-defined accumulators */ - Map> getAccumulatorsSerialized(); + Map>> getAccumulatorsSerialized(); /** * Returns whether this execution graph was archived. diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java index d471f86ffd3c16..24897215335fae 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java @@ -25,6 +25,7 @@ import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedValue; @@ -86,7 +87,7 @@ public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializabl private final StringifiedAccumulatorResult[] archivedUserAccumulators; private final ArchivedExecutionConfig archivedExecutionConfig; private final boolean isStoppable; - private final Map> serializedUserAccumulators; + private final Map>> serializedUserAccumulators; @Nullable private final CheckpointCoordinatorConfiguration jobCheckpointingConfiguration; @@ -104,7 +105,7 @@ public ArchivedExecutionGraph( @Nullable ErrorInfo failureCause, String jsonPlan, StringifiedAccumulatorResult[] archivedUserAccumulators, - Map> serializedUserAccumulators, + Map>> serializedUserAccumulators, ArchivedExecutionConfig executionConfig, boolean isStoppable, @Nullable CheckpointCoordinatorConfiguration jobCheckpointingConfiguration, @@ -245,7 +246,7 @@ public StringifiedAccumulatorResult[] getAccumulatorResultsStringified() { } @Override - public Map> getAccumulatorsSerialized() { + public Map>> getAccumulatorsSerialized() { return serializedUserAccumulators; } @@ -312,7 +313,8 @@ public static ArchivedExecutionGraph createFrom(ExecutionGraph executionGraph) { archivedTasks.put(task.getJobVertexId(), archivedTask); } - final Map> serializedUserAccumulators = executionGraph.getAccumulatorsSerialized(); + final Map>> serializedUserAccumulators = + executionGraph.getAccumulatorsSerialized(); final long[] timestamps = new long[JobStatus.values().length]; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java index 2fb831a2c62b8d..1af4fc99f9ec74 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java @@ -53,6 +53,7 @@ import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.OptionalFailure; import org.slf4j.Logger; @@ -69,6 +70,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.stream.Collectors; import static org.apache.flink.runtime.execution.ExecutionState.CANCELED; import static org.apache.flink.runtime.execution.ExecutionState.CANCELING; @@ -1366,7 +1368,13 @@ public void setAccumulators(Map> userAccumulators) { @Override public StringifiedAccumulatorResult[] getUserAccumulatorsStringified() { - return StringifiedAccumulatorResult.stringifyAccumulatorResults(userAccumulators); + Map>> accumulators = + userAccumulators == null ? + null : + userAccumulators.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> OptionalFailure.of(entry.getValue()))); + return StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulators); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java index 590fb075fe49ab..22c11efae873bd 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java @@ -24,7 +24,6 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.accumulators.Accumulator; import org.apache.flink.api.common.accumulators.AccumulatorHelper; -import org.apache.flink.api.common.accumulators.FailedAccumulatorSerialization; import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.JobException; @@ -69,6 +68,7 @@ import org.apache.flink.types.Either; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; @@ -103,6 +103,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.stream.Collectors; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -749,9 +750,9 @@ public Executor getFutureExecutor() { * Merges all accumulator results from the tasks previously executed in the Executions. * @return The accumulator map */ - public Map> aggregateUserAccumulators() { + public Map>> aggregateUserAccumulators() { - Map> userAccumulators = new HashMap<>(); + Map>> userAccumulators = new HashMap<>(); for (ExecutionVertex vertex : getAllExecutionVertices()) { Map> next = vertex.getCurrentExecutionAttempt().getUserAccumulators(); @@ -768,28 +769,29 @@ public Executor getFutureExecutor() { * @return The accumulator map with serialized accumulator values. */ @Override - public Map> getAccumulatorsSerialized() { - - Map> accumulatorMap = aggregateUserAccumulators(); - - Map> result = new HashMap<>(accumulatorMap.size()); - for (Map.Entry> entry : accumulatorMap.entrySet()) { + public Map>> getAccumulatorsSerialized() { + return aggregateUserAccumulators() + .entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> serializeAccumulator(entry.getKey(), entry.getValue()))); + } + private static SerializedValue> serializeAccumulator(String name, OptionalFailure> accumulator) { + try { + if (accumulator.isFailure()) { + return new SerializedValue<>(OptionalFailure.ofFailure(accumulator.getFailureCause())); + } + return new SerializedValue<>(OptionalFailure.of(accumulator.getUnchecked().getLocalValue())); + } catch (IOException ioe) { + LOG.error("Could not serialize accumulator " + name + '.', ioe); try { - final SerializedValue serializedValue = new SerializedValue<>(entry.getValue().getLocalValue()); - result.put(entry.getKey(), serializedValue); - } catch (IOException ioe) { - LOG.error("Could not serialize accumulator " + entry.getKey() + '.', ioe); - - try { - result.put(entry.getKey(), new SerializedValue<>(new FailedAccumulatorSerialization(ioe))); - } catch (IOException e) { - throw new RuntimeException("It should never happen that we cannot serialize the accumulator serialization exception.", e); - } + return new SerializedValue<>(OptionalFailure.ofFailure(ioe)); + } catch (IOException e) { + throw new RuntimeException("It should never happen that we cannot serialize the accumulator serialization exception.", e); } } - - return result; } /** @@ -798,7 +800,7 @@ public Map> getAccumulatorsSerialized() { */ @Override public StringifiedAccumulatorResult[] getAccumulatorResultsStringified() { - Map> accumulatorMap = aggregateUserAccumulators(); + Map>> accumulatorMap = aggregateUserAccumulators(); return StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java index 6e578fa0bda95b..e5b7aa5d5bd9b3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java @@ -48,6 +48,7 @@ import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.types.Either; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedValue; @@ -597,7 +598,7 @@ public void resetForNewExecution(final long timestamp, final long expectedGlobal // -------------------------------------------------------------------------------------------- public StringifiedAccumulatorResult[] getAggregatedUserAccumulatorsStringified() { - Map> userAccumulators = new HashMap>(); + Map>> userAccumulators = new HashMap<>(); for (ExecutionVertex vertex : taskVertices) { Map> next = vertex.getCurrentExecutionAttempt().getUserAccumulators(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java index 28fbc3002a70e5..76884046791aee 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java @@ -27,6 +27,7 @@ import org.apache.flink.runtime.executiongraph.ErrorInfo; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.util.FlinkException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; @@ -53,7 +54,7 @@ public class JobResult implements Serializable { private final JobID jobId; - private final Map> accumulatorResults; + private final Map>> accumulatorResults; private final long netRuntime; @@ -63,7 +64,7 @@ public class JobResult implements Serializable { private JobResult( final JobID jobId, - final Map> accumulatorResults, + final Map>> accumulatorResults, final long netRuntime, @Nullable final SerializedThrowable serializedThrowable) { @@ -86,7 +87,7 @@ public JobID getJobId() { return jobId; } - public Map> getAccumulatorResults() { + public Map>> getAccumulatorResults() { return accumulatorResults; } @@ -133,7 +134,7 @@ public static class Builder { private JobID jobId; - private Map> accumulatorResults; + private Map>> accumulatorResults; private long netRuntime = -1; @@ -144,7 +145,7 @@ public Builder jobId(final JobID jobId) { return this; } - public Builder accumulatorResults(final Map> accumulatorResults) { + public Builder accumulatorResults(final Map>> accumulatorResults) { this.accumulatorResults = accumulatorResults; return this; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java index 0fe920171dcfa2..964aee3bb91a76 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java @@ -31,6 +31,7 @@ import org.apache.flink.runtime.rest.messages.MessageHeaders; import org.apache.flink.runtime.webmonitor.RestfulGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedValue; import java.util.ArrayList; @@ -87,7 +88,7 @@ protected JobAccumulatorsInfo handleRequest(HandlerRequest> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized(); + Map>> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized(); accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators); } else { accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java index 22621204a7f9aa..192eaa0246b1c5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java @@ -21,6 +21,7 @@ import org.apache.flink.runtime.rest.handler.job.JobAccumulatorsHandler; import org.apache.flink.runtime.rest.messages.json.SerializedValueDeserializer; import org.apache.flink.runtime.rest.messages.json.SerializedValueSerializer; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedValue; @@ -50,13 +51,13 @@ public class JobAccumulatorsInfo implements ResponseBody { @JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS) @JsonSerialize(contentUsing = SerializedValueSerializer.class) - private Map> serializedUserAccumulators; + private Map>> serializedUserAccumulators; @JsonCreator public JobAccumulatorsInfo( @JsonProperty(FIELD_NAME_JOB_ACCUMULATORS) List jobAccumulators, @JsonProperty(FIELD_NAME_USER_TASK_ACCUMULATORS) List userAccumulators, - @JsonDeserialize(contentUsing = SerializedValueDeserializer.class) @JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS) Map> serializedUserAccumulators) { + @JsonDeserialize(contentUsing = SerializedValueDeserializer.class) @JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS) Map>> serializedUserAccumulators) { this.jobAccumulators = Preconditions.checkNotNull(jobAccumulators); this.userAccumulators = Preconditions.checkNotNull(userAccumulators); this.serializedUserAccumulators = Preconditions.checkNotNull(serializedUserAccumulators); @@ -73,7 +74,7 @@ public List getUserAccumulators() { } @JsonIgnore - public Map> getSerializedUserAccumulators() { + public Map>> getSerializedUserAccumulators() { return serializedUserAccumulators; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java index 52bb43cf69004f..e568f476c7ed1a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java @@ -20,6 +20,7 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.jobmaster.JobResult; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; @@ -69,7 +70,7 @@ public JobResult deserialize(final JsonParser p, final DeserializationContext ct JobID jobId = null; long netRuntime = -1; SerializedThrowable serializedThrowable = null; - Map> accumulatorResults = null; + Map>> accumulatorResults = null; while (true) { final JsonToken jsonToken = p.nextToken(); @@ -117,11 +118,11 @@ public JobResult deserialize(final JsonParser p, final DeserializationContext ct } @SuppressWarnings("unchecked") - private Map> parseAccumulatorResults( + private Map>> parseAccumulatorResults( final JsonParser p, final DeserializationContext ctxt) throws IOException { - final Map> accumulatorResults = new HashMap<>(); + final Map>> accumulatorResults = new HashMap<>(); while (true) { final JsonToken jsonToken = p.nextToken(); assertNotEndOfInput(p, jsonToken); @@ -132,7 +133,7 @@ private Map> parseAccumulatorResults( p.nextValue(); accumulatorResults.put( accumulatorName, - (SerializedValue) serializedValueDeserializer.deserialize(p, ctxt)); + (SerializedValue>) serializedValueDeserializer.deserialize(p, ctxt)); } return accumulatorResults; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java index a53716aab56918..694fa2f529bff0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.rest.messages.json; import org.apache.flink.runtime.jobmaster.JobResult; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; @@ -77,10 +78,10 @@ public void serialize( gen.writeFieldName(FIELD_NAME_ACCUMULATOR_RESULTS); gen.writeStartObject(); - final Map> accumulatorResults = result.getAccumulatorResults(); - for (final Map.Entry> nameValue : accumulatorResults.entrySet()) { + final Map>> accumulatorResults = result.getAccumulatorResults(); + for (final Map.Entry>> nameValue : accumulatorResults.entrySet()) { final String name = nameValue.getKey(); - final SerializedValue value = nameValue.getValue(); + final SerializedValue> value = nameValue.getValue(); gen.writeFieldName(name); serializedValueSerializer.serialize(value, gen, provider); diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala index 107ba825864c62..9ed01aa350e669 100644 --- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala +++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala @@ -20,7 +20,7 @@ package org.apache.flink.runtime.messages.accumulators import org.apache.flink.api.common.JobID import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult -import org.apache.flink.util.SerializedValue +import org.apache.flink.util.{OptionalFailure, SerializedValue} /** * Base trait of all accumulator messages @@ -62,8 +62,9 @@ case class RequestAccumulatorResultsStringified(jobID: JobID) * @param jobID Job Id of the job that the accumulator belongs to * @param result The accumulator result values, in serialized form. */ -case class AccumulatorResultsFound(jobID: JobID, - result: java.util.Map[String, SerializedValue[Object]]) +case class AccumulatorResultsFound( + jobID: JobID, + result: java.util.Map[String, SerializedValue[OptionalFailure[Object]]]) extends AccumulatorResultsResponse /** diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java index 65c4b7a39d53a1..a06bdfce2af673 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java @@ -22,6 +22,8 @@ import org.apache.flink.api.common.accumulators.IntCounter; import org.apache.flink.api.common.accumulators.SimpleAccumulator; import org.apache.flink.core.testutils.CommonTestUtils; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.OptionalFailure; import org.junit.Test; @@ -31,6 +33,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Tests for the {@link StringifiedAccumulatorResult}. @@ -63,8 +66,8 @@ public void stringifyingResultsShouldIncorporateAccumulatorLocalValueDirectly() final int targetValue = 314159; final IntCounter acc = new IntCounter(); acc.add(targetValue); - final Map> accumulatorMap = new HashMap<>(); - accumulatorMap.put(name, acc); + final Map>> accumulatorMap = new HashMap<>(); + accumulatorMap.put(name, OptionalFailure.of(acc)); final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap); @@ -80,8 +83,8 @@ public void stringifyingResultsShouldIncorporateAccumulatorLocalValueDirectly() public void stringifyingResultsShouldReportNullLocalValueAsNonnullValueString() { final String name = "a"; final NullBearingAccumulator acc = new NullBearingAccumulator(); - final Map> accumulatorMap = new HashMap<>(); - accumulatorMap.put(name, acc); + final Map>> accumulatorMap = new HashMap<>(); + accumulatorMap.put(name, OptionalFailure.of(acc)); final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap); @@ -97,7 +100,7 @@ public void stringifyingResultsShouldReportNullLocalValueAsNonnullValueString() @Test public void stringifyingResultsShouldReportNullAccumulatorWithNonnullValueAndTypeString() { final String name = "a"; - final Map> accumulatorMap = new HashMap<>(); + final Map>> accumulatorMap = new HashMap<>(); accumulatorMap.put(name, null); final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap); @@ -111,6 +114,23 @@ public void stringifyingResultsShouldReportNullAccumulatorWithNonnullValueAndTyp assertEquals("null", firstResult.getValue()); } + @Test + public void stringifyingFailureResults() { + final String name = "a"; + final Map>> accumulatorMap = new HashMap<>(); + accumulatorMap.put(name, OptionalFailure.ofFailure(new FlinkRuntimeException("Test"))); + + final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap); + + assertEquals(1, results.length); + + // Note the use of String values with content of "null" rather than null values + final StringifiedAccumulatorResult firstResult = results[0]; + assertEquals(name, firstResult.getName()); + assertEquals("null", firstResult.getType()); + assertTrue(firstResult.getValue().startsWith("org.apache.flink.util.FlinkRuntimeException: Test")); + } + private static class NullBearingAccumulator implements SimpleAccumulator { @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java index 38447e2a201d3e..c16acec88f667e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java @@ -21,19 +21,28 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.common.JobID; import org.apache.flink.core.testutils.CommonTestUtils; +import org.apache.flink.runtime.operators.testutils.ExpectedTestException; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedValue; +import org.apache.flink.util.TestLogger; + import org.junit.Test; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests for the SerializedJobExecutionResult */ -public class SerializedJobExecutionResultTest { +public class SerializedJobExecutionResultTest extends TestLogger { @Test public void testSerialization() throws Exception { @@ -42,9 +51,10 @@ public void testSerialization() throws Exception { JobID origJobId = new JobID(); long origTime = 65927436589267L; - Map> origMap = new HashMap>(); - origMap.put("name1", new SerializedValue(723L)); - origMap.put("name2", new SerializedValue("peter")); + Map>> origMap = new HashMap<>(); + origMap.put("name1", new SerializedValue<>(OptionalFailure.of(723L))); + origMap.put("name2", new SerializedValue<>(OptionalFailure.of("peter"))); + origMap.put("name3", new SerializedValue<>(OptionalFailure.ofFailure(new ExpectedTestException()))); SerializedJobExecutionResult result = new SerializedJobExecutionResult(origJobId, origTime, origMap); @@ -67,11 +77,29 @@ public void testSerialization() throws Exception { assertEquals(origTime, jResultCopied.getNetRuntime()); assertEquals(origTime, jResultCopied.getNetRuntime(TimeUnit.MILLISECONDS)); - for (Map.Entry> entry : origMap.entrySet()) { + for (Map.Entry>> entry : origMap.entrySet()) { String name = entry.getKey(); - Object value = entry.getValue().deserializeValue(classloader); - assertEquals(value, jResult.getAccumulatorResult(name)); - assertEquals(value, jResultCopied.getAccumulatorResult(name)); + OptionalFailure value = entry.getValue().deserializeValue(classloader); + if (value.isFailure()) { + try { + jResult.getAccumulatorResult(name); + fail("expected failure"); + } + catch (FlinkRuntimeException ex) { + assertTrue(ExceptionUtils.findThrowable(ex, ExpectedTestException.class).isPresent()); + } + try { + jResultCopied.getAccumulatorResult(name); + fail("expected failure"); + } + catch (FlinkRuntimeException ex) { + assertTrue(ExceptionUtils.findThrowable(ex, ExpectedTestException.class).isPresent()); + } + } + else { + assertEquals(value.get(), jResult.getAccumulatorResult(name)); + assertEquals(value.get(), jResultCopied.getAccumulatorResult(name)); + } } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java index f15dca1fcc9c63..7c2a02c3314f2f 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java @@ -44,6 +44,7 @@ import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedValue; import org.apache.flink.util.TestLogger; @@ -58,6 +59,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -310,11 +312,13 @@ private static void compareStringifiedAccumulators(StringifiedAccumulatorResult[ } } - private static void compareSerializedAccumulators(Map> runtimeAccs, Map> archivedAccs) throws IOException, ClassNotFoundException { + private static void compareSerializedAccumulators( + Map>> runtimeAccs, + Map>> archivedAccs) throws IOException, ClassNotFoundException { assertEquals(runtimeAccs.size(), archivedAccs.size()); - for (Map.Entry> runtimeAcc : runtimeAccs.entrySet()) { - long runtimeUserAcc = (long) runtimeAcc.getValue().deserializeValue(ClassLoader.getSystemClassLoader()); - long archivedUserAcc = (long) archivedAccs.get(runtimeAcc.getKey()).deserializeValue(ClassLoader.getSystemClassLoader()); + for (Entry>> runtimeAcc : runtimeAccs.entrySet()) { + long runtimeUserAcc = (long) runtimeAcc.getValue().deserializeValue(ClassLoader.getSystemClassLoader()).getUnchecked(); + long archivedUserAcc = (long) archivedAccs.get(runtimeAcc.getKey()).deserializeValue(ClassLoader.getSystemClassLoader()).getUnchecked(); assertEquals(runtimeUserAcc, archivedUserAcc); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java index 318541d288612f..df4ff04bf7427b 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java @@ -36,6 +36,8 @@ import org.apache.flink.runtime.rest.messages.job.SubtaskExecutionAttemptAccumulatorsInfo; import org.apache.flink.runtime.rest.messages.job.UserAccumulator; import org.apache.flink.runtime.testingUtils.TestingUtils; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.TestLogger; import org.junit.Test; @@ -76,9 +78,10 @@ public void testHandleRequest() throws Exception { new SubtaskAttemptMessageParameters() ); - final Map> userAccumulators = new HashMap<>(2); - userAccumulators.put("IntCounter", new IntCounter(10)); - userAccumulators.put("LongCounter", new LongCounter(100L)); + final Map>> userAccumulators = new HashMap<>(3); + userAccumulators.put("IntCounter", OptionalFailure.of(new IntCounter(10))); + userAccumulators.put("LongCounter", OptionalFailure.of(new LongCounter(100L))); + userAccumulators.put("Failure", OptionalFailure.ofFailure(new FlinkRuntimeException("Test"))); // Instance the expected result. final StringifiedAccumulatorResult[] accumulatorResults = diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java index ee7ceda1400bc1..689e1b316d2795 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java @@ -26,6 +26,7 @@ import org.apache.flink.runtime.executiongraph.ErrorInfo; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.JobVertexID; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedValue; @@ -53,7 +54,7 @@ public class ArchivedExecutionGraphBuilder { private StringifiedAccumulatorResult[] archivedUserAccumulators; private ArchivedExecutionConfig archivedExecutionConfig; private boolean isStoppable; - private Map> serializedUserAccumulators; + private Map>> serializedUserAccumulators; public ArchivedExecutionGraphBuilder setJobID(JobID jobID) { this.jobID = jobID; @@ -111,7 +112,7 @@ public ArchivedExecutionGraphBuilder setStoppable(boolean stoppable) { return this; } - public ArchivedExecutionGraphBuilder setSerializedUserAccumulators(Map> serializedUserAccumulators) { + public ArchivedExecutionGraphBuilder setSerializedUserAccumulators(Map>> serializedUserAccumulators) { this.serializedUserAccumulators = serializedUserAccumulators; return this; } @@ -134,7 +135,7 @@ public ArchivedExecutionGraph build() { failureCause, jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}", archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0], - serializedUserAccumulators != null ? serializedUserAccumulators : Collections.>emptyMap(), + serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(), archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(), isStoppable, null, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java index d7dd7eb725c7c8..9534d2bca7cc68 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.rest.messages.RestResponseMarshallingTestBase; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; @@ -54,7 +55,7 @@ public class JobExecutionResultResponseBodyTest private static final String TEST_ACCUMULATOR_NAME = "test"; - private static final Map> TEST_ACCUMULATORS = Collections.singletonMap( + private static final Map>> TEST_ACCUMULATORS = Collections.singletonMap( TEST_ACCUMULATOR_NAME, SerializedValue.fromBytes(TEST_ACCUMULATOR_VALUE)); diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala index ce3231621c63d7..c8529a9e07ac21 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala @@ -30,6 +30,7 @@ import org.apache.flink.runtime.instance.ActorGateway import org.apache.flink.runtime.jobgraph.JobStatus import org.apache.flink.runtime.messages.RequiresLeaderSessionID import org.apache.flink.runtime.messages.checkpoint.AbstractCheckpointMessage +import org.apache.flink.util.OptionalFailure object TestingJobManagerMessages { @@ -108,7 +109,7 @@ object TestingJobManagerMessages { * Reports updated accumulators back to the listener. */ case class UpdatedAccumulators(jobID: JobID, - userAccumulators: Map[String, Accumulator[_,_]]) + userAccumulators: Map[String, OptionalFailure[Accumulator[_,_]]]) /** Notifies the sender when the [[TestingJobManager]] has been elected as the leader * diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java index 282b1924943368..3d90833cbd02f5 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java @@ -18,25 +18,24 @@ package org.apache.flink.test.accumulators; +import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.accumulators.Accumulator; import org.apache.flink.api.common.accumulators.DoubleCounter; import org.apache.flink.api.common.accumulators.LongCounter; import org.apache.flink.api.common.functions.RichMapFunction; -import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.api.java.io.DiscardingOutputFormat; -import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.client.JobExecutionException; -import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster; -import org.apache.flink.test.util.TestEnvironment; +import org.apache.flink.test.util.MiniClusterResource; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -45,84 +44,80 @@ * b) are not compatible with existing accumulator. */ public class AccumulatorErrorITCase extends TestLogger { - - private static LocalFlinkMiniCluster cluster; - - private static ExecutionEnvironment env; - - @BeforeClass - public static void startCluster() { + private static final String FAULTY_CLONE_ACCUMULATOR = "faulty-clone"; + private static final String FAULTY_MERGE_ACCUMULATOR = "faulty-merge"; + private static final String INCOMPATIBLE_ACCUMULATORS_NAME = "incompatible-accumulators"; + + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfiguration(), + 2, + 3)); + + public static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2); - config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 3); config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 12L); - cluster = new LocalFlinkMiniCluster(config, false); - - cluster.start(); - - env = new TestEnvironment(cluster, 6, false); - } - - @AfterClass - public static void shutdownCluster() { - cluster.stop(); - cluster = null; + return config; } @Test public void testFaultyAccumulator() throws Exception { - + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); // Test Exception forwarding with faulty Accumulator implementation - DataSet input = env.generateSequence(0, 10000); + env.generateSequence(0, 10000) + .map(new FaultyAccumulatorUsingMapper()) + .output(new DiscardingOutputFormat<>()); - DataSet map = input.map(new FaultyAccumulatorUsingMapper()); - - map.output(new DiscardingOutputFormat()); - - try { - env.execute(); - fail("Should have failed."); - } catch (JobExecutionException e) { - Assert.assertTrue("Root cause should be:", - e.getCause() instanceof CustomException); - } + assertAccumulatorsShouldFail(env.execute()); } @Test public void testInvalidTypeAccumulator() throws Exception { + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); // Test Exception forwarding with faulty Accumulator implementation - DataSet input = env.generateSequence(0, 10000); - - DataSet mappers = input.map(new IncompatibleAccumulatorTypesMapper()) - .map(new IncompatibleAccumulatorTypesMapper2()); - - mappers.output(new DiscardingOutputFormat()); + env.generateSequence(0, 10000) + .map(new IncompatibleAccumulatorTypesMapper()) + .map(new IncompatibleAccumulatorTypesMapper2()) + .output(new DiscardingOutputFormat<>()); try { env.execute(); fail("Should have failed."); } catch (JobExecutionException e) { - Assert.assertTrue("Root cause should be:", + assertTrue("Root cause should be:", e.getCause() instanceof Exception); - Assert.assertTrue("Root cause should be:", + assertTrue("Root cause should be:", e.getCause().getCause() instanceof UnsupportedOperationException); } } + @Test + public void testFaultyMergeAccumulator() throws Exception { + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + env.getConfig().disableSysoutLogging(); + + // Test Exception forwarding with faulty Accumulator implementation + env.generateSequence(0, 10000) + .map(new FaultyMergeAccumulatorUsingMapper()) + .output(new DiscardingOutputFormat<>()); + + assertAccumulatorsShouldFail(env.execute()); + } + /* testFaultyAccumulator */ private static class FaultyAccumulatorUsingMapper extends RichMapFunction { - private static final long serialVersionUID = 42; @Override public void open(Configuration parameters) throws Exception { - getRuntimeContext().addAccumulator("test", new FaultyAccumulator()); + getRuntimeContext().addAccumulator(FAULTY_CLONE_ACCUMULATOR, new FaultyCloneAccumulator()); } @Override @@ -131,8 +126,7 @@ public Long map(Long value) throws Exception { } } - private static class FaultyAccumulator extends LongCounter { - + private static class FaultyCloneAccumulator extends LongCounter { private static final long serialVersionUID = 42; @Override @@ -141,19 +135,14 @@ public LongCounter clone() { } } - private static class CustomException extends RuntimeException { - private static final long serialVersionUID = 42; - } - /* testInvalidTypeAccumulator */ private static class IncompatibleAccumulatorTypesMapper extends RichMapFunction { - private static final long serialVersionUID = 42; @Override public void open(Configuration parameters) throws Exception { - getRuntimeContext().addAccumulator("test", new LongCounter()); + getRuntimeContext().addAccumulator(INCOMPATIBLE_ACCUMULATORS_NAME, new LongCounter()); } @Override @@ -163,12 +152,27 @@ public Long map(Long value) throws Exception { } private static class IncompatibleAccumulatorTypesMapper2 extends RichMapFunction { + private static final long serialVersionUID = 42; + + @Override + public void open(Configuration parameters) throws Exception { + getRuntimeContext().addAccumulator(INCOMPATIBLE_ACCUMULATORS_NAME, new DoubleCounter()); + } + + @Override + public Long map(Long value) throws Exception { + return -1L; + } + } + + /** */ + private static class FaultyMergeAccumulatorUsingMapper extends RichMapFunction { private static final long serialVersionUID = 42; @Override public void open(Configuration parameters) throws Exception { - getRuntimeContext().addAccumulator("test", new DoubleCounter()); + getRuntimeContext().addAccumulator(FAULTY_MERGE_ACCUMULATOR, new FaultyMergeAccumulator()); } @Override @@ -177,4 +181,31 @@ public Long map(Long value) throws Exception { } } + private static class FaultyMergeAccumulator extends LongCounter { + private static final long serialVersionUID = 42; + + @Override + public void merge(Accumulator other) { + throw new CustomException(); + } + + @Override + public LongCounter clone() { + return new FaultyMergeAccumulator(); + } + } + + private static class CustomException extends RuntimeException { + private static final long serialVersionUID = 42; + } + + private static void assertAccumulatorsShouldFail(JobExecutionResult result) { + try { + result.getAllAccumulatorResults(); + fail("Should have failed"); + } + catch (Exception ex) { + assertTrue(ExceptionUtils.findThrowable(ex, CustomException.class).isPresent()); + } + } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java index b7f54facff4b49..9f87a5638f8075 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java @@ -76,6 +76,7 @@ protected void postSubmit() throws Exception { System.out.println(AccumulatorHelper.getResultsFormatted(res.getAllAccumulatorResults())); Assert.assertEquals(Integer.valueOf(3), res.getAccumulatorResult("num-lines")); + Assert.assertEquals(Integer.valueOf(3), res.getIntCounterResult("num-lines")); Assert.assertEquals(Double.valueOf(getParallelism()), res.getAccumulatorResult("open-close-counter")); diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java index ff362dde50246c..4a2219ae24b4ae 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java @@ -160,7 +160,7 @@ private static void submitJobAndVerifyResults(JobGraph jobGraph) throws Exceptio deadline, accumulators -> accumulators.size() == 1 && accumulators.containsKey(ACCUMULATOR_NAME) - && (int) accumulators.get(ACCUMULATOR_NAME) == NUM_ITERATIONS, + && (int) accumulators.get(ACCUMULATOR_NAME).getUnchecked() == NUM_ITERATIONS, TestingUtils.defaultScheduledExecutor() ).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS); diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java index 6595b100c15813..b273ade2dd2717 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/LegacyAccumulatorLiveITCase.java @@ -49,6 +49,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.util.Collector; +import org.apache.flink.util.OptionalFailure; import org.apache.flink.util.TestLogger; import akka.actor.ActorRef; @@ -195,7 +196,7 @@ private static void verifyResults() { expectMsgClass(TIMEOUT, JobManagerMessages.JobSubmitSuccess.class); TestingJobManagerMessages.UpdatedAccumulators msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT); - Map> userAccumulators = msg.userAccumulators(); + Map>> userAccumulators = msg.userAccumulators(); ExecutionAttemptID mapperTaskID = null; @@ -264,9 +265,9 @@ private static void verifyResults() { }}; } - private static boolean checkUserAccumulators(int expected, Map> accumulatorMap) { + private static boolean checkUserAccumulators(int expected, Map>> accumulatorMap) { LOG.info("checking user accumulators"); - return accumulatorMap.containsKey(ACCUMULATOR_NAME) && expected == ((IntCounter) accumulatorMap.get(ACCUMULATOR_NAME)).getLocalValue(); + return accumulatorMap.containsKey(ACCUMULATOR_NAME) && expected == ((IntCounter) accumulatorMap.get(ACCUMULATOR_NAME).getUnchecked()).getLocalValue(); } /** diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java index 91b5de8ca5e256..8f2aaa1b1fe020 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/SavepointMigrationTestBase.java @@ -32,6 +32,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.test.util.TestBaseUtils; +import org.apache.flink.util.OptionalFailure; import org.apache.commons.io.FileUtils; import org.junit.BeforeClass; @@ -138,11 +139,11 @@ protected final void executeAndSavepoint( boolean done = false; while (DEADLINE.hasTimeLeft()) { Thread.sleep(100); - Map accumulators = client.getAccumulators(jobSubmissionResult.getJobID()); + Map> accumulators = client.getAccumulators(jobSubmissionResult.getJobID()); boolean allDone = true; for (Tuple2 acc : expectedAccumulators) { - Integer numFinished = (Integer) accumulators.get(acc.f0); + Integer numFinished = (Integer) accumulators.get(acc.f0).get(); if (numFinished == null) { allDone = false; break; @@ -211,16 +212,16 @@ protected final void restoreAndExecute( } Thread.sleep(100); - Map accumulators = client.getAccumulators(jobId); + Map> accumulators = client.getAccumulators(jobId); boolean allDone = true; for (Tuple2 acc : expectedAccumulators) { - Integer numFinished = (Integer) accumulators.get(acc.f0); + OptionalFailure numFinished = accumulators.get(acc.f0); if (numFinished == null) { allDone = false; break; } - if (!numFinished.equals(acc.f1)) { + if (!numFinished.get().equals(acc.f1)) { allDone = false; break; } From d8a376a85e0fd54c725e9117396fc5819ce17dfa Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Wed, 14 Mar 2018 15:38:48 +0100 Subject: [PATCH 0256/2294] [FLINK-8852] [sql-client] Add support for FLIP-6 in SQL Client This closes #5704. --- flink-clients/pom.xml | 12 + .../flink/client/cli/CliFrontendParser.java | 2 +- .../flink/client/program/ClusterClient.java | 10 +- .../conf/sql-client-defaults.yaml | 2 - flink-libraries/flink-sql-client/pom.xml | 9 + .../table/client/SqlClientException.java | 4 + .../flink/table/client/config/Deployment.java | 43 ++ .../local/ChangelogCollectStreamResult.java | 4 +- .../client/gateway/local/ChangelogResult.java | 4 +- .../gateway/local/CollectStreamResult.java | 13 +- .../client/gateway/local/DynamicResult.java | 9 +- .../gateway/local/ExecutionContext.java | 256 ++++++++---- .../client/gateway/local/LocalExecutor.java | 395 ++++++++++-------- .../MaterializedCollectStreamResult.java | 4 +- .../gateway/local/MaterializedResult.java | 4 +- .../client/gateway/local/ResultStore.java | 13 +- .../client/gateway/local/DependencyTest.java | 4 +- .../gateway/local/LocalExecutorITCase.java | 60 ++- .../resources/test-sql-client-defaults.yaml | 1 - .../resources/test-sql-client-factory.yaml | 1 - 20 files changed, 561 insertions(+), 289 deletions(-) diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml index 50379c7fb320a6..d15e3ffad14af8 100644 --- a/flink-clients/pom.xml +++ b/flink-clients/pom.xml @@ -147,6 +147,18 @@ under the License. + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java index eb6826436a5c88..5a6c0ff1838cbf 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontendParser.java @@ -420,7 +420,7 @@ public static CommandLine parse(Options options, String[] args, boolean stopAtNo * @param optionsB options to merge, can be null if none * @return */ - static Options mergeOptions(@Nullable Options optionsA, @Nullable Options optionsB) { + public static Options mergeOptions(@Nullable Options optionsA, @Nullable Options optionsB) { final Options resultOptions = new Options(); if (optionsA != null) { for (Option option : optionsA.getOptions()) { diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index 79a53831095819..f50206d1492c43 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -460,7 +460,7 @@ public JobSubmissionResult run( public JobSubmissionResult run(FlinkPlan compiledPlan, List libraries, List classpaths, ClassLoader classLoader, SavepointRestoreSettings savepointSettings) throws ProgramInvocationException { - JobGraph job = getJobGraph(compiledPlan, libraries, classpaths, savepointSettings); + JobGraph job = getJobGraph(flinkConfig, compiledPlan, libraries, classpaths, savepointSettings); return submitJob(job, classLoader); } @@ -882,17 +882,17 @@ private static OptimizedPlan getOptimizedPlan(Optimizer compiler, JobWithJars pr return getOptimizedPlan(compiler, prog.getPlan(), parallelism); } - public JobGraph getJobGraph(PackagedProgram prog, FlinkPlan optPlan, SavepointRestoreSettings savepointSettings) throws ProgramInvocationException { - return getJobGraph(optPlan, prog.getAllLibraries(), prog.getClasspaths(), savepointSettings); + public static JobGraph getJobGraph(Configuration flinkConfig, PackagedProgram prog, FlinkPlan optPlan, SavepointRestoreSettings savepointSettings) throws ProgramInvocationException { + return getJobGraph(flinkConfig, optPlan, prog.getAllLibraries(), prog.getClasspaths(), savepointSettings); } - public JobGraph getJobGraph(FlinkPlan optPlan, List jarFiles, List classpaths, SavepointRestoreSettings savepointSettings) { + public static JobGraph getJobGraph(Configuration flinkConfig, FlinkPlan optPlan, List jarFiles, List classpaths, SavepointRestoreSettings savepointSettings) { JobGraph job; if (optPlan instanceof StreamingPlan) { job = ((StreamingPlan) optPlan).getJobGraph(); job.setSavepointRestoreSettings(savepointSettings); } else { - JobGraphGenerator gen = new JobGraphGenerator(this.flinkConfig); + JobGraphGenerator gen = new JobGraphGenerator(flinkConfig); job = gen.compileJobGraph((OptimizedPlan) optPlan); } diff --git a/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml b/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml index 35584222e22f91..4ec64d69e52d08 100644 --- a/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml +++ b/flink-libraries/flink-sql-client/conf/sql-client-defaults.yaml @@ -64,8 +64,6 @@ execution: # programs are submitted to. deployment: - # only the 'standalone' deployment is supported - type: standalone # general cluster communication timeout in ms response-timeout: 5000 # (optional) address from cluster to gateway diff --git a/flink-libraries/flink-sql-client/pom.xml b/flink-libraries/flink-sql-client/pom.xml index 300f6ceadf213f..64ae1be0488fea 100644 --- a/flink-libraries/flink-sql-client/pom.xml +++ b/flink-libraries/flink-sql-client/pom.xml @@ -129,6 +129,15 @@ under the License. test + + org.apache.flink + + flink-clients_2.11 + ${project.version} + test-jar + test + + diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClientException.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClientException.java index bbd64b3a4959ba..ae3b84c7be78cc 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClientException.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/SqlClientException.java @@ -32,4 +32,8 @@ public SqlClientException(String message) { public SqlClientException(String message, Throwable e) { super(message, e); } + + public SqlClientException(Throwable e) { + super(e); + } } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Deployment.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Deployment.java index a87f2e40287fdf..0e809c47a49dfc 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Deployment.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/Deployment.java @@ -18,8 +18,16 @@ package org.apache.flink.table.client.config; +import org.apache.flink.client.cli.CliFrontendParser; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -58,6 +66,41 @@ public int getGatewayPort() { return Integer.parseInt(properties.getOrDefault(PropertyStrings.DEPLOYMENT_GATEWAY_PORT, Integer.toString(0))); } + /** + * Parses the given command line options from the deployment properties. Ignores properties + * that are not defined by options. + */ + public CommandLine getCommandLine(Options commandLineOptions) throws Exception { + final List args = new ArrayList<>(); + + properties.forEach((k, v) -> { + // only add supported options + if (commandLineOptions.hasOption(k)) { + final Option o = commandLineOptions.getOption(k); + final String argument = "--" + o.getLongOpt(); + // options without args + if (!o.hasArg()) { + final Boolean flag = Boolean.parseBoolean(v); + // add key only + if (flag) { + args.add(argument); + } + } + // add key and value + else if (!o.hasArgs()) { + args.add(argument); + args.add(v); + } + // options with multiple args are not supported yet + else { + throw new IllegalArgumentException("Option '" + o + "' is not supported yet."); + } + } + }); + + return CliFrontendParser.parse(commandLineOptions, args.toArray(new String[args.size()]), true); + } + public Map toProperties() { final Map copy = new HashMap<>(); properties.forEach((k, v) -> copy.put(PropertyStrings.DEPLOYMENT + "." + k, v)); diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogCollectStreamResult.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogCollectStreamResult.java index 4302481a8833b9..237558454f2c0f 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogCollectStreamResult.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogCollectStreamResult.java @@ -30,8 +30,10 @@ /** * Collects results and returns them as a changelog. + * + * @param cluster id to which this result belongs to */ -public class ChangelogCollectStreamResult extends CollectStreamResult implements ChangelogResult { +public class ChangelogCollectStreamResult extends CollectStreamResult implements ChangelogResult { private List> changeRecordBuffer; private static final int CHANGE_RECORD_BUFFER_SIZE = 5_000; diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogResult.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogResult.java index d607bfafffae19..6d4f95acf61907 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogResult.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ChangelogResult.java @@ -26,8 +26,10 @@ /** * A result that is represented as a changelog consisting of insert and delete records. + * + * @param cluster id to which this result belongs to */ -public interface ChangelogResult extends DynamicResult { +public interface ChangelogResult extends DynamicResult { /** * Retrieves the available result records. diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/CollectStreamResult.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/CollectStreamResult.java index a6f52af0ff452b..83f9ff2190017c 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/CollectStreamResult.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/CollectStreamResult.java @@ -37,8 +37,10 @@ /** * A result that works similarly to {@link DataStreamUtils#collect(DataStream)}. + * + * @param cluster id to which this result belongs to */ -public abstract class CollectStreamResult implements DynamicResult { +public abstract class CollectStreamResult implements DynamicResult { private final TypeInformation outputType; private final SocketStreamIterator> iterator; @@ -46,6 +48,7 @@ public abstract class CollectStreamResult implements DynamicResult { private final ResultRetrievalThread retrievalThread; private final JobMonitoringThread monitoringThread; private Runnable program; + private C clusterId; protected final Object resultLock; protected SqlExecutionException executionException; @@ -73,6 +76,14 @@ public CollectStreamResult(TypeInformation outputType, ExecutionConfig conf monitoringThread = new JobMonitoringThread(); } + @Override + public void setClusterId(C clusterId) { + if (this.clusterId != null) { + throw new IllegalStateException("Cluster id is already present."); + } + this.clusterId = clusterId; + } + @Override public TypeInformation getOutputType() { return outputType; diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/DynamicResult.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/DynamicResult.java index 432cc65d0445da..2042e1a4ccb0a9 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/DynamicResult.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/DynamicResult.java @@ -26,8 +26,15 @@ * A result of a dynamic table program. * *

    Note: Make sure to call close() after the result is not needed anymore. + * + * @param cluster id to which this result belongs to */ -public interface DynamicResult { +public interface DynamicResult { + + /** + * Sets the cluster id of the cluster this result comes from. This method should only be called once. + */ + void setClusterId(C clusterId); /** * Returns whether this result is materialized such that snapshots can be taken or results diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java index a013afcc0a424e..81931e2641f0d9 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java @@ -22,6 +22,11 @@ import org.apache.flink.api.common.Plan; import org.apache.flink.api.common.time.Time; import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.client.cli.CliArgsException; +import org.apache.flink.client.cli.CustomCommandLine; +import org.apache.flink.client.cli.RunOptions; +import org.apache.flink.client.deployment.ClusterDescriptor; +import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.optimizer.DataStatistics; @@ -29,48 +34,56 @@ import org.apache.flink.optimizer.costs.DefaultCostEstimator; import org.apache.flink.optimizer.plan.FlinkPlan; import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; +import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.table.api.BatchQueryConfig; import org.apache.flink.table.api.QueryConfig; import org.apache.flink.table.api.StreamQueryConfig; import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.client.config.Deployment; import org.apache.flink.table.client.config.Environment; import org.apache.flink.table.client.gateway.SessionContext; +import org.apache.flink.table.client.gateway.SqlExecutionException; import org.apache.flink.table.sources.TableSource; import org.apache.flink.table.sources.TableSourceFactoryService; +import org.apache.flink.util.FlinkException; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; import java.net.URL; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** - * Context for executing table programs. It contains configured environments and environment - * specific logic such as plan translation. + * Context for executing table programs. This class caches everything that can be cached across + * multiple queries as long as the session context does not change. This must be thread-safe as + * it might be reused across different query submission. + * + * @param cluster id */ -public class ExecutionContext { +public class ExecutionContext { private final SessionContext sessionContext; private final Environment mergedEnv; - private final ExecutionEnvironment execEnv; - private final StreamExecutionEnvironment streamExecEnv; - private final TableEnvironment tableEnv; + private final List dependencies; private final ClassLoader classLoader; - private final QueryConfig queryConfig; + private final Map> tableSources; + private final Configuration flinkConfig; + private final CommandLine commandLine; + private final CustomCommandLine activeCommandLine; + private final RunOptions runOptions; + private final T clusterId; + private final ClusterSpecification clusterSpec; - public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List dependencies) { + public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List dependencies, + Configuration flinkConfig, Options commandLineOptions, List> availableCommandLines) { this.sessionContext = sessionContext; this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment()); - - // create environments - if (mergedEnv.getExecution().isStreamingExecution()) { - streamExecEnv = createStreamExecutionEnvironment(); - execEnv = null; - tableEnv = TableEnvironment.getTableEnvironment(streamExecEnv); - } else { - streamExecEnv = null; - execEnv = createExecutionEnvironment(); - tableEnv = TableEnvironment.getTableEnvironment(execEnv); - } + this.dependencies = dependencies; + this.flinkConfig = flinkConfig; // create class loader classLoader = FlinkUserCodeClassLoaders.parentFirst( @@ -78,90 +91,189 @@ public ExecutionContext(Environment defaultEnvironment, SessionContext sessionCo this.getClass().getClassLoader()); // create table sources + tableSources = new HashMap<>(); mergedEnv.getSources().forEach((name, source) -> { - TableSource tableSource = TableSourceFactoryService.findAndCreateTableSource(source, classLoader); - tableEnv.registerTableSource(name, tableSource); + final TableSource tableSource = TableSourceFactoryService.findAndCreateTableSource(source, classLoader); + tableSources.put(name, tableSource); }); - // create query config - queryConfig = createQueryConfig(); + // convert deployment options into command line options that describe a cluster + commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions); + activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine); + runOptions = createRunOptions(commandLine); + clusterId = activeCommandLine.getClusterId(commandLine); + clusterSpec = createClusterSpecification(activeCommandLine, commandLine); } public SessionContext getSessionContext() { return sessionContext; } - public ExecutionEnvironment getExecutionEnvironment() { - return execEnv; + public ClassLoader getClassLoader() { + return classLoader; } - public StreamExecutionEnvironment getStreamExecutionEnvironment() { - return streamExecEnv; + public Environment getMergedEnvironment() { + return mergedEnv; } - public TableEnvironment getTableEnvironment() { - return tableEnv; + public ClusterSpecification getClusterSpec() { + return clusterSpec; } - public ClassLoader getClassLoader() { - return classLoader; + public T getClusterId() { + return clusterId; } - public Environment getMergedEnvironment() { - return mergedEnv; + public ClusterDescriptor createClusterDescriptor() throws Exception { + return activeCommandLine.createClusterDescriptor(commandLine); } - public QueryConfig getQueryConfig() { - return queryConfig; + public EnvironmentInstance createEnvironmentInstance() { + return new EnvironmentInstance(); } - public ExecutionConfig getExecutionConfig() { - if (streamExecEnv != null) { - return streamExecEnv.getConfig(); - } else { - return execEnv.getConfig(); + // -------------------------------------------------------------------------------------------- + + private static CommandLine createCommandLine(Deployment deployment, Options commandLineOptions) { + try { + return deployment.getCommandLine(commandLineOptions); + } catch (Exception e) { + throw new SqlExecutionException("Invalid deployment options.", e); } } - public FlinkPlan createPlan(String name, Configuration flinkConfig) { - if (streamExecEnv != null) { - final StreamGraph graph = streamExecEnv.getStreamGraph(); - graph.setJobName(name); - return graph; - } else { - final int parallelism = execEnv.getParallelism(); - final Plan unoptimizedPlan = execEnv.createProgramPlan(); - unoptimizedPlan.setJobName(name); - final Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); - return ClusterClient.getOptimizedPlan(compiler, unoptimizedPlan, parallelism); + @SuppressWarnings("unchecked") + private static CustomCommandLine findActiveCommandLine(List> availableCommandLines, CommandLine commandLine) { + for (CustomCommandLine cli : availableCommandLines) { + if (cli.isActive(commandLine)) { + return (CustomCommandLine) cli; + } } + throw new SqlExecutionException("Could not find a matching deployment."); } - // -------------------------------------------------------------------------------------------- - - private ExecutionEnvironment createExecutionEnvironment() { - final ExecutionEnvironment execEnv = ExecutionEnvironment.getExecutionEnvironment(); - execEnv.setParallelism(mergedEnv.getExecution().getParallelism()); - return execEnv; + private static RunOptions createRunOptions(CommandLine commandLine) { + try { + return new RunOptions(commandLine); + } catch (CliArgsException e) { + throw new SqlExecutionException("Invalid deployment run options.", e); + } } - private StreamExecutionEnvironment createStreamExecutionEnvironment() { - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setParallelism(mergedEnv.getExecution().getParallelism()); - env.setMaxParallelism(mergedEnv.getExecution().getMaxParallelism()); - env.setStreamTimeCharacteristic(mergedEnv.getExecution().getTimeCharacteristic()); - return env; + private static ClusterSpecification createClusterSpecification(CustomCommandLine activeCommandLine, CommandLine commandLine) { + try { + return activeCommandLine.getClusterSpecification(commandLine); + } catch (FlinkException e) { + throw new SqlExecutionException("Could not create cluster specification for the given deployment.", e); + } } - private QueryConfig createQueryConfig() { - if (streamExecEnv != null) { - final StreamQueryConfig config = new StreamQueryConfig(); - final long minRetention = mergedEnv.getExecution().getMinStateRetention(); - final long maxRetention = mergedEnv.getExecution().getMaxStateRetention(); - config.withIdleStateRetentionTime(Time.milliseconds(minRetention), Time.milliseconds(maxRetention)); - return config; - } else { - return new BatchQueryConfig(); + // -------------------------------------------------------------------------------------------- + + /** + * {@link ExecutionEnvironment} and {@link StreamExecutionEnvironment} cannot be reused + * across multiple queries because they are stateful. This class abstracts execution + * environments and table environments. + */ + public class EnvironmentInstance { + + private final QueryConfig queryConfig; + private final ExecutionEnvironment execEnv; + private final StreamExecutionEnvironment streamExecEnv; + private final TableEnvironment tableEnv; + + private EnvironmentInstance() { + // create environments + if (mergedEnv.getExecution().isStreamingExecution()) { + streamExecEnv = createStreamExecutionEnvironment(); + execEnv = null; + tableEnv = TableEnvironment.getTableEnvironment(streamExecEnv); + } else { + streamExecEnv = null; + execEnv = createExecutionEnvironment(); + tableEnv = TableEnvironment.getTableEnvironment(execEnv); + } + + // create query config + queryConfig = createQueryConfig(); + + // register table sources + tableSources.forEach(tableEnv::registerTableSource); + } + + public QueryConfig getQueryConfig() { + return queryConfig; + } + + public ExecutionEnvironment getExecutionEnvironment() { + return execEnv; + } + + public StreamExecutionEnvironment getStreamExecutionEnvironment() { + return streamExecEnv; + } + + public TableEnvironment getTableEnvironment() { + return tableEnv; + } + + public ExecutionConfig getExecutionConfig() { + if (streamExecEnv != null) { + return streamExecEnv.getConfig(); + } else { + return execEnv.getConfig(); + } + } + + public JobGraph createJobGraph(String name) { + final FlinkPlan plan = createPlan(name, flinkConfig); + return ClusterClient.getJobGraph( + flinkConfig, + plan, + dependencies, + runOptions.getClasspaths(), + runOptions.getSavepointRestoreSettings()); + } + + private FlinkPlan createPlan(String name, Configuration flinkConfig) { + if (streamExecEnv != null) { + final StreamGraph graph = streamExecEnv.getStreamGraph(); + graph.setJobName(name); + return graph; + } else { + final int parallelism = execEnv.getParallelism(); + final Plan unoptimizedPlan = execEnv.createProgramPlan(); + unoptimizedPlan.setJobName(name); + final Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); + return ClusterClient.getOptimizedPlan(compiler, unoptimizedPlan, parallelism); + } + } + + private ExecutionEnvironment createExecutionEnvironment() { + final ExecutionEnvironment execEnv = ExecutionEnvironment.getExecutionEnvironment(); + execEnv.setParallelism(mergedEnv.getExecution().getParallelism()); + return execEnv; + } + + private StreamExecutionEnvironment createStreamExecutionEnvironment() { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(mergedEnv.getExecution().getParallelism()); + env.setMaxParallelism(mergedEnv.getExecution().getMaxParallelism()); + env.setStreamTimeCharacteristic(mergedEnv.getExecution().getTimeCharacteristic()); + return env; + } + + private QueryConfig createQueryConfig() { + if (streamExecEnv != null) { + final StreamQueryConfig config = new StreamQueryConfig(); + final long minRetention = mergedEnv.getExecution().getMinStateRetention(); + final long maxRetention = mergedEnv.getExecution().getMaxStateRetention(); + config.withIdleStateRetentionTime(Time.milliseconds(minRetention), Time.milliseconds(maxRetention)); + return config; + } else { + return new BatchQueryConfig(); + } } } } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java index fa6c9d2fd26cd7..30fa3c0998a9d2 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java @@ -21,35 +21,32 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.client.cli.CliFrontend; +import org.apache.flink.client.cli.CliFrontendParser; +import org.apache.flink.client.cli.CustomCommandLine; import org.apache.flink.client.deployment.ClusterDescriptor; -import org.apache.flink.client.deployment.ClusterRetrieveException; -import org.apache.flink.client.deployment.StandaloneClusterDescriptor; -import org.apache.flink.client.deployment.StandaloneClusterId; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.JobWithJars; -import org.apache.flink.client.program.ProgramInvocationException; -import org.apache.flink.configuration.AkkaOptions; +import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; -import org.apache.flink.optimizer.plan.FlinkPlan; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.client.SqlClientException; -import org.apache.flink.table.client.config.Deployment; import org.apache.flink.table.client.config.Environment; import org.apache.flink.table.client.gateway.Executor; import org.apache.flink.table.client.gateway.ResultDescriptor; import org.apache.flink.table.client.gateway.SessionContext; import org.apache.flink.table.client.gateway.SqlExecutionException; import org.apache.flink.table.client.gateway.TypedResult; -import org.apache.flink.table.sinks.TableSink; import org.apache.flink.types.Row; import org.apache.flink.util.StringUtils; +import org.apache.commons.cli.Options; + import java.io.File; import java.io.IOException; import java.net.MalformedURLException; @@ -69,16 +66,23 @@ public class LocalExecutor implements Executor { private static final String DEFAULT_ENV_FILE = "sql-client-defaults.yaml"; + // deployment + private final Environment defaultEnvironment; private final List dependencies; private final Configuration flinkConfig; + private final List> commandLines; + private final Options commandLineOptions; + + // result maintenance + private final ResultStore resultStore; /** * Cached execution context for unmodified sessions. Do not access this variable directly * but through {@link LocalExecutor#getOrCreateExecutionContext}. */ - private ExecutionContext executionContext; + private ExecutionContext executionContext; /** * Creates a local executor for submitting table programs and retrieving results. @@ -92,6 +96,18 @@ public LocalExecutor(URL defaultEnv, List jars, List libraries) { // load the global configuration this.flinkConfig = GlobalConfiguration.loadConfiguration(flinkConfigDir); + + // initialize default file system + try { + FileSystem.initialize(this.flinkConfig); + } catch (IOException e) { + throw new SqlClientException( + "Error while setting the default filesystem scheme from configuration.", e); + } + + // load command lines for deployment + this.commandLines = CliFrontend.loadCustomCommandLines(flinkConfig, flinkConfigDir); + this.commandLineOptions = collectCommandLineOptions(commandLines); } catch (Exception e) { throw new SqlClientException("Could not load Flink configuration.", e); } @@ -107,7 +123,7 @@ public LocalExecutor(URL defaultEnv, List jars, List libraries) { try { defaultEnv = Path.fromLocalFile(file).toUri().toURL(); } catch (MalformedURLException e) { - throw new RuntimeException(e); + throw new SqlClientException(e); } } else { System.out.println("not found."); @@ -127,38 +143,7 @@ public LocalExecutor(URL defaultEnv, List jars, List libraries) { } // discover dependencies - dependencies = new ArrayList<>(); - try { - // find jar files - for (URL url : jars) { - JobWithJars.checkJarFile(url); - dependencies.add(url); - } - - // find jar files in library directories - for (URL libUrl : libraries) { - final File dir = new File(libUrl.toURI()); - if (!dir.isDirectory()) { - throw new SqlClientException("Directory expected: " + dir); - } else if (!dir.canRead()) { - throw new SqlClientException("Directory cannot be read: " + dir); - } - final File[] files = dir.listFiles(); - if (files == null) { - throw new SqlClientException("Directory cannot be read: " + dir); - } - for (File f : files) { - // only consider jars - if (f.isFile() && f.getAbsolutePath().toLowerCase().endsWith(".jar")) { - final URL url = f.toURI().toURL(); - JobWithJars.checkJarFile(url); - dependencies.add(url); - } - } - } - } catch (Exception e) { - throw new SqlClientException("Could not load all required JAR files.", e); - } + dependencies = discoverDependencies(jars, libraries); // prepare result store resultStore = new ResultStore(flinkConfig); @@ -167,10 +152,12 @@ public LocalExecutor(URL defaultEnv, List jars, List libraries) { /** * Constructor for testing purposes. */ - public LocalExecutor(Environment defaultEnvironment, List dependencies, Configuration flinkConfig) { + public LocalExecutor(Environment defaultEnvironment, List dependencies, Configuration flinkConfig, CustomCommandLine commandLine) { this.defaultEnvironment = defaultEnvironment; this.dependencies = dependencies; this.flinkConfig = flinkConfig; + this.commandLines = Collections.singletonList(commandLine); + this.commandLineOptions = collectCommandLineOptions(commandLines); // prepare result store resultStore = new ResultStore(flinkConfig); @@ -183,7 +170,8 @@ public void start() { @Override public Map getSessionProperties(SessionContext session) throws SqlExecutionException { - final Environment env = getOrCreateExecutionContext(session).getMergedEnvironment(); + final Environment env = getOrCreateExecutionContext(session) + .getMergedEnvironment(); final Map properties = new HashMap<>(); properties.putAll(env.getExecution().toProperties()); properties.putAll(env.getDeployment().toProperties()); @@ -192,13 +180,17 @@ public Map getSessionProperties(SessionContext session) throws S @Override public List listTables(SessionContext session) throws SqlExecutionException { - final TableEnvironment tableEnv = getOrCreateExecutionContext(session).getTableEnvironment(); + final TableEnvironment tableEnv = getOrCreateExecutionContext(session) + .createEnvironmentInstance() + .getTableEnvironment(); return Arrays.asList(tableEnv.listTables()); } @Override public TableSchema getTableSchema(SessionContext session, String name) throws SqlExecutionException { - final TableEnvironment tableEnv = getOrCreateExecutionContext(session).getTableEnvironment(); + final TableEnvironment tableEnv = getOrCreateExecutionContext(session) + .createEnvironmentInstance() + .getTableEnvironment(); try { return tableEnv.scan(name).getSchema(); } catch (Throwable t) { @@ -209,12 +201,14 @@ public TableSchema getTableSchema(SessionContext session, String name) throws Sq @Override public String explainStatement(SessionContext session, String statement) throws SqlExecutionException { - final ExecutionContext context = getOrCreateExecutionContext(session); + final TableEnvironment tableEnv = getOrCreateExecutionContext(session) + .createEnvironmentInstance() + .getTableEnvironment(); // translate try { - final Table table = createTable(context, statement); - return context.getTableEnvironment().explain(table); + final Table table = createTable(tableEnv, statement); + return tableEnv.explain(table); } catch (Throwable t) { // catch everything such that the query does not crash the executor throw new SqlExecutionException("Invalid SQL statement.", t); @@ -223,60 +217,8 @@ public String explainStatement(SessionContext session, String statement) throws @Override public ResultDescriptor executeQuery(SessionContext session, String query) throws SqlExecutionException { - final ExecutionContext context = getOrCreateExecutionContext(session); - final Environment mergedEnv = context.getMergedEnvironment(); - - // create table here to fail quickly for wrong queries - final Table table = createTable(context, query); - final TableSchema resultSchema = table.getSchema().withoutTimeAttributes(); - - // deployment - final ClusterClient clusterClient = createDeployment(mergedEnv.getDeployment()); - - // initialize result - final DynamicResult result = resultStore.createResult( - mergedEnv, - resultSchema, - context.getExecutionConfig()); - - // create job graph with jars - final JobGraph jobGraph; - try { - jobGraph = createJobGraph(context, context.getSessionContext().getName() + ": " + query, table, - result.getTableSink(), - clusterClient); - } catch (Throwable t) { - // the result needs to be closed as long as - // it not stored in the result store - result.close(); - throw t; - } - - // store the result with a unique id (the job id for now) - final String resultId = jobGraph.getJobID().toString(); - resultStore.storeResult(resultId, result); - - // create execution - final Runnable program = () -> { - // we need to submit the job attached for now - // otherwise it is not possible to retrieve the reason why an execution failed - try { - clusterClient.run(jobGraph, context.getClassLoader()); - } catch (ProgramInvocationException e) { - throw new SqlExecutionException("Could not execute table program.", e); - } finally { - try { - clusterClient.shutdown(); - } catch (Exception e) { - // ignore - } - } - }; - - // start result retrieval - result.startRetrieval(program); - - return new ResultDescriptor(resultId, resultSchema, result.isMaterialized()); + final ExecutionContext context = getOrCreateExecutionContext(session); + return executeQueryInternal(context, query); } @Override @@ -289,7 +231,7 @@ public TypedResult>> retrieveResultChanges(SessionCont if (result.isMaterialized()) { throw new SqlExecutionException("Invalid result retrieval mode."); } - return ((ChangelogResult) result).retrieveChanges(); + return ((ChangelogResult) result).retrieveChanges(); } @Override @@ -301,7 +243,7 @@ public TypedResult snapshotResult(SessionContext session, String result if (!result.isMaterialized()) { throw new SqlExecutionException("Invalid result retrieval mode."); } - return ((MaterializedResult) result).snapshot(pageSize); + return ((MaterializedResult) result).snapshot(pageSize); } @Override @@ -313,34 +255,13 @@ public List retrieveResultPage(String resultId, int page) throws SqlExecuti if (!result.isMaterialized()) { throw new SqlExecutionException("Invalid result retrieval mode."); } - return ((MaterializedResult) result).retrievePage(page); + return ((MaterializedResult) result).retrievePage(page); } @Override public void cancelQuery(SessionContext session, String resultId) throws SqlExecutionException { - final DynamicResult result = resultStore.getResult(resultId); - if (result == null) { - throw new SqlExecutionException("Could not find a result with result identifier '" + resultId + "'."); - } - - // stop retrieval and remove the result - result.close(); - resultStore.removeResult(resultId); - - // stop Flink job - final Environment mergedEnv = getOrCreateExecutionContext(session).getMergedEnvironment(); - final ClusterClient clusterClient = createDeployment(mergedEnv.getDeployment()); - try { - clusterClient.cancel(new JobID(StringUtils.hexStringToByte(resultId))); - } catch (Throwable t) { - // the job might has finished earlier - } finally { - try { - clusterClient.shutdown(); - } catch (Throwable t) { - // ignore - } - } + final ExecutionContext context = getOrCreateExecutionContext(session); + cancelQueryInternal(context, resultId); } @Override @@ -356,72 +277,105 @@ public void stop(SessionContext session) { // -------------------------------------------------------------------------------------------- - private Table createTable(ExecutionContext context, String query) { - // parse and validate query - try { - return context.getTableEnvironment().sqlQuery(query); - } catch (Throwable t) { - // catch everything such that the query does not crash the executor - throw new SqlExecutionException("Invalid SQL statement.", t); + private void cancelQueryInternal(ExecutionContext context, String resultId) { + final DynamicResult result = resultStore.getResult(resultId); + if (result == null) { + throw new SqlExecutionException("Could not find a result with result identifier '" + resultId + "'."); + } + + // stop retrieval and remove the result + result.close(); + resultStore.removeResult(resultId); + + // stop Flink job + try (final ClusterDescriptor clusterDescriptor = context.createClusterDescriptor()) { + ClusterClient clusterClient = null; + try { + // retrieve existing cluster + clusterClient = clusterDescriptor.retrieve(context.getClusterId()); + try { + clusterClient.cancel(new JobID(StringUtils.hexStringToByte(resultId))); + } catch (Throwable t) { + // the job might has finished earlier + } + } catch (Exception e) { + throw new SqlExecutionException("Could not retrieve or create a cluster.", e); + } finally { + try { + if (clusterClient != null) { + clusterClient.shutdown(); + } + } catch (Exception e) { + // ignore + } + } + } catch (SqlExecutionException e) { + throw e; + } catch (Exception e) { + throw new SqlExecutionException("Could not locate a cluster.", e); } } - private JobGraph createJobGraph(ExecutionContext context, String name, Table table, - TableSink sink, ClusterClient clusterClient) { + private ResultDescriptor executeQueryInternal(ExecutionContext context, String query) { + final ExecutionContext.EnvironmentInstance envInst = context.createEnvironmentInstance(); - // translate + // create table + final Table table = createTable(envInst.getTableEnvironment(), query); + + // initialize result + final DynamicResult result = resultStore.createResult( + context.getMergedEnvironment(), + table.getSchema().withoutTimeAttributes(), + envInst.getExecutionConfig()); + + // create job graph with dependencies + final String jobName = context.getSessionContext().getName() + ": " + query; + final JobGraph jobGraph; try { - table.writeToSink(sink, context.getQueryConfig()); + table.writeToSink(result.getTableSink(), envInst.getQueryConfig()); + jobGraph = envInst.createJobGraph(jobName); } catch (Throwable t) { + // the result needs to be closed as long as + // it not stored in the result store + result.close(); // catch everything such that the query does not crash the executor throw new SqlExecutionException("Invalid SQL statement.", t); } - // extract plan - final FlinkPlan plan = context.createPlan(name, clusterClient.getFlinkConfiguration()); - - // create job graph - return clusterClient.getJobGraph( - plan, - dependencies, - Collections.emptyList(), - SavepointRestoreSettings.none()); - } - - private ClusterClient createDeployment(Deployment deploy) { + // store the result with a unique id (the job id for now) + final String resultId = jobGraph.getJobID().toString(); + resultStore.storeResult(resultId, result); - // change some configuration options for being more responsive - flinkConfig.setString(AkkaOptions.LOOKUP_TIMEOUT, deploy.getResponseTimeout() + " ms"); - flinkConfig.setString(AkkaOptions.CLIENT_TIMEOUT, deploy.getResponseTimeout() + " ms"); + // create execution + final Runnable program = () -> deployJob(context, jobGraph, result); - // get cluster client - final ClusterClient clusterClient; - if (deploy.isStandaloneDeployment()) { - clusterClient = createStandaloneClusterClient(flinkConfig); - clusterClient.setPrintStatusDuringExecution(false); - } else { - throw new SqlExecutionException("Unsupported deployment."); - } + // start result retrieval + result.startRetrieval(program); - return clusterClient; + return new ResultDescriptor( + resultId, + table.getSchema().withoutTimeAttributes(), + result.isMaterialized()); } - private ClusterClient createStandaloneClusterClient(Configuration configuration) { - final ClusterDescriptor descriptor = new StandaloneClusterDescriptor(configuration); + private Table createTable(TableEnvironment tableEnv, String query) { + // parse and validate query try { - return descriptor.retrieve(StandaloneClusterId.getInstance()); - } catch (ClusterRetrieveException e) { - throw new SqlExecutionException("Could not retrievePage standalone cluster.", e); + return tableEnv.sqlQuery(query); + } catch (Throwable t) { + // catch everything such that the query does not crash the executor + throw new SqlExecutionException("Invalid SQL statement.", t); } } /** * Creates or reuses the execution context. */ - private synchronized ExecutionContext getOrCreateExecutionContext(SessionContext session) throws SqlExecutionException { + private synchronized ExecutionContext getOrCreateExecutionContext(SessionContext session) throws SqlExecutionException { if (executionContext == null || !executionContext.getSessionContext().equals(session)) { try { - executionContext = new ExecutionContext(defaultEnvironment, session, dependencies); + executionContext = new ExecutionContext<>(defaultEnvironment, session, dependencies, + flinkConfig, commandLineOptions, commandLines); } catch (Throwable t) { // catch everything such that a configuration does not crash the executor throw new SqlExecutionException("Could not create execution context.", t); @@ -429,4 +383,101 @@ private synchronized ExecutionContext getOrCreateExecutionContext(SessionContext } return executionContext; } + + /** + * Deploys a job. Depending on the deployment create a new job cluster. It saves cluster id in + * the result and blocks until job completion. + */ + private void deployJob(ExecutionContext context, JobGraph jobGraph, DynamicResult result) { + // create or retrieve cluster and deploy job + try (final ClusterDescriptor clusterDescriptor = context.createClusterDescriptor()) { + ClusterClient clusterClient = null; + try { + // new cluster + if (context.getClusterId() == null) { + // deploy job cluster with job attached + clusterClient = clusterDescriptor.deployJobCluster(context.getClusterSpec(), jobGraph, false); + // save the new cluster id + result.setClusterId(clusterClient.getClusterId()); + // we need to hard cast for now + ((RestClusterClient) clusterClient) + .requestJobResult(jobGraph.getJobID()) + .get() + .toJobExecutionResult(context.getClassLoader()); // throws exception if job fails + } + // reuse existing cluster + else { + // retrieve existing cluster + clusterClient = clusterDescriptor.retrieve(context.getClusterId()); + // save the cluster id + result.setClusterId(clusterClient.getClusterId()); + // submit the job + clusterClient.setDetached(false); + clusterClient.submitJob(jobGraph, context.getClassLoader()); // throws exception if job fails + } + } catch (Exception e) { + throw new SqlExecutionException("Could not retrieve or create a cluster.", e); + } finally { + try { + if (clusterClient != null) { + clusterClient.shutdown(); + } + } catch (Exception e) { + // ignore + } + } + } catch (SqlExecutionException e) { + throw e; + } catch (Exception e) { + throw new SqlExecutionException("Could not locate a cluster.", e); + } + } + + // -------------------------------------------------------------------------------------------- + + private static List discoverDependencies(List jars, List libraries) { + final List dependencies = new ArrayList<>(); + try { + // find jar files + for (URL url : jars) { + JobWithJars.checkJarFile(url); + dependencies.add(url); + } + + // find jar files in library directories + for (URL libUrl : libraries) { + final File dir = new File(libUrl.toURI()); + if (!dir.isDirectory()) { + throw new SqlClientException("Directory expected: " + dir); + } else if (!dir.canRead()) { + throw new SqlClientException("Directory cannot be read: " + dir); + } + final File[] files = dir.listFiles(); + if (files == null) { + throw new SqlClientException("Directory cannot be read: " + dir); + } + for (File f : files) { + // only consider jars + if (f.isFile() && f.getAbsolutePath().toLowerCase().endsWith(".jar")) { + final URL url = f.toURI().toURL(); + JobWithJars.checkJarFile(url); + dependencies.add(url); + } + } + } + } catch (Exception e) { + throw new SqlClientException("Could not load all required JAR files.", e); + } + return dependencies; + } + + private static Options collectCommandLineOptions(List> commandLines) { + final Options customOptions = new Options(); + for (CustomCommandLine customCommandLine : commandLines) { + customCommandLine.addRunOptions(customOptions); + } + return CliFrontendParser.mergeOptions( + CliFrontendParser.getRunCommandOptions(), + customOptions); + } } diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java index 7935da63e0bf8f..0ce270ee13f20c 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedCollectStreamResult.java @@ -33,8 +33,10 @@ /** * Collects results and returns them as table snapshots. + * + * @param cluster id to which this result belongs to */ -public class MaterializedCollectStreamResult extends CollectStreamResult implements MaterializedResult { +public class MaterializedCollectStreamResult extends CollectStreamResult implements MaterializedResult { private final List materializedTable; private final Map> rowPositions; // positions of rows in table for faster access diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedResult.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedResult.java index 9306e7863efa9a..858af4d931c13d 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedResult.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/MaterializedResult.java @@ -25,8 +25,10 @@ /** * A result that is materialized and can be viewed by navigating through a snapshot. + * + * @param cluster id to which this result belongs to */ -public interface MaterializedResult extends DynamicResult { +public interface MaterializedResult extends DynamicResult { /** * Takes a snapshot of the current table and returns the number of pages for navigating diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java index 19a440e22246b4..7e17ee7d186baf 100644 --- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java +++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java @@ -46,7 +46,7 @@ public class ResultStore { private Configuration flinkConfig; - private Map results; + private Map> results; public ResultStore(Configuration flinkConfig) { this.flinkConfig = flinkConfig; @@ -57,7 +57,7 @@ public ResultStore(Configuration flinkConfig) { /** * Creates a result. Might start threads or opens sockets so every created result must be closed. */ - public DynamicResult createResult(Environment env, TableSchema schema, ExecutionConfig config) { + public DynamicResult createResult(Environment env, TableSchema schema, ExecutionConfig config) { if (!env.getExecution().isStreamingExecution()) { throw new SqlExecutionException("Emission is only supported in streaming environments yet."); } @@ -68,9 +68,9 @@ public DynamicResult createResult(Environment env, TableSchema schema, Execution final int gatewayPort = getGatewayPort(env.getDeployment()); if (env.getExecution().isChangelogMode()) { - return new ChangelogCollectStreamResult(outputType, config, gatewayAddress, gatewayPort); + return new ChangelogCollectStreamResult<>(outputType, config, gatewayAddress, gatewayPort); } else { - return new MaterializedCollectStreamResult(outputType, config, gatewayAddress, gatewayPort); + return new MaterializedCollectStreamResult<>(outputType, config, gatewayAddress, gatewayPort); } } @@ -78,8 +78,9 @@ public void storeResult(String resultId, DynamicResult result) { results.put(resultId, result); } - public DynamicResult getResult(String resultId) { - return results.get(resultId); + @SuppressWarnings("unchecked") + public DynamicResult getResult(String resultId) { + return (DynamicResult) results.get(resultId); } public void removeResult(String resultId) { diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java index 40a1c2cf137c2a..bbb2024c649e44 100644 --- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/DependencyTest.java @@ -18,6 +18,7 @@ package org.apache.flink.table.client.gateway.local; +import org.apache.flink.client.cli.Flip6DefaultCLI; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.Types; @@ -57,7 +58,8 @@ public void testTableSourceFactoryDiscovery() throws Exception { final LocalExecutor executor = new LocalExecutor( env, Collections.singletonList(dependency), - new Configuration()); + new Configuration(), + new Flip6DefaultCLI(new Configuration())); final SessionContext session = new SessionContext("test-session", new Environment()); diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java index 45369784f563ca..378d475107a99e 100644 --- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java +++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java @@ -22,9 +22,12 @@ import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.client.cli.util.DummyCustomCommandLine; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.runtime.minicluster.StandaloneMiniCluster; +import org.apache.flink.configuration.WebOptions; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.client.config.Environment; import org.apache.flink.table.client.gateway.Executor; @@ -32,12 +35,13 @@ import org.apache.flink.table.client.gateway.SessionContext; import org.apache.flink.table.client.gateway.TypedResult; import org.apache.flink.table.client.gateway.utils.EnvironmentFileUtil; +import org.apache.flink.test.util.MiniClusterResource; import org.apache.flink.test.util.TestBaseUtils; import org.apache.flink.types.Row; import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; import java.net.URL; @@ -62,25 +66,36 @@ public class LocalExecutorITCase extends TestLogger { private static final String DEFAULTS_ENVIRONMENT_FILE = "test-sql-client-defaults.yaml"; - private static StandaloneMiniCluster cluster; + private static final int NUM_TMS = 2; + private static final int NUM_SLOTS_PER_TM = 2; - @BeforeClass - public static void before() throws Exception { - final Configuration config = new Configuration(); - config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); + @ClassRule + public static final MiniClusterResource MINI_CLUSTER_RESOURCE = new MiniClusterResource( + new MiniClusterResource.MiniClusterResourceConfiguration( + getConfig(), + NUM_TMS, + NUM_SLOTS_PER_TM), + true); + + private static ClusterClient clusterClient; - cluster = new StandaloneMiniCluster(config); + @BeforeClass + public static void setup() { + clusterClient = MINI_CLUSTER_RESOURCE.getClusterClient(); } - @AfterClass - public static void after() throws Exception { - cluster.close(); - cluster = null; + private static Configuration getConfig() { + Configuration config = new Configuration(); + config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 4L); + config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); + config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); + config.setBoolean(WebOptions.SUBMIT_ENABLE, false); + return config; } @Test public void testListTables() throws Exception { - final Executor executor = createDefaultExecutor(); + final Executor executor = createDefaultExecutor(clusterClient); final SessionContext session = new SessionContext("test-session", new Environment()); final List actualTables = executor.listTables(session); @@ -91,7 +106,7 @@ public void testListTables() throws Exception { @Test public void testGetSessionProperties() throws Exception { - final Executor executor = createDefaultExecutor(); + final Executor executor = createDefaultExecutor(clusterClient); final SessionContext session = new SessionContext("test-session", new Environment()); // modify defaults @@ -107,7 +122,6 @@ public void testGetSessionProperties() throws Exception { expectedProperties.put("execution.max-idle-state-retention", "0"); expectedProperties.put("execution.min-idle-state-retention", "0"); expectedProperties.put("execution.result-mode", "table"); - expectedProperties.put("deployment.type", "standalone"); expectedProperties.put("deployment.response-timeout", "5000"); assertEquals(expectedProperties, actualProperties); @@ -115,7 +129,7 @@ public void testGetSessionProperties() throws Exception { @Test public void testTableSchema() throws Exception { - final Executor executor = createDefaultExecutor(); + final Executor executor = createDefaultExecutor(clusterClient); final SessionContext session = new SessionContext("test-session", new Environment()); final TableSchema actualTableSchema = executor.getTableSchema(session, "TableNumber2"); @@ -136,7 +150,7 @@ public void testQueryExecutionChangelog() throws Exception { replaceVars.put("$VAR_1", "/"); replaceVars.put("$VAR_2", "changelog"); - final Executor executor = createModifiedExecutor(replaceVars); + final Executor executor = createModifiedExecutor(clusterClient, replaceVars); final SessionContext session = new SessionContext("test-session", new Environment()); try { @@ -183,7 +197,7 @@ public void testQueryExecutionTable() throws Exception { replaceVars.put("$VAR_1", "/"); replaceVars.put("$VAR_2", "table"); - final Executor executor = createModifiedExecutor(replaceVars); + final Executor executor = createModifiedExecutor(clusterClient, replaceVars); final SessionContext session = new SessionContext("test-session", new Environment()); try { @@ -223,17 +237,19 @@ public void testQueryExecutionTable() throws Exception { } } - private LocalExecutor createDefaultExecutor() throws Exception { + private LocalExecutor createDefaultExecutor(ClusterClient clusterClient) throws Exception { return new LocalExecutor( EnvironmentFileUtil.parseUnmodified(DEFAULTS_ENVIRONMENT_FILE), Collections.emptyList(), - cluster.getConfiguration()); + clusterClient.getFlinkConfiguration(), + new DummyCustomCommandLine(clusterClient)); } - private LocalExecutor createModifiedExecutor(Map replaceVars) throws Exception { + private LocalExecutor createModifiedExecutor(ClusterClient clusterClient, Map replaceVars) throws Exception { return new LocalExecutor( EnvironmentFileUtil.parseModified(DEFAULTS_ENVIRONMENT_FILE, replaceVars), Collections.emptyList(), - cluster.getConfiguration()); + clusterClient.getFlinkConfiguration(), + new DummyCustomCommandLine(clusterClient)); } } diff --git a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml index 5a598f15ab42bf..8f11d2332532fd 100644 --- a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml +++ b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-defaults.yaml @@ -71,7 +71,6 @@ execution: result-mode: "$VAR_2" deployment: - type: standalone response-timeout: 5000 diff --git a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml index daa1fd167b5f22..d0caf842c89503 100644 --- a/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml +++ b/flink-libraries/flink-sql-client/src/test/resources/test-sql-client-factory.yaml @@ -46,7 +46,6 @@ execution: parallelism: 1 deployment: - type: standalone response-timeout: 5000 From 5803950ef0c8c57534b11de459b92d01da4d3fc0 Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Wed, 21 Mar 2018 16:21:46 +0100 Subject: [PATCH 0257/2294] [FLINK-8972] [e2eTests] Add DataSetAllroundTestProgram and e2e test script. --- .../flink-dataset-allround-test/pom.xml | 104 +++++++ .../tests/DataSetAllroundTestProgram.java | 285 ++++++++++++++++++ flink-end-to-end-tests/pom.xml | 1 + flink-end-to-end-tests/run-nightly-tests.sh | 8 + .../test-scripts/test_batch_allround.sh | 36 +++ 5 files changed, 434 insertions(+) create mode 100644 flink-end-to-end-tests/flink-dataset-allround-test/pom.xml create mode 100644 flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java create mode 100755 flink-end-to-end-tests/test-scripts/test_batch_allround.sh diff --git a/flink-end-to-end-tests/flink-dataset-allround-test/pom.xml b/flink-end-to-end-tests/flink-dataset-allround-test/pom.xml new file mode 100644 index 00000000000000..b701dfdeeb8aa8 --- /dev/null +++ b/flink-end-to-end-tests/flink-dataset-allround-test/pom.xml @@ -0,0 +1,104 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-end-to-end-tests + 1.6-SNAPSHOT + .. + + + flink-dataset-allround-test + flink-dataset-allround-test + jar + + + + org.apache.flink + flink-core + ${project.version} + + + org.apache.flink + flink-java + ${project.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + DataSetAllroundTestProgram + package + + jar + + + DataSetAllroundTestProgram + + + + org.apache.flink.batch.tests.DataSetAllroundTestProgram + + + + + org/apache/flink/batch/tests/DataSetAllroundTestProgram.class + org/apache/flink/batch/tests/DataSetAllroundTestProgram$*.class + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + rename + package + + run + + + + + + + + + + + + + diff --git a/flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java b/flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java new file mode 100644 index 00000000000000..039753545b59ef --- /dev/null +++ b/flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.batch.tests; + +import org.apache.flink.api.common.functions.CoGroupFunction; +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.common.functions.GroupReduceFunction; +import org.apache.flink.api.common.io.DefaultInputSplitAssigner; +import org.apache.flink.api.common.io.InputFormat; +import org.apache.flink.api.common.io.statistics.BaseStatistics; +import org.apache.flink.api.common.operators.Order; +import org.apache.flink.api.common.operators.base.JoinOperatorBase; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.DataSet; +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.api.java.operators.IterativeDataSet; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.tuple.Tuple4; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.io.GenericInputSplit; +import org.apache.flink.core.io.InputSplitAssigner; +import org.apache.flink.util.Preconditions; + +import java.io.IOException; + +/** + * Program to test a large chunk of DataSet API operators and primitives: + *

      + *
    • Map, FlatMap, Filter
      • + *
    • GroupReduce, Reduce
      • + *
    • Join
      • + *
    • CoGroup
      • + *
    • BulkIteration
      • + *
    • Different key definitions (position, name, KeySelector)
      • + *
    + * + *

    Program parameters: + *

      + *
    • loadFactor (int): controls generated data volume. Does not affect result.
      • + *
    • outputPath (String): path to write the result
      • + *
    + */ +public class DataSetAllroundTestProgram { + + public static void main(String[] args) throws Exception { + + // get parameters + ParameterTool params = ParameterTool.fromArgs(args); + int loadFactor = Integer.parseInt(params.getRequired("loadFactor")); + String outputPath = params.getRequired("outputPath"); + + ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + + int numKeys = loadFactor * 128 * 1024; + DataSet> x1Keys = env.createInput(new Generator(numKeys, 1)).setParallelism(4); + DataSet> x2Keys = env.createInput(new Generator(numKeys * 32, 2)).setParallelism(4); + DataSet> x8Keys = env.createInput(new Generator(numKeys, 8)).setParallelism(4); + + DataSet> joined = x2Keys + // shift keys (check for correct handling of key positions) + .map(x -> Tuple4.of("0-0", 0L, 1, x.f0)) + .returns(Types.TUPLE(Types.STRING, Types.LONG, Types.INT, Types.STRING)) + // join datasets on non-unique fields (m-n join) + // Result: (key, 1) 16 * #keys records, all keys are preserved + .join(x8Keys).where(3).equalTo(0).with((l, r) -> Tuple2.of(l.f3, 1)) + .returns(Types.TUPLE(Types.STRING, Types.INT)) + // key definition with key selector function + .groupBy( + new KeySelector, String>() { + @Override + public String getKey(Tuple2 value) throws Exception { + return value.f0; + } + } + ) + // reduce + // Result: (key, cnt), #keys records with unique keys, cnt = 16 + .reduce((value1, value2) -> Tuple2.of(value1.f0, value1.f1 + value2.f1)); + + // co-group two datasets on their primary keys. + // we filter both inputs such that only 6.25% of the keys overlap. + // result: (key, cnt), #keys records with unique keys, cnt = (6.25%: 2, 93.75%: 1) + DataSet> coGrouped = x1Keys + .filter(x -> x.f1 > 59) + .coGroup(x1Keys.filter(x -> x.f1 < 68)).where("f0").equalTo("f0").with( + (CoGroupFunction, Tuple2, Tuple2>) + (l, r, out) -> { + int cnt = 0; + String key = ""; + for (Tuple2 t : l) { + cnt++; + key = t.f0; + } + for (Tuple2 t : r) { + cnt++; + key = t.f0; + } + out.collect(Tuple2.of(key, cnt)); + } + ) + .returns(Types.TUPLE(Types.STRING, Types.INT)); + + // join datasets on keys (1-1 join) and replicate by 16 (previously computed count) + // result: (key, cnt), 16 * #keys records, all keys preserved, cnt = (6.25%: 2, 93.75%: 1) + DataSet> joined2 = joined.join(coGrouped, JoinOperatorBase.JoinHint.REPARTITION_SORT_MERGE) + .where(0).equalTo("f0") + .flatMap( + (FlatMapFunction, Tuple2>, Tuple2>) + (p, out) -> { + for (int i = 0; i < p.f0.f1; i++) { + out.collect(Tuple2.of(p.f0.f0, p.f1.f1)); + } + } + ) + .returns(Types.TUPLE(Types.STRING, Types.INT)); + + // iteration. double the count field until all counts are at 32 or more + // result: (key, cnt), 16 * #keys records, all keys preserved, cnt = (6.25%: 64, 93.75%: 32) + IterativeDataSet> initial = joined2.iterate(16); + DataSet> iteration = initial + .map(x -> Tuple2.of(x.f0, x.f1 * 2)) + .returns(Types.TUPLE(Types.STRING, Types.INT)); + DataSet termination = iteration + // stop iteration if all values are larger/equal 32 + .flatMap( + (FlatMapFunction, Boolean>) + (x, out) -> { + if (x.f1 < 32) { + out.collect(false); + } + } + ) + .returns(Types.BOOLEAN); + DataSet result = initial.closeWith(iteration, termination) + // group on the count field and count records + // result: two records: (32, cnt1) and (64, cnt2) where cnt1 = x * 15/16, cnt2 = x * 1/16 + .groupBy(1) + .reduceGroup( + (GroupReduceFunction, Tuple2>) + (g, out) -> { + int key = 0; + int cnt = 0; + for (Tuple2 r : g) { + key = r.f1; + cnt++; + } + out.collect(Tuple2.of(key, cnt)); + } + ) + .returns(Types.TUPLE(Types.INT, Types.INT)) + // normalize result by load factor + // result: two records: (32: 15360) and (64, 1024). (x = 16384) + .map(x -> Tuple2.of(x.f0, x.f1 / (loadFactor * 128))) + .returns(Types.TUPLE(Types.INT, Types.INT)); + + // sort and emit result + result + .sortPartition(0, Order.ASCENDING).setParallelism(1) + .writeAsText(outputPath, FileSystem.WriteMode.OVERWRITE).setParallelism(1); + + env.execute(); + + } + + /** + * InputFormat that generates a deterministic DataSet of Tuple2(String, Integer) + *
      + *
    • String: key, can be repeated.
      • + *
    • Integer: uniformly distributed int between 0 and 127
      • + *
    + */ + public static class Generator implements InputFormat, GenericInputSplit> { + + // total number of records + private final long numRecords; + // total number of keys + private final long numKeys; + + // records emitted per partition + private long recordsPerPartition; + // number of keys per partition + private long keysPerPartition; + + // number of currently emitted records + private long recordCnt; + + // id of current partition + private int partitionId; + // total number of partitions + private int numPartitions; + + public Generator(long numKeys, int recordsPerKey) { + this.numKeys = numKeys; + this.numRecords = numKeys * recordsPerKey; + } + + @Override + public void configure(Configuration parameters) { } + + @Override + public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException { + return null; + } + + @Override + public GenericInputSplit[] createInputSplits(int minNumSplits) throws IOException { + + GenericInputSplit[] splits = new GenericInputSplit[minNumSplits]; + for (int i = 0; i < minNumSplits; i++) { + splits[i] = new GenericInputSplit(i, minNumSplits); + } + return splits; + } + + @Override + public InputSplitAssigner getInputSplitAssigner(GenericInputSplit[] inputSplits) { + return new DefaultInputSplitAssigner(inputSplits); + } + + @Override + public void open(GenericInputSplit split) throws IOException { + this.partitionId = split.getSplitNumber(); + this.numPartitions = split.getTotalNumberOfSplits(); + + // ensure even distribution of records and keys + Preconditions.checkArgument( + numRecords % numPartitions == 0, + "Records cannot be evenly distributed among partitions"); + Preconditions.checkArgument( + numKeys % numPartitions == 0, + "Keys cannot be evenly distributed among partitions"); + + this.recordsPerPartition = numRecords / numPartitions; + this.keysPerPartition = numKeys / numPartitions; + + this.recordCnt = 0; + } + + @Override + public boolean reachedEnd() throws IOException { + return this.recordCnt >= this.recordsPerPartition; + } + + @Override + public Tuple2 nextRecord(Tuple2 reuse) throws IOException { + + // build key from partition id and count per partition + String key = String.format( + "%d-%d", + this.partitionId, + this.recordCnt % this.keysPerPartition); + // 128 values to filter on + int filterVal = (int) this.recordCnt % 128; + + this.recordCnt++; + + reuse.f0 = key; + reuse.f1 = filterVal; + return reuse; + } + + @Override + public void close() throws IOException { } + } + +} diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml index 32a6e78ae50c3e..f8913d448d8761 100644 --- a/flink-end-to-end-tests/pom.xml +++ b/flink-end-to-end-tests/pom.xml @@ -36,6 +36,7 @@ under the License. flink-parent-child-classloading-test + flink-dataset-allround-test diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh index 8ee526bc23ed0e..71224e09f1b7f7 100755 --- a/flink-end-to-end-tests/run-nightly-tests.sh +++ b/flink-end-to-end-tests/run-nightly-tests.sh @@ -47,5 +47,13 @@ EXIT_CODE=0 # EXIT_CODE=$? # fi +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running DataSet allround nightly end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_batch_allround.sh + EXIT_CODE=$? +fi + # Exit code for Travis build success/failure exit $EXIT_CODE diff --git a/flink-end-to-end-tests/test-scripts/test_batch_allround.sh b/flink-end-to-end-tests/test-scripts/test_batch_allround.sh new file mode 100755 index 00000000000000..1e05c7f0446484 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_batch_allround.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh + +TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/flink-dataset-allround-test/target/DataSetAllroundTestProgram.jar + + echo "Run DataSet-Allround-Test Program" + +start_cluster +$FLINK_DIR/bin/taskmanager.sh start +$FLINK_DIR/bin/taskmanager.sh start +$FLINK_DIR/bin/taskmanager.sh start + +$FLINK_DIR/bin/flink run -p 4 $TEST_PROGRAM_JAR --loadFactor 2 --outputPath $TEST_DATA_DIR/out/dataset_allround + +stop_cluster +$FLINK_DIR/bin/taskmanager.sh stop-all + +check_result_hash "DataSet-Allround-Test" $TEST_DATA_DIR/out/dataset_allround "d3cf2aeaa9320c772304cba42649eb47" From 5e52bd869827121bfae319d3f54c8404233663ad Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 26 Mar 2018 15:55:04 +0200 Subject: [PATCH 0258/2294] [FLINK-8972] [e2eTests] Remove IDE warnings and simplify pom.xml This closes #5752. --- .../flink-dataset-allround-test/pom.xml | 23 +------------------ .../tests/DataSetAllroundTestProgram.java | 20 ++++++++-------- .../test-scripts/test_batch_allround.sh | 2 +- 3 files changed, 11 insertions(+), 34 deletions(-) diff --git a/flink-end-to-end-tests/flink-dataset-allround-test/pom.xml b/flink-end-to-end-tests/flink-dataset-allround-test/pom.xml index b701dfdeeb8aa8..f1e422483d40b4 100644 --- a/flink-end-to-end-tests/flink-dataset-allround-test/pom.xml +++ b/flink-end-to-end-tests/flink-dataset-allround-test/pom.xml @@ -61,7 +61,7 @@ under the License. jar - DataSetAllroundTestProgram + DataSetAllroundTestProgram @@ -77,27 +77,6 @@ under the License. - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - rename - package - - run - - - - - - - - -     diff --git a/flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java b/flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java index 039753545b59ef..4abbb3548b0eb6 100644 --- a/flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java +++ b/flink-end-to-end-tests/flink-dataset-allround-test/src/main/java/org/apache/flink/batch/tests/DataSetAllroundTestProgram.java @@ -40,8 +40,6 @@ import org.apache.flink.core.io.InputSplitAssigner; import org.apache.flink.util.Preconditions; -import java.io.IOException; - /** * Program to test a large chunk of DataSet API operators and primitives: *
      @@ -61,6 +59,7 @@ */ public class DataSetAllroundTestProgram { + @SuppressWarnings("Convert2Lambda") public static void main(String[] args) throws Exception { // get parameters @@ -87,7 +86,7 @@ public static void main(String[] args) throws Exception { .groupBy( new KeySelector, String>() { @Override - public String getKey(Tuple2 value) throws Exception { + public String getKey(Tuple2 value) { return value.f0; } } @@ -150,7 +149,7 @@ public String getKey(Tuple2 value) throws Exception { } ) .returns(Types.BOOLEAN); - DataSet result = initial.closeWith(iteration, termination) + DataSet> result = initial.closeWith(iteration, termination) // group on the count field and count records // result: two records: (32, cnt1) and (64, cnt2) where cnt1 = x * 15/16, cnt2 = x * 1/16 .groupBy(1) @@ -178,7 +177,6 @@ public String getKey(Tuple2 value) throws Exception { .writeAsText(outputPath, FileSystem.WriteMode.OVERWRITE).setParallelism(1); env.execute(); - } /** @@ -217,12 +215,12 @@ public Generator(long numKeys, int recordsPerKey) { public void configure(Configuration parameters) { } @Override - public BaseStatistics getStatistics(BaseStatistics cachedStatistics) throws IOException { + public BaseStatistics getStatistics(BaseStatistics cachedStatistics) { return null; } @Override - public GenericInputSplit[] createInputSplits(int minNumSplits) throws IOException { + public GenericInputSplit[] createInputSplits(int minNumSplits) { GenericInputSplit[] splits = new GenericInputSplit[minNumSplits]; for (int i = 0; i < minNumSplits; i++) { @@ -237,7 +235,7 @@ public InputSplitAssigner getInputSplitAssigner(GenericInputSplit[] inputSplits) } @Override - public void open(GenericInputSplit split) throws IOException { + public void open(GenericInputSplit split) { this.partitionId = split.getSplitNumber(); this.numPartitions = split.getTotalNumberOfSplits(); @@ -256,12 +254,12 @@ public void open(GenericInputSplit split) throws IOException { } @Override - public boolean reachedEnd() throws IOException { + public boolean reachedEnd() { return this.recordCnt >= this.recordsPerPartition; } @Override - public Tuple2 nextRecord(Tuple2 reuse) throws IOException { + public Tuple2 nextRecord(Tuple2 reuse) { // build key from partition id and count per partition String key = String.format( @@ -279,7 +277,7 @@ public Tuple2 nextRecord(Tuple2 reuse) throws } @Override - public void close() throws IOException { } + public void close() { } } } diff --git a/flink-end-to-end-tests/test-scripts/test_batch_allround.sh b/flink-end-to-end-tests/test-scripts/test_batch_allround.sh index 1e05c7f0446484..acdc37e685fda2 100755 --- a/flink-end-to-end-tests/test-scripts/test_batch_allround.sh +++ b/flink-end-to-end-tests/test-scripts/test_batch_allround.sh @@ -21,7 +21,7 @@ source "$(dirname "$0")"/common.sh TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/flink-dataset-allround-test/target/DataSetAllroundTestProgram.jar - echo "Run DataSet-Allround-Test Program" +echo "Run DataSet-Allround-Test Program" start_cluster $FLINK_DIR/bin/taskmanager.sh start From 1277e920d6b5123e2b67eb7e4688ad1c30ef7ab0 Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Wed, 14 Mar 2018 18:11:34 +0100 Subject: [PATCH 0259/2294] [FLINK-8833] [sql-client] Create a SQL Client JSON format fat-jar This closes #5700. --- flink-formats/flink-json/pom.xml | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml index 3a80b0eaae9fe2..2c8db2da7fa461 100644 --- a/flink-formats/flink-json/pom.xml +++ b/flink-formats/flink-json/pom.xml @@ -63,6 +63,13 @@ under the License. + + org.apache.flink + flink-test-utils-junit + ${project.version} + test + + org.apache.flink @@ -79,4 +86,30 @@ under the License. test + + + + + release + + + + org.apache.maven.plugins + maven-jar-plugin + + + package + + jar + + + sql-jar + + + + + + + + From 86731eab2c2fafbbfc1b8d05694337e6504e925a Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Mon, 26 Mar 2018 19:03:28 +0200 Subject: [PATCH 0260/2294] [hotfix] [sql-client] Add flink-sql-client as 'provided' to flink-dist --- flink-dist/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index d8477fe2ec6a9b..44ce9df6e63a43 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -308,6 +308,13 @@ under the License. provided + + org.apache.flink + flink-sql-client + ${project.version} + provided + + org.apache.flink flink-s3-fs-hadoop From 8ea02ec2af5491a172fbe795d5f39b5835b7a83a Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 23 Mar 2018 11:29:50 +0100 Subject: [PATCH 0261/2294] [FLINK-8789] Throw IllegalStateException when calling Task#stopExecution on not running Task Before we threw an UnsupportedOperationException when calling Task#stopExecution on a Task where we did not set the invokable yet. This can be a bit misleading and, thus, this commit throws an IllegalStateException with an respective message. This closes #5753. --- .../flink/runtime/taskmanager/Task.java | 45 +++++++++++-------- .../runtime/taskmanager/TaskManagerTest.java | 4 ++ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java index ccb850e85bc127..d99472bb6b6ff4 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java @@ -628,8 +628,7 @@ else if (current == ExecutionState.CANCELING) { // next, kick off the background copying of files for the distributed cache try { for (Map.Entry entry : - DistributedCache.readFileInfoFromConfig(jobConfiguration)) - { + DistributedCache.readFileInfoFromConfig(jobConfiguration)) { LOG.info("Obtaining local cache file for '{}'.", entry.getKey()); Future cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId); distributedCacheEntries.put(entry.getKey(), cp); @@ -943,26 +942,34 @@ private boolean transitionState(ExecutionState currentState, ExecutionState newS * This method never blocks. *

      * - * @throws UnsupportedOperationException - * if the {@link AbstractInvokable} does not implement {@link StoppableTask} + * @throws UnsupportedOperationException if the {@link AbstractInvokable} does not implement {@link StoppableTask} + * @throws IllegalStateException if the {@link Task} is not yet running */ - public void stopExecution() throws UnsupportedOperationException { - LOG.info("Attempting to stop task {} ({}).", taskNameWithSubtask, executionId); - if (invokable instanceof StoppableTask) { - Runnable runnable = new Runnable() { - @Override - public void run() { - try { - ((StoppableTask) invokable).stop(); - } catch (RuntimeException e) { - LOG.error("Stopping task {} ({}) failed.", taskNameWithSubtask, executionId, e); - taskManagerActions.failTask(executionId, e); + public void stopExecution() { + if (invokable != null) { + LOG.info("Attempting to stop task {} ({}).", taskNameWithSubtask, executionId); + if (invokable instanceof StoppableTask) { + Runnable runnable = new Runnable() { + @Override + public void run() { + try { + ((StoppableTask) invokable).stop(); + } catch (RuntimeException e) { + LOG.error("Stopping task {} ({}) failed.", taskNameWithSubtask, executionId, e); + taskManagerActions.failTask(executionId, e); + } } - } - }; - executeAsyncCallRunnable(runnable, String.format("Stopping source task %s (%s).", taskNameWithSubtask, executionId)); + }; + executeAsyncCallRunnable(runnable, String.format("Stopping source task %s (%s).", taskNameWithSubtask, executionId)); + } else { + throw new UnsupportedOperationException(String.format("Stopping not supported by task %s (%s).", taskNameWithSubtask, executionId)); + } } else { - throw new UnsupportedOperationException(String.format("Stopping not supported by task %s (%s).", taskNameWithSubtask, executionId)); + throw new IllegalStateException( + String.format( + "Cannot stop task %s (%s) because it is not yet running.", + taskNameWithSubtask, + executionId)); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java index 93ec3adcb2a42c..977ab9ed4fb145 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java @@ -1734,6 +1734,10 @@ public void testStopTaskFailure() throws Exception { Await.result(submitResponse, timeout); + final Future taskRunning = taskManager.ask(new TestingTaskManagerMessages.NotifyWhenTaskIsRunning(executionAttemptId), timeout); + + Await.result(taskRunning, timeout); + Future stopResponse = taskManager.ask(new StopTask(executionAttemptId), timeout); try { From bee630296236fe41510ada24bcb962cc147a78bd Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 23 Mar 2018 12:01:09 +0100 Subject: [PATCH 0262/2294] [FLINK-8901] [yarn] Set proper Yarn application name When deploying a session cluster, Flink will register under "Flink session cluster". When deploying a per-job cluster, Flink will register under "Flink per-job cluster". This closes #5754. --- .../yarn/AbstractYarnClusterDescriptor.java | 17 +++++++---------- .../flink/yarn/Flip6YarnClusterDescriptor.java | 1 + 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java index caf7a7614c7f21..c80818a2e4b518 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java @@ -394,6 +394,7 @@ public ClusterClient deploySessionCluster(ClusterSpecification cl try { return deployInternal( clusterSpecification, + "Flink session cluster", getYarnSessionClusterEntrypoint(), null, false); @@ -437,12 +438,14 @@ private void validateClusterSpecification(ClusterSpecification clusterSpecificat * This method will block until the ApplicationMaster/JobManager have been deployed on YARN. * * @param clusterSpecification Initial cluster specification for the Flink cluster to be deployed + * @param applicationName name of the Yarn application to start * @param yarnClusterEntrypoint Class name of the Yarn cluster entry point. * @param jobGraph A job graph which is deployed with the Flink cluster, {@code null} if none * @param detached True if the cluster should be started in detached mode */ protected ClusterClient deployInternal( ClusterSpecification clusterSpecification, + String applicationName, String yarnClusterEntrypoint, @Nullable JobGraph jobGraph, boolean detached) throws Exception { @@ -517,6 +520,7 @@ protected ClusterClient deployInternal( ApplicationReport report = startAppMaster( flinkConfiguration, + applicationName, yarnClusterEntrypoint, jobGraph, yarnClient, @@ -670,6 +674,7 @@ private void checkYarnQueues(YarnClient yarnClient) { public ApplicationReport startAppMaster( Configuration configuration, + String applicationName, String yarnClusterEntrypoint, JobGraph jobGraph, YarnClient yarnClient, @@ -1005,17 +1010,9 @@ public ApplicationReport startAppMaster( capability.setMemory(clusterSpecification.getMasterMemoryMB()); capability.setVirtualCores(1); - String name; - if (customName == null) { - name = "Flink session with " + clusterSpecification.getNumberTaskManagers() + " TaskManagers"; - if (detached) { - name += " (detached)"; - } - } else { - name = customName; - } + final String customApplicationName = customName != null ? customName : applicationName; - appContext.setApplicationName(name); + appContext.setApplicationName(customApplicationName); appContext.setApplicationType("Apache Flink"); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java index 9860363c00e32e..1374ca2c4419d0 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/Flip6YarnClusterDescriptor.java @@ -75,6 +75,7 @@ public ClusterClient deployJobCluster( try { return deployInternal( clusterSpecification, + "Flink per-job cluster", getYarnJobClusterEntrypoint(), jobGraph, detached); From c64eccbe0301a69aa22d0be8ec28c8ac340c20bb Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 27 Mar 2018 08:05:32 +0200 Subject: [PATCH 0263/2294] [hotfix] Remove unused applicationName parameter from FlinkYarnSessionCli#createDescriptor --- .../flink/yarn/cli/FlinkYarnSessionCli.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index 2311e875c2f13a..446377ffab728f 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -265,11 +265,10 @@ public FlinkYarnSessionCli( } private AbstractYarnClusterDescriptor createDescriptor( - Configuration configuration, - YarnConfiguration yarnConfiguration, - String configurationDirectory, - String defaultApplicationName, - CommandLine cmd) { + Configuration configuration, + YarnConfiguration yarnConfiguration, + String configurationDirectory, + CommandLine cmd) { AbstractYarnClusterDescriptor yarnClusterDescriptor = getClusterDescriptor( configuration, @@ -356,11 +355,6 @@ private AbstractYarnClusterDescriptor createDescriptor( if (cmd.hasOption(name.getOpt())) { yarnClusterDescriptor.setName(cmd.getOptionValue(name.getOpt())); - } else { - // set the default application name, if none is specified - if (defaultApplicationName != null) { - yarnClusterDescriptor.setName(defaultApplicationName); - } } if (cmd.hasOption(zookeeperNamespace.getOpt())) { @@ -456,7 +450,6 @@ public AbstractYarnClusterDescriptor createClusterDescriptor(CommandLine command effectiveConfiguration, yarnConfiguration, configurationDirectory, - null, commandLine); } From 19fa504d26544faef667cace0a394fc0f097fb54 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Fri, 23 Mar 2018 16:21:55 +0100 Subject: [PATCH 0264/2294] [FLINK-5411] [flip6] Fix JobLeaderIdService shut down in ResourceManager The JobLeaderIdService was formerly closed at two different locations. Once in the ResourceManager and once in the ResourceManagerRuntimeServices. Since the JobLeaderIdService is a RM specific component. It should also be closed in the scope of the RM. This closes #5757. --- .../runtime/resourcemanager/ResourceManagerRunner.java | 9 +-------- .../resourcemanager/ResourceManagerRuntimeServices.java | 6 ------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java index 9daf96ef48a98a..ff9b4f0789e557 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRunner.java @@ -20,7 +20,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.clusterframework.types.ResourceID; -import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; @@ -106,13 +105,7 @@ public CompletableFuture closeAsync() { synchronized (lock) { resourceManager.shutDown(); - return FutureUtils.runAfterwards( - resourceManager.getTerminationFuture(), - () -> { - synchronized (lock) { - resourceManagerRuntimeServices.shutDown(); - } - }); + return resourceManager.getTerminationFuture(); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java index ed8f1e0cd9689a..7f5af2445930e5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices.java @@ -45,12 +45,6 @@ public JobLeaderIdService getJobLeaderIdService() { return jobLeaderIdService; } - // -------------------- Lifecycle methods ----------------------------------- - - public void shutDown() throws Exception { - jobLeaderIdService.stop(); - } - // -------------------- Static methods -------------------------------------- public static ResourceManagerRuntimeServices fromConfiguration( From bd715c663c226b6f0042a20d768e57433aa97623 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Sun, 25 Mar 2018 19:12:51 +0200 Subject: [PATCH 0265/2294] [FLINK-8940] [flip6] Add support for dispose savepoint Adds an AsynchronousOperationHandler for disposing savepoints. The handler is registered under '/savepoint-disposal' and requires a SavepointDisposalRequest JSON object containing the path to the savepoint to be disposed. The RestClusterClient polls the status registered under '/savepoint-disposal/:triggerId' until the operation has been completed. This closes #5764. --- .../apache/flink/client/cli/CliFrontend.java | 3 +- .../flink/client/program/ClusterClient.java | 4 +- .../client/program/MiniClusterClient.java | 4 +- .../program/rest/RestClusterClient.java | 38 ++++++ .../client/cli/CliFrontendSavepointTest.java | 17 ++- .../client/program/ClusterClientTest.java | 4 +- .../program/rest/RestClusterClientTest.java | 107 +++++++++++++++++ .../flink/runtime/checkpoint/Checkpoints.java | 11 +- .../flink/runtime/dispatcher/Dispatcher.java | 21 ++++ .../runtime/minicluster/MiniCluster.java | 12 ++ .../rest/handler/async/OperationKey.java | 4 +- .../savepoints/SavepointDisposalHandlers.java | 112 ++++++++++++++++++ .../savepoints/SavepointDisposalRequest.java | 49 ++++++++ .../SavepointDisposalStatusHeaders.java | 75 ++++++++++++ ...epointDisposalStatusMessageParameters.java | 46 +++++++ .../SavepointDisposalTriggerHeaders.java | 67 +++++++++++ .../runtime/webmonitor/RestfulGateway.java | 13 ++ .../webmonitor/WebMonitorEndpoint.java | 19 +++ .../runtime/dispatcher/DispatcherTest.java | 63 +++++++++- .../SavepointDisposalRequestTest.java | 47 ++++++++ 20 files changed, 691 insertions(+), 25 deletions(-) create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointDisposalHandlers.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequest.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusHeaders.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusMessageParameters.java create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalTriggerHeaders.java create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequestTest.java diff --git a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java index d636ef7c9615e1..a874891eb25c78 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java +++ b/flink-clients/src/main/java/org/apache/flink/client/cli/CliFrontend.java @@ -47,7 +47,6 @@ import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.runtime.client.JobStatusMessage; -import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.messages.Acknowledge; @@ -706,7 +705,7 @@ private void disposeSavepoint(ClusterClient clusterClient, String savepointPa logAndSysout("Disposing savepoint '" + savepointPath + "'."); - final CompletableFuture disposeFuture = clusterClient.disposeSavepoint(savepointPath, FutureUtils.toTime(clientTimeout)); + final CompletableFuture disposeFuture = clusterClient.disposeSavepoint(savepointPath); logAndSysout("Waiting for response..."); diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java index f50206d1492c43..fbaa5156520a3d 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java @@ -726,14 +726,14 @@ public CompletableFuture triggerSavepoint(JobID jobId, @Nullable String }); } - public CompletableFuture disposeSavepoint(String savepointPath, Time timeout) throws FlinkException { + public CompletableFuture disposeSavepoint(String savepointPath) throws FlinkException { final ActorGateway jobManager = getJobManagerGateway(); Object msg = new JobManagerMessages.DisposeSavepoint(savepointPath); CompletableFuture responseFuture = FutureUtils.toJava( jobManager.ask( msg, - FutureUtils.toFiniteDuration(timeout))); + timeout)); return responseFuture.thenApply( (Object response) -> { diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java index 44f6ef630d26a7..802622ee4ae0c3 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java @@ -123,8 +123,8 @@ public CompletableFuture triggerSavepoint(JobID jobId, @Nullable String } @Override - public CompletableFuture disposeSavepoint(String savepointPath, Time timeout) throws FlinkException { - throw new UnsupportedOperationException("MiniClusterClient does not yet support this operation."); + public CompletableFuture disposeSavepoint(String savepointPath) throws FlinkException { + return guardWithSingleRetry(() -> miniCluster.disposeSavepoint(savepointPath), scheduledExecutor); } @Override diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 2e1ffb02298183..912971456f6680 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -72,6 +72,10 @@ import org.apache.flink.runtime.rest.messages.job.JobSubmitHeaders; import org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody; import org.apache.flink.runtime.rest.messages.job.JobSubmitResponseBody; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalRequest; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusHeaders; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusMessageParameters; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalTriggerHeaders; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointInfo; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusHeaders; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusMessageParameters; @@ -530,6 +534,40 @@ public CompletableFuture rescaleJob(JobID jobId, int newParallelism }); } + @Override + public CompletableFuture disposeSavepoint(String savepointPath, Time timeout) { + final SavepointDisposalRequest savepointDisposalRequest = new SavepointDisposalRequest(savepointPath); + + final CompletableFuture savepointDisposalTriggerFuture = sendRequest( + SavepointDisposalTriggerHeaders.getInstance(), + EmptyMessageParameters.getInstance(), + savepointDisposalRequest); + + final CompletableFuture savepointDisposalFuture = savepointDisposalTriggerFuture.thenCompose( + (TriggerResponse triggerResponse) -> { + final TriggerId triggerId = triggerResponse.getTriggerId(); + final SavepointDisposalStatusHeaders savepointDisposalStatusHeaders = SavepointDisposalStatusHeaders.getInstance(); + final SavepointDisposalStatusMessageParameters savepointDisposalStatusMessageParameters = savepointDisposalStatusHeaders.getUnresolvedMessageParameters(); + savepointDisposalStatusMessageParameters.triggerIdPathParameter.resolve(triggerId); + + return pollResourceAsync( + () -> sendRetryableRequest( + savepointDisposalStatusHeaders, + savepointDisposalStatusMessageParameters, + EmptyRequestBody.getInstance(), + isConnectionProblemException())); + }); + + return savepointDisposalFuture.thenApply( + (AsynchronousOperationInfo asynchronousOperationInfo) -> { + if (asynchronousOperationInfo.getFailureCause() == null) { + return Acknowledge.get(); + } else { + throw new CompletionException(asynchronousOperationInfo.getFailureCause()); + } + }); + } + /** * Creates a {@code CompletableFuture} that polls a {@code AsynchronouslyCreatedResource} until * its {@link AsynchronouslyCreatedResource#queueStatus() QueueStatus} becomes diff --git a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java index 3195a6baf6dc12..75f3b3df18439d 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendSavepointTest.java @@ -19,7 +19,6 @@ package org.apache.flink.client.cli; import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.time.Time; import org.apache.flink.client.cli.util.MockedCliFrontend; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.StandaloneClusterClient; @@ -41,7 +40,7 @@ import java.io.FileOutputStream; import java.io.PrintStream; import java.util.concurrent.CompletableFuture; -import java.util.function.BiFunction; +import java.util.function.Function; import java.util.zip.ZipOutputStream; import static org.junit.Assert.assertEquals; @@ -196,7 +195,7 @@ public void testDisposeSavepointSuccess() throws Exception { String savepointPath = "expectedSavepointPath"; ClusterClient clusterClient = new DisposeSavepointClusterClient( - (String path, Time timeout) -> CompletableFuture.completedFuture(Acknowledge.get()), getConfiguration()); + (String path) -> CompletableFuture.completedFuture(Acknowledge.get()), getConfiguration()); try { @@ -225,7 +224,7 @@ public void testDisposeWithJar() throws Exception { final CompletableFuture disposeSavepointFuture = new CompletableFuture<>(); final DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient( - (String savepointPath, Time timeout) -> { + (String savepointPath) -> { disposeSavepointFuture.complete(savepointPath); return CompletableFuture.completedFuture(Acknowledge.get()); }, getConfiguration()); @@ -260,7 +259,7 @@ public void testDisposeSavepointFailure() throws Exception { Exception testException = new Exception("expectedTestException"); - DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient((String path, Time timeout) -> FutureUtils.completedExceptionally(testException), getConfiguration()); + DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient((String path) -> FutureUtils.completedExceptionally(testException), getConfiguration()); try { CliFrontend frontend = new MockedCliFrontend(clusterClient); @@ -285,17 +284,17 @@ public void testDisposeSavepointFailure() throws Exception { private static final class DisposeSavepointClusterClient extends StandaloneClusterClient { - private final BiFunction> disposeSavepointFunction; + private final Function> disposeSavepointFunction; - DisposeSavepointClusterClient(BiFunction> disposeSavepointFunction, Configuration configuration) { + DisposeSavepointClusterClient(Function> disposeSavepointFunction, Configuration configuration) { super(configuration, new TestingHighAvailabilityServices(), false); this.disposeSavepointFunction = Preconditions.checkNotNull(disposeSavepointFunction); } @Override - public CompletableFuture disposeSavepoint(String savepointPath, Time timeout) { - return disposeSavepointFunction.apply(savepointPath, timeout); + public CompletableFuture disposeSavepoint(String savepointPath) { + return disposeSavepointFunction.apply(savepointPath); } } diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java index f30fd192e8388a..07b3821ce2afcc 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/ClusterClientTest.java @@ -181,7 +181,7 @@ public void testDisposeSavepointUnknownResponse() throws Exception { final TestClusterClient clusterClient = new TestClusterClient(configuration, jobManagerGateway); - CompletableFuture acknowledgeCompletableFuture = clusterClient.disposeSavepoint(savepointPath, timeout); + CompletableFuture acknowledgeCompletableFuture = clusterClient.disposeSavepoint(savepointPath); try { acknowledgeCompletableFuture.get(); @@ -203,7 +203,7 @@ public void testDisposeClassNotFoundException() throws Exception { final TestClusterClient clusterClient = new TestClusterClient(configuration, jobManagerGateway); - CompletableFuture acknowledgeCompletableFuture = clusterClient.disposeSavepoint(savepointPath, timeout); + CompletableFuture acknowledgeCompletableFuture = clusterClient.disposeSavepoint(savepointPath); try { acknowledgeCompletableFuture.get(); diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index 77a4113f59ed43..926da924c5cd31 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -35,6 +35,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobmaster.JobResult; +import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.messages.webmonitor.JobDetails; import org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails; import org.apache.flink.runtime.rest.RestClient; @@ -45,6 +46,7 @@ import org.apache.flink.runtime.rest.handler.HandlerRequest; import org.apache.flink.runtime.rest.handler.RestHandlerException; import org.apache.flink.runtime.rest.handler.RestHandlerSpecification; +import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationInfo; import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationResult; import org.apache.flink.runtime.rest.handler.async.TriggerResponse; import org.apache.flink.runtime.rest.messages.AccumulatorsIncludeSerializedValueQueryParameter; @@ -72,6 +74,10 @@ import org.apache.flink.runtime.rest.messages.job.JobSubmitHeaders; import org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody; import org.apache.flink.runtime.rest.messages.job.JobSubmitResponseBody; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalRequest; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusHeaders; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusMessageParameters; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalTriggerHeaders; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointInfo; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusHeaders; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusMessageParameters; @@ -84,7 +90,9 @@ import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.testutils.category.Flip6; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FlinkException; import org.apache.flink.util.OptionalFailure; +import org.apache.flink.util.Preconditions; import org.apache.flink.util.SerializedThrowable; import org.apache.flink.util.SerializedValue; import org.apache.flink.util.TestLogger; @@ -103,6 +111,7 @@ import javax.annotation.Nonnull; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -112,6 +121,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -536,6 +546,103 @@ protected CompletableFuture> handleRe } } + @Test + public void testDisposeSavepoint() throws Exception { + final String savepointPath = "foobar"; + final String exceptionMessage = "Test exception."; + final FlinkException testException = new FlinkException(exceptionMessage); + + final TestSavepointDisposalHandlers testSavepointDisposalHandlers = new TestSavepointDisposalHandlers(savepointPath); + final TestSavepointDisposalHandlers.TestSavepointDisposalTriggerHandler testSavepointDisposalTriggerHandler = testSavepointDisposalHandlers.new TestSavepointDisposalTriggerHandler(); + final TestSavepointDisposalHandlers.TestSavepointDisposalStatusHandler testSavepointDisposalStatusHandler = testSavepointDisposalHandlers.new TestSavepointDisposalStatusHandler( + OptionalFailure.of(AsynchronousOperationInfo.complete()), + OptionalFailure.of(AsynchronousOperationInfo.completeExceptional(new SerializedThrowable(testException))), + OptionalFailure.ofFailure(testException)); + + try (TestRestServerEndpoint ignored = createRestServerEndpoint( + testSavepointDisposalStatusHandler, + testSavepointDisposalTriggerHandler)) { + { + final CompletableFuture disposeSavepointFuture = restClusterClient.disposeSavepoint(savepointPath); + assertThat(disposeSavepointFuture.get(), is(Acknowledge.get())); + } + + { + final CompletableFuture disposeSavepointFuture = restClusterClient.disposeSavepoint(savepointPath); + + try { + disposeSavepointFuture.get(); + fail("Expected an exception"); + } catch (ExecutionException ee) { + assertThat(ExceptionUtils.findThrowableWithMessage(ee, exceptionMessage).isPresent(), is(true)); + } + } + + { + try { + restClusterClient.disposeSavepoint(savepointPath).get(); + fail("Expected an exception."); + } catch (ExecutionException ee) { + assertThat(ExceptionUtils.findThrowable(ee, RestClientException.class).isPresent(), is(true)); + } + } + } + } + + private class TestSavepointDisposalHandlers { + + private final TriggerId triggerId = new TriggerId(); + + private final String savepointPath; + + private TestSavepointDisposalHandlers(String savepointPath) { + this.savepointPath = Preconditions.checkNotNull(savepointPath); + } + + private class TestSavepointDisposalTriggerHandler extends TestHandler { + private TestSavepointDisposalTriggerHandler() { + super(SavepointDisposalTriggerHeaders.getInstance()); + } + + @Override + protected CompletableFuture handleRequest(@Nonnull HandlerRequest request, @Nonnull DispatcherGateway gateway) { + assertThat(request.getRequestBody().getSavepointPath(), is(savepointPath)); + return CompletableFuture.completedFuture(new TriggerResponse(triggerId)); + } + } + + private class TestSavepointDisposalStatusHandler extends TestHandler, SavepointDisposalStatusMessageParameters> { + + private final Queue> responses; + + private TestSavepointDisposalStatusHandler(OptionalFailure... responses) { + super(SavepointDisposalStatusHeaders.getInstance()); + this.responses = new ArrayDeque<>(Arrays.asList(responses)); + } + + @Override + protected CompletableFuture> handleRequest(@Nonnull HandlerRequest request, @Nonnull DispatcherGateway gateway) throws RestHandlerException { + final TriggerId actualTriggerId = request.getPathParameter(TriggerIdPathParameter.class); + + if (actualTriggerId.equals(triggerId)) { + final OptionalFailure nextResponse = responses.poll(); + + if (nextResponse != null) { + if (nextResponse.isFailure()) { + throw new RestHandlerException("Failure", HttpResponseStatus.BAD_REQUEST, nextResponse.getFailureCause()); + } else { + return CompletableFuture.completedFuture(AsynchronousOperationResult.completed(nextResponse.getUnchecked())); + } + } else { + throw new AssertionError(); + } + } else { + throw new AssertionError(); + } + } + } + } + @Test public void testListJobs() throws Exception { try (TestRestServerEndpoint ignored = createRestServerEndpoint(new TestListJobsHandler())) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/Checkpoints.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/Checkpoints.java index 72b7c53ab95896..60fdc17c819de0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/Checkpoints.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/Checkpoints.java @@ -38,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.DataInputStream; @@ -292,6 +293,13 @@ public static void disposeSavepoint( checkNotNull(configuration, "configuration"); checkNotNull(classLoader, "classLoader"); + StateBackend backend = loadStateBackend(configuration, classLoader, logger); + + disposeSavepoint(pointer, backend, classLoader); + } + + @Nonnull + public static StateBackend loadStateBackend(Configuration configuration, ClassLoader classLoader, @Nullable Logger logger) { if (logger != null) { logger.info("Attempting to load configured state backend for savepoint disposal"); } @@ -318,8 +326,7 @@ public static void disposeSavepoint( // FileSystem-based for metadata backend = new MemoryStateBackend(); } - - disposeSavepoint(pointer, backend, classLoader); + return backend; } // ------------------------------------------------------------------------ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java index 68b40468a29b81..008d4dcdfe9548 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java @@ -24,6 +24,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobServer; +import org.apache.flink.runtime.checkpoint.Checkpoints; import org.apache.flink.runtime.client.JobSubmissionException; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.concurrent.FutureUtils; @@ -75,6 +76,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.stream.Collectors; /** @@ -319,6 +321,25 @@ public CompletableFuture> listJobs(Time timeout) { Collections.unmodifiableSet(new HashSet<>(jobManagerRunners.keySet()))); } + @Override + public CompletableFuture disposeSavepoint(String savepointPath, Time timeout) { + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + return CompletableFuture.supplyAsync( + () -> { + log.info("Disposing savepoint {}.", savepointPath); + + try { + Checkpoints.disposeSavepoint(savepointPath, configuration, classLoader, log); + } catch (IOException | FlinkException e) { + throw new CompletionException(new FlinkException(String.format("Could not dispose savepoint %s.", savepointPath), e)); + } + + return Acknowledge.get(); + }, + jobManagerSharedServices.getScheduledExecutorService()); + } + @Override public CompletableFuture cancelJob(JobID jobId, Time timeout) { JobManagerRunner jobManagerRunner = jobManagerRunners.get(jobId); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java index 0da6f333b6830e..66770c52bdd77c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/minicluster/MiniCluster.java @@ -534,6 +534,18 @@ public CompletableFuture triggerSavepoint(JobID jobId, String targetDire } } + public CompletableFuture disposeSavepoint(String savepointPath) { + try { + return getDispatcherGateway().disposeSavepoint(savepointPath, rpcTimeout); + } catch (LeaderRetrievalException | InterruptedException e) { + ExceptionUtils.checkInterrupted(e); + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Could not dispose savepoint %s.", savepointPath), + e)); + } + } + public CompletableFuture getExecutionGraph(JobID jobId) { try { return getDispatcherGateway().requestJob(jobId, rpcTimeout); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/async/OperationKey.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/async/OperationKey.java index a601e56a8831bc..2f6e4bd8513197 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/async/OperationKey.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/async/OperationKey.java @@ -27,11 +27,11 @@ * Any operation key for the {@link AbstractAsynchronousOperationHandlers} must extend this class. * It is used to store the trigger id. */ -public abstract class OperationKey { +public class OperationKey { private final TriggerId triggerId; - protected OperationKey(TriggerId triggerId) { + public OperationKey(TriggerId triggerId) { this.triggerId = Preconditions.checkNotNull(triggerId); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointDisposalHandlers.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointDisposalHandlers.java new file mode 100644 index 00000000000000..3cf5f59ca694b3 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointDisposalHandlers.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.handler.job.savepoints; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.messages.Acknowledge; +import org.apache.flink.runtime.rest.handler.HandlerRequest; +import org.apache.flink.runtime.rest.handler.async.AbstractAsynchronousOperationHandlers; +import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationInfo; +import org.apache.flink.runtime.rest.handler.async.OperationKey; +import org.apache.flink.runtime.rest.messages.EmptyMessageParameters; +import org.apache.flink.runtime.rest.messages.EmptyRequestBody; +import org.apache.flink.runtime.rest.messages.TriggerId; +import org.apache.flink.runtime.rest.messages.TriggerIdPathParameter; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalRequest; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusHeaders; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusMessageParameters; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalTriggerHeaders; +import org.apache.flink.runtime.rpc.RpcUtils; +import org.apache.flink.runtime.webmonitor.RestfulGateway; +import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; +import org.apache.flink.util.SerializedThrowable; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * Handlers to trigger the disposal of a savepoint. + */ +public class SavepointDisposalHandlers extends AbstractAsynchronousOperationHandlers { + + /** + * {@link TriggerHandler} implementation for the savepoint disposal operation. + */ + public class SavepointDisposalTriggerHandler extends TriggerHandler { + + public SavepointDisposalTriggerHandler( + CompletableFuture localRestAddress, + GatewayRetriever leaderRetriever, + Time timeout, + Map responseHeaders) { + super( + localRestAddress, + leaderRetriever, + timeout, + responseHeaders, + SavepointDisposalTriggerHeaders.getInstance()); + } + + @Override + protected CompletableFuture triggerOperation(HandlerRequest request, RestfulGateway gateway) { + final String savepointPath = request.getRequestBody().getSavepointPath(); + return gateway.disposeSavepoint(savepointPath, RpcUtils.INF_TIMEOUT); + } + + @Override + protected OperationKey createOperationKey(HandlerRequest request) { + return new OperationKey(new TriggerId()); + } + } + + /** + * {@link StatusHandler} implementation for the savepoint disposal operation. + */ + public class SavepointDisposalStatusHandler extends StatusHandler { + + public SavepointDisposalStatusHandler( + CompletableFuture localRestAddress, + GatewayRetriever leaderRetriever, + Time timeout, + Map responseHeaders) { + super( + localRestAddress, + leaderRetriever, + timeout, + responseHeaders, + SavepointDisposalStatusHeaders.getInstance()); + } + + @Override + protected OperationKey getOperationKey(HandlerRequest request) { + final TriggerId triggerId = request.getPathParameter(TriggerIdPathParameter.class); + return new OperationKey(triggerId); + } + + @Override + protected AsynchronousOperationInfo exceptionalOperationResultResponse(Throwable throwable) { + return AsynchronousOperationInfo.completeExceptional(new SerializedThrowable(throwable)); + } + + @Override + protected AsynchronousOperationInfo operationResultResponse(Acknowledge operationResult) { + return AsynchronousOperationInfo.complete(); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequest.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequest.java new file mode 100644 index 00000000000000..229ae9107aba87 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.messages.job.savepoints; + +import org.apache.flink.runtime.rest.messages.RequestBody; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nonnull; + +/** + * Request body for a savepoint disposal call. + */ +public class SavepointDisposalRequest implements RequestBody { + + private static final String FIELD_NAME_SAVEPOINT_PATH = "savepoint-path"; + + @JsonProperty(FIELD_NAME_SAVEPOINT_PATH) + private final String savepointPath; + + @JsonCreator + public SavepointDisposalRequest( + @JsonProperty(FIELD_NAME_SAVEPOINT_PATH) @Nonnull String savepointPath) { + this.savepointPath = savepointPath; + } + + @JsonIgnore + public String getSavepointPath() { + return savepointPath; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusHeaders.java new file mode 100644 index 00000000000000..74deffdc492b06 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusHeaders.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.messages.job.savepoints; + +import org.apache.flink.runtime.rest.HttpMethodWrapper; +import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationInfo; +import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationStatusMessageHeaders; +import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationTriggerMessageHeaders; +import org.apache.flink.runtime.rest.handler.job.savepoints.SavepointDisposalHandlers; +import org.apache.flink.runtime.rest.messages.EmptyRequestBody; +import org.apache.flink.runtime.rest.messages.TriggerIdPathParameter; + +import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus; + +/** + * {@link AsynchronousOperationTriggerMessageHeaders} implementation for the {@link SavepointDisposalHandlers.SavepointDisposalStatusHandler}. + */ +public class SavepointDisposalStatusHeaders extends AsynchronousOperationStatusMessageHeaders { + + private static final SavepointDisposalStatusHeaders INSTANCE = new SavepointDisposalStatusHeaders(); + + private static final String URL = String.format("/savepoint-disposal/:%s", TriggerIdPathParameter.KEY); + + private SavepointDisposalStatusHeaders() {} + + @Override + public HttpResponseStatus getResponseStatusCode() { + return HttpResponseStatus.OK; + } + + @Override + public Class getRequestClass() { + return EmptyRequestBody.class; + } + + @Override + public SavepointDisposalStatusMessageParameters getUnresolvedMessageParameters() { + return new SavepointDisposalStatusMessageParameters(); + } + + @Override + public HttpMethodWrapper getHttpMethod() { + return HttpMethodWrapper.GET; + } + + @Override + public String getTargetRestEndpointURL() { + return URL; + } + + public static SavepointDisposalStatusHeaders getInstance() { + return INSTANCE; + } + + @Override + protected Class getValueClass() { + return AsynchronousOperationInfo.class; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusMessageParameters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusMessageParameters.java new file mode 100644 index 00000000000000..d8804c9de6a6ae --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalStatusMessageParameters.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.messages.job.savepoints; + +import org.apache.flink.runtime.rest.handler.job.savepoints.SavepointDisposalHandlers; +import org.apache.flink.runtime.rest.messages.MessageParameters; +import org.apache.flink.runtime.rest.messages.MessagePathParameter; +import org.apache.flink.runtime.rest.messages.MessageQueryParameter; +import org.apache.flink.runtime.rest.messages.TriggerIdPathParameter; + +import java.util.Collection; +import java.util.Collections; + +/** + * {@link MessageParameters} for the {@link SavepointDisposalHandlers.SavepointDisposalStatusHandler}. + */ +public class SavepointDisposalStatusMessageParameters extends MessageParameters { + + public final TriggerIdPathParameter triggerIdPathParameter = new TriggerIdPathParameter(); + + @Override + public Collection> getPathParameters() { + return Collections.singleton(triggerIdPathParameter); + } + + @Override + public Collection> getQueryParameters() { + return Collections.emptyList(); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalTriggerHeaders.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalTriggerHeaders.java new file mode 100644 index 00000000000000..5786498505a866 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalTriggerHeaders.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.messages.job.savepoints; + +import org.apache.flink.runtime.rest.HttpMethodWrapper; +import org.apache.flink.runtime.rest.handler.async.AsynchronousOperationTriggerMessageHeaders; +import org.apache.flink.runtime.rest.handler.job.savepoints.SavepointDisposalHandlers; +import org.apache.flink.runtime.rest.messages.EmptyMessageParameters; + +import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus; + +/** + * {@link AsynchronousOperationTriggerMessageHeaders} for the {@link SavepointDisposalHandlers.SavepointDisposalTriggerHandler}. + */ +public class SavepointDisposalTriggerHeaders extends AsynchronousOperationTriggerMessageHeaders { + + private static final SavepointDisposalTriggerHeaders INSTANCE = new SavepointDisposalTriggerHeaders(); + + private static final String URL = "/savepoint-disposal"; + + private SavepointDisposalTriggerHeaders() {} + + @Override + public HttpResponseStatus getResponseStatusCode() { + return HttpResponseStatus.OK; + } + + @Override + public Class getRequestClass() { + return SavepointDisposalRequest.class; + } + + @Override + public EmptyMessageParameters getUnresolvedMessageParameters() { + return EmptyMessageParameters.getInstance(); + } + + @Override + public HttpMethodWrapper getHttpMethod() { + return HttpMethodWrapper.POST; + } + + @Override + public String getTargetRestEndpointURL() { + return URL; + } + + public static SavepointDisposalTriggerHeaders getInstance() { + return INSTANCE; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java index 471420604c259a..6bb088c9775802 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/RestfulGateway.java @@ -144,6 +144,19 @@ default CompletableFuture triggerSavepoint( throw new UnsupportedOperationException(); } + /** + * Dispose the given savepoint. + * + * @param savepointPath identifying the savepoint to dispose + * @param timeout RPC timeout + * @return A future acknowledge if the disposal succeeded + */ + default CompletableFuture disposeSavepoint( + final String savepointPath, + @RpcTimeout final Time timeout) { + throw new UnsupportedOperationException(); + } + /** * Request the {@link JobStatus} of the given job. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java index d4aa94e19feaa3..af346a7f3ddf4b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java @@ -64,6 +64,7 @@ import org.apache.flink.runtime.rest.handler.job.rescaling.RescalingHandlers; import org.apache.flink.runtime.rest.handler.job.rescaling.RescalingStatusHeaders; import org.apache.flink.runtime.rest.handler.job.rescaling.RescalingTriggerHeaders; +import org.apache.flink.runtime.rest.handler.job.savepoints.SavepointDisposalHandlers; import org.apache.flink.runtime.rest.handler.job.savepoints.SavepointHandlers; import org.apache.flink.runtime.rest.handler.legacy.ConstantTextHandler; import org.apache.flink.runtime.rest.handler.legacy.ExecutionGraphCache; @@ -108,6 +109,8 @@ import org.apache.flink.runtime.rest.messages.job.metrics.JobVertexMetricsHeaders; import org.apache.flink.runtime.rest.messages.job.metrics.SubtaskMetricsHeaders; import org.apache.flink.runtime.rest.messages.job.metrics.TaskManagerMetricsHeaders; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalStatusHeaders; +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointDisposalTriggerHeaders; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusHeaders; import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerHeaders; import org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerDetailsHeaders; @@ -499,6 +502,20 @@ protected List> initiali executor, metricFetcher); + final SavepointDisposalHandlers savepointDisposalHandlers = new SavepointDisposalHandlers(); + + final SavepointDisposalHandlers.SavepointDisposalTriggerHandler savepointDisposalTriggerHandler = savepointDisposalHandlers.new SavepointDisposalTriggerHandler( + restAddressFuture, + leaderRetriever, + timeout, + responseHeaders); + + final SavepointDisposalHandlers.SavepointDisposalStatusHandler savepointDisposalStatusHandler = savepointDisposalHandlers.new SavepointDisposalStatusHandler( + restAddressFuture, + leaderRetriever, + timeout, + responseHeaders); + final Path webUiDir = restConfiguration.getWebUiDir(); Optional> optWebContent; @@ -549,6 +566,8 @@ protected List> initiali handlers.add(Tuple2.of(JobVertexDetailsHeaders.getInstance(), jobVertexDetailsHandler)); handlers.add(Tuple2.of(RescalingTriggerHeaders.getInstance(), rescalingTriggerHandler)); handlers.add(Tuple2.of(RescalingStatusHeaders.getInstance(), rescalingStatusHandler)); + handlers.add(Tuple2.of(SavepointDisposalTriggerHeaders.getInstance(), savepointDisposalTriggerHandler)); + handlers.add(Tuple2.of(SavepointDisposalStatusHeaders.getInstance(), savepointDisposalStatusHandler)); // TODO: Remove once the Yarn proxy can forward all REST verbs handlers.add(Tuple2.of(YarnCancelJobTerminationHeaders.getInstance(), jobCancelTerminationHandler)); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java index 71c391f20a7c3a..8ea686bfe530c7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java @@ -25,7 +25,9 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.blob.VoidBlobStore; +import org.apache.flink.runtime.checkpoint.Checkpoints; import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory; +import org.apache.flink.runtime.checkpoint.savepoint.SavepointV2; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; import org.apache.flink.runtime.executiongraph.ErrorInfo; @@ -54,6 +56,11 @@ import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.rpc.TestingRpcService; +import org.apache.flink.runtime.state.CheckpointMetadataOutputStream; +import org.apache.flink.runtime.state.CheckpointStorage; +import org.apache.flink.runtime.state.CheckpointStorageLocation; +import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation; +import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.testtasks.NoOpInvokable; import org.apache.flink.runtime.testutils.InMemorySubmittedJobGraphStore; import org.apache.flink.runtime.util.TestingFatalErrorHandler; @@ -72,9 +79,18 @@ import org.junit.rules.TestName; import org.mockito.Mockito; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collection; +import java.util.Collections; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; @@ -126,6 +142,8 @@ public class DispatcherTest extends TestLogger { private RunningJobsRegistry runningJobsRegistry; + private Configuration configuration; + /** Instance under test. */ private TestingDispatcher dispatcher; @@ -165,18 +183,19 @@ public void setUp() throws Exception { haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService()); runningJobsRegistry = haServices.getRunningJobsRegistry(); - final Configuration blobServerConfig = new Configuration(); - blobServerConfig.setString( + configuration = new Configuration(); + + configuration.setString( BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath()); dispatcher = new TestingDispatcher( rpcService, Dispatcher.DISPATCHER_NAME + '_' + name.getMethodName(), - new Configuration(), + configuration, haServices, mock(ResourceManagerGateway.class), - new BlobServer(blobServerConfig, new VoidBlobStore()), + new BlobServer(configuration, new VoidBlobStore()), heartbeatServices, UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), null, @@ -344,6 +363,42 @@ public void testJobRecovery() throws Exception { assertThat(jobIds, contains(jobGraph.getJobID())); } + /** + * Tests that we can dispose a savepoint. + */ + @Test + public void testSavepointDisposal() throws Exception { + final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class); + + dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); + + final URI externalPointer = createTestingSavepoint(); + final Path savepointPath = Paths.get(externalPointer); + + assertThat(Files.exists(savepointPath), is(true)); + + dispatcherGateway.disposeSavepoint(externalPointer.toString(), TIMEOUT).get(); + + assertThat(Files.exists(savepointPath), is(false)); + } + + @Nonnull + private URI createTestingSavepoint() throws IOException, URISyntaxException { + final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log); + final CheckpointStorage checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID()); + final File savepointFile = temporaryFolder.newFolder(); + final long checkpointId = 1L; + + final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath()); + + final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream(); + Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream); + + final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint(); + + return new URI(completedCheckpointStorageLocation.getExternalPointer()); + } + private static class TestingDispatcher extends Dispatcher { private final CountDownLatch submitJobLatch = new CountDownLatch(2); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequestTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequestTest.java new file mode 100644 index 00000000000000..3d5a90a6b9dd2d --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointDisposalRequestTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.rest.messages.job.savepoints; + +import org.apache.flink.runtime.rest.messages.RestRequestMarshallingTestBase; + +import org.hamcrest.Matchers; +import org.junit.Assert; + +import java.util.UUID; + +/** + * Tests the un/marshalling of the {@link SavepointDisposalRequest}. + */ +public class SavepointDisposalRequestTest extends RestRequestMarshallingTestBase { + + @Override + protected Class getTestRequestClass() { + return SavepointDisposalRequest.class; + } + + @Override + protected SavepointDisposalRequest getTestRequestInstance() { + return new SavepointDisposalRequest(UUID.randomUUID().toString()); + } + + @Override + protected void assertOriginalEqualsToUnmarshalled(SavepointDisposalRequest expected, SavepointDisposalRequest actual) { + Assert.assertThat(actual.getSavepointPath(), Matchers.is(Matchers.equalTo(expected.getSavepointPath()))); + } +} From d46c182b3ebf2ba067aef3764e87fa3f86aea695 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Mon, 26 Mar 2018 11:25:57 +0200 Subject: [PATCH 0266/2294] [hotfix] Improve logging in AkkaRpcActors --- .../java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java | 4 ++-- .../apache/flink/runtime/rpc/akka/FencedAkkaRpcActor.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java index a7d15d6e46c12b..9f68ede224fc6d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActor.java @@ -140,8 +140,8 @@ public void onReceive(final Object message) { rpcEndpoint.getClass().getName(), message.getClass().getName()); - sendErrorIfSender(new AkkaRpcException("Discard message, because " + - "the rpc endpoint has not been started yet.")); + sendErrorIfSender(new AkkaRpcException( + String.format("Discard message, because the rpc endpoint %s has not been started yet.", rpcEndpoint.getAddress()))); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/FencedAkkaRpcActor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/FencedAkkaRpcActor.java index 57280fdd8b09f4..6096439dbf6f00 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/FencedAkkaRpcActor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/FencedAkkaRpcActor.java @@ -55,7 +55,10 @@ protected void handleMessage(Object message) { sendErrorIfSender( new FencingTokenException( - "Fencing token not set: Ignoring message " + message + " because the fencing token is null.")); + String.format( + "Fencing token not set: Ignoring message %s sent to %s because the fencing token is null.", + message, + rpcEndpoint.getAddress()))); } else { @SuppressWarnings("unchecked") FencedMessage fencedMessage = ((FencedMessage) message); From 4c234fe107fb285ab4a88279b352dc77662346f8 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Tue, 27 Mar 2018 08:31:56 +0200 Subject: [PATCH 0267/2294] [hotfix] Poll invariant variables out of polling loop in RestClusterClient#rescaleJob --- .../program/rest/RestClusterClient.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java index 912971456f6680..cf683742240d41 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java +++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java @@ -507,21 +507,18 @@ public CompletableFuture rescaleJob(JobID jobId, int newParallelism final CompletableFuture rescalingOperationFuture = rescalingTriggerResponseFuture.thenCompose( (TriggerResponse triggerResponse) -> { final TriggerId triggerId = triggerResponse.getTriggerId(); + final RescalingStatusHeaders rescalingStatusHeaders = RescalingStatusHeaders.getInstance(); + final RescalingStatusMessageParameters rescalingStatusMessageParameters = rescalingStatusHeaders.getUnresolvedMessageParameters(); + + rescalingStatusMessageParameters.jobPathParameter.resolve(jobId); + rescalingStatusMessageParameters.triggerIdPathParameter.resolve(triggerId); return pollResourceAsync( - () -> { - final RescalingStatusHeaders rescalingStatusHeaders = RescalingStatusHeaders.getInstance(); - final RescalingStatusMessageParameters rescalingStatusMessageParameters = rescalingStatusHeaders.getUnresolvedMessageParameters(); - - rescalingStatusMessageParameters.jobPathParameter.resolve(jobId); - rescalingStatusMessageParameters.triggerIdPathParameter.resolve(triggerId); - return sendRetryableRequest( - rescalingStatusHeaders, - rescalingStatusMessageParameters, - EmptyRequestBody.getInstance(), - isConnectionProblemException()); - } - ); + () -> sendRetryableRequest( + rescalingStatusHeaders, + rescalingStatusMessageParameters, + EmptyRequestBody.getInstance(), + isConnectionProblemException())); }); return rescalingOperationFuture.thenApply( @@ -535,7 +532,7 @@ public CompletableFuture rescaleJob(JobID jobId, int newParallelism } @Override - public CompletableFuture disposeSavepoint(String savepointPath, Time timeout) { + public CompletableFuture disposeSavepoint(String savepointPath) { final SavepointDisposalRequest savepointDisposalRequest = new SavepointDisposalRequest(savepointPath); final CompletableFuture savepointDisposalTriggerFuture = sendRequest( From a0ba69e7ae5513d1ec11f7b8a2cd0858f7ad1b53 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Mon, 26 Mar 2018 13:46:56 +0200 Subject: [PATCH 0268/2294] [hotfix] Add Assert.fail to RestClusterClientTest --- .../apache/flink/client/program/rest/RestClusterClientTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java index 926da924c5cd31..4202de292f0ea6 100644 --- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java +++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java @@ -472,6 +472,7 @@ testSavepointHandlers.new TestSavepointHandler( try { restClusterClient.triggerSavepoint(new JobID(), null).get(); + fail("Expected exception not thrown."); } catch (final ExecutionException e) { assertTrue( "RestClientException not in causal chain", From c28096a32c9efc3d955b027b4e6152d32b5dd7a2 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Mon, 26 Mar 2018 15:16:09 +0200 Subject: [PATCH 0269/2294] [hotfix] Set RescalingHandlers timeout to RpcUtils.INF_TIMEOUT --- .../runtime/rest/handler/job/rescaling/RescalingHandlers.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/rescaling/RescalingHandlers.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/rescaling/RescalingHandlers.java index 3e4ae5a4e35e4b..0efdd280766ad3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/rescaling/RescalingHandlers.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/rescaling/RescalingHandlers.java @@ -32,6 +32,7 @@ import org.apache.flink.runtime.rest.messages.RescalingParallelismQueryParameter; import org.apache.flink.runtime.rest.messages.TriggerId; import org.apache.flink.runtime.rest.messages.TriggerIdPathParameter; +import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.webmonitor.RestfulGateway; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.util.SerializedThrowable; @@ -80,7 +81,7 @@ protected CompletableFuture triggerOperation(HandlerRequest Date: Fri, 23 Mar 2018 21:42:53 +0100 Subject: [PATCH 0270/2294] [FLINK-9067] [e2eTests] Add StreamSQLTestProgram and test run script. --- .../flink-stream-sql-test/pom.xml | 126 +++++++ .../flink/sql/tests/StreamSQLTestProgram.java | 309 ++++++++++++++++++ flink-end-to-end-tests/pom.xml | 1 + flink-end-to-end-tests/run-nightly-tests.sh | 8 + .../test-scripts/test_streaming_sql.sh | 43 +++ 5 files changed, 487 insertions(+) create mode 100644 flink-end-to-end-tests/flink-stream-sql-test/pom.xml create mode 100644 flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java create mode 100755 flink-end-to-end-tests/test-scripts/test_streaming_sql.sh diff --git a/flink-end-to-end-tests/flink-stream-sql-test/pom.xml b/flink-end-to-end-tests/flink-stream-sql-test/pom.xml new file mode 100644 index 00000000000000..860432ce2daac6 --- /dev/null +++ b/flink-end-to-end-tests/flink-stream-sql-test/pom.xml @@ -0,0 +1,126 @@ + + + + + + flink-end-to-end-tests + org.apache.flink + 1.6-SNAPSHOT + .. + + + 4.0.0 + + flink-stream-sql-test_${scala.binary.version} + flink-stream-sql-test + jar + + + + org.apache.flink + flink-core + ${project.version} + provided + + + org.apache.flink + flink-streaming-scala_${scala.binary.version} + ${project.version} + provided + + + org.apache.flink + flink-table_${scala.binary.version} + ${project.version} + provided + + + org.apache.flink + flink-connector-filesystem_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.apache.flink.sql.tests.StreamSQLTestProgram + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + rename + package + + run + + + + + + + + + + + + + + diff --git a/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java b/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java new file mode 100644 index 00000000000000..91fb4c11acf6ed --- /dev/null +++ b/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.sql.tests; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.TimeCharacteristic; +import org.apache.flink.streaming.api.checkpoint.ListCheckpointed; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.streaming.connectors.fs.bucketing.BasePathBucketer; +import org.apache.flink.streaming.connectors.fs.bucketing.BucketingSink; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.table.sources.DefinedFieldMapping; +import org.apache.flink.table.sources.DefinedRowtimeAttributes; +import org.apache.flink.table.sources.RowtimeAttributeDescriptor; +import org.apache.flink.table.sources.StreamTableSource; +import org.apache.flink.table.sources.tsextractors.ExistingField; +import org.apache.flink.table.sources.wmstrategies.BoundedOutOfOrderTimestamps; +import org.apache.flink.types.Row; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * End-to-end test for Stream SQL queries. + * + *

      Includes the following SQL features: + * - OVER window aggregation + * - keyed and non-keyed GROUP BY TUMBLE aggregation + * - windowed INNER JOIN + * - TableSource with event-time attribute + * + *

      The stream is bounded and will complete after about a minute. + * The result is always constant. + * The job is killed on the first attemped and restarted. + * + *

      Parameters: + * -outputPath Sets the path to where the result data is written. + */ +public class StreamSQLTestProgram { + + public static void main(String[] args) throws Exception { + + ParameterTool params = ParameterTool.fromArgs(args); + String outputPath = params.getRequired("outputPath"); + + StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + sEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); + sEnv.enableCheckpointing(4000); + sEnv.getConfig().setAutoWatermarkInterval(1000); + + StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(sEnv); + + tEnv.registerTableSource("table1", new GeneratorTableSource(10, 100, 60, 0)); + tEnv.registerTableSource("table2", new GeneratorTableSource(5, 0.2f, 60, 5)); + + int overWindowSizeSeconds = 1; + int tumbleWindowSizeSeconds = 10; + + String overQuery = String.format( + "SELECT " + + " key, " + + " rowtime, " + + " COUNT(*) OVER (PARTITION BY key ORDER BY rowtime RANGE BETWEEN INTERVAL '%d' SECOND PRECEDING AND CURRENT ROW) AS cnt " + + "FROM table1", + overWindowSizeSeconds); + + String tumbleQuery = String.format( + "SELECT " + + " key, " + + " CASE SUM(cnt) / COUNT(*) WHEN 101 THEN 1 ELSE 99 END AS correct, " + + " TUMBLE_START(rowtime, INTERVAL '%d' SECOND) AS wStart, " + + " TUMBLE_ROWTIME(rowtime, INTERVAL '%d' SECOND) AS rowtime " + + "FROM (%s) " + + "WHERE rowtime > TIMESTAMP '1970-01-01 00:00:01' " + + "GROUP BY key, TUMBLE(rowtime, INTERVAL '%d' SECOND)", + tumbleWindowSizeSeconds, + tumbleWindowSizeSeconds, + overQuery, + tumbleWindowSizeSeconds); + + String joinQuery = String.format( + "SELECT " + + " t1.key, " + + " t2.rowtime AS rowtime, " + + " t2.correct," + + " t2.wStart " + + "FROM table2 t1, (%s) t2 " + + "WHERE " + + " t1.key = t2.key AND " + + " t1.rowtime BETWEEN t2.rowtime AND t2.rowtime + INTERVAL '%d' SECOND", + tumbleQuery, + tumbleWindowSizeSeconds); + + String finalAgg = String.format( + "SELECT " + + " SUM(correct) AS correct, " + + " TUMBLE_START(rowtime, INTERVAL '20' SECOND) AS rowtime " + + "FROM (%s) " + + "GROUP BY TUMBLE(rowtime, INTERVAL '20' SECOND)", + joinQuery); + + // get Table for SQL query + Table result = tEnv.sqlQuery(finalAgg); + // convert Table into append-only DataStream + DataStream resultStream = + tEnv.toAppendStream(result, Types.ROW(Types.INT, Types.SQL_TIMESTAMP)); + + // define bucketing sink to emit the result + BucketingSink sink = new BucketingSink(outputPath) + .setBucketer(new BasePathBucketer<>()); + + resultStream + // inject a KillMapper that forwards all records but terminates the first execution attempt + .map(new KillMapper()).setParallelism(1) + // add sink function + .addSink(sink).setParallelism(1); + + sEnv.execute(); + } + + /** + * TableSource for generated data. + */ + public static class GeneratorTableSource + implements StreamTableSource, DefinedRowtimeAttributes, DefinedFieldMapping { + + private final int numKeys; + private final float recordsPerKeyAndSecond; + private final int durationSeconds; + private final int offsetSeconds; + + public GeneratorTableSource(int numKeys, float recordsPerKeyAndSecond, int durationSeconds, int offsetSeconds) { + this.numKeys = numKeys; + this.recordsPerKeyAndSecond = recordsPerKeyAndSecond; + this.durationSeconds = durationSeconds; + this.offsetSeconds = offsetSeconds; + } + + @Override + public DataStream getDataStream(StreamExecutionEnvironment execEnv) { + return execEnv.addSource(new Generator(numKeys, recordsPerKeyAndSecond, durationSeconds, offsetSeconds)); + } + + @Override + public TypeInformation getReturnType() { + return Types.ROW(Types.INT, Types.LONG, Types.STRING); + } + + @Override + public TableSchema getTableSchema() { + return new TableSchema( + new String[] {"key", "rowtime", "payload"}, + new TypeInformation[] {Types.INT, Types.SQL_TIMESTAMP, Types.STRING}); + } + + @Override + public String explainSource() { + return "GeneratorTableSource"; + } + + @Override + public List getRowtimeAttributeDescriptors() { + return Collections.singletonList( + new RowtimeAttributeDescriptor( + "rowtime", + new ExistingField("ts"), + new BoundedOutOfOrderTimestamps(100))); + } + + @Override + public Map getFieldMapping() { + Map mapping = new HashMap<>(); + mapping.put("key", "f0"); + mapping.put("ts", "f1"); + mapping.put("payload", "f2"); + return mapping; + } + } + + /** + * Data-generating source function. + */ + public static class Generator implements SourceFunction, ResultTypeQueryable, ListCheckpointed { + + private final int numKeys; + private final int offsetSeconds; + + private final int sleepMs; + private final int durationMs; + + private long ms = 0; + + public Generator(int numKeys, float rowsPerKeyAndSecond, int durationSeconds, int offsetSeconds) { + this.numKeys = numKeys; + this.durationMs = durationSeconds * 1000; + this.offsetSeconds = offsetSeconds; + + this.sleepMs = (int) (1000 / rowsPerKeyAndSecond); + } + + @Override + public void run(SourceContext ctx) throws Exception { + long offsetMS = offsetSeconds * 2000L; + + while (ms < durationMs) { + synchronized (ctx.getCheckpointLock()) { + for (int i = 0; i < numKeys; i++) { + ctx.collect(Row.of(i, ms + offsetMS, "Some payload...")); + } + ms += sleepMs; + } + Thread.sleep(sleepMs); + } + } + + @Override + public void cancel() { } + + @Override + public TypeInformation getProducedType() { + return Types.ROW(Types.INT, Types.LONG, Types.STRING); + } + + @Override + public List snapshotState(long checkpointId, long timestamp) throws Exception { + return Collections.singletonList(ms); + } + + @Override + public void restoreState(List state) throws Exception { + for (Long l : state) { + ms += l; + } + } + } + + /** + * Kills the first execution attempt of an application when it receives the second record. + */ + public static class KillMapper implements MapFunction, ListCheckpointed, ResultTypeQueryable { + + // counts all processed records of all previous execution attempts + private int saveRecordCnt = 0; + // counts all processed records of this execution attempt + private int lostRecordCnt = 0; + + @Override + public Row map(Row value) throws Exception { + + // the both counts are the same only in the first execution attempt + if (saveRecordCnt == 1 && lostRecordCnt == 1) { + throw new RuntimeException("Kill this Job!"); + } + + // update checkpointed counter + saveRecordCnt++; + // update non-checkpointed counter + lostRecordCnt++; + + // forward record + return value; + } + + @Override + public TypeInformation getProducedType() { + return Types.ROW(Types.INT, Types.SQL_TIMESTAMP); + } + + @Override + public List snapshotState(long checkpointId, long timestamp) throws Exception { + return Collections.singletonList(saveRecordCnt); + } + + @Override + public void restoreState(List state) throws Exception { + for (Integer i : state) { + saveRecordCnt += i; + } + } + } + +} diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml index f8913d448d8761..35002dd07f5599 100644 --- a/flink-end-to-end-tests/pom.xml +++ b/flink-end-to-end-tests/pom.xml @@ -37,6 +37,7 @@ under the License. flink-parent-child-classloading-test flink-dataset-allround-test + flink-stream-sql-test diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh index 71224e09f1b7f7..eca2e4db47af59 100755 --- a/flink-end-to-end-tests/run-nightly-tests.sh +++ b/flink-end-to-end-tests/run-nightly-tests.sh @@ -55,5 +55,13 @@ if [ $EXIT_CODE == 0 ]; then EXIT_CODE=$? fi +if [ $EXIT_CODE == 0 ]; then + printf "\n==============================================================================\n" + printf "Running Streaming SQL nightly end-to-end test\n" + printf "==============================================================================\n" + $END_TO_END_DIR/test-scripts/test_streaming_sql.sh + EXIT_CODE=$? +fi + # Exit code for Travis build success/failure exit $EXIT_CODE diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh b/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh new file mode 100755 index 00000000000000..ed7b16c852b608 --- /dev/null +++ b/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +source "$(dirname "$0")"/common.sh + +TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/flink-stream-sql-test/target/StreamSQLTestProgram.jar + +# copy flink-table jar into lib folder +cp $FLINK_DIR/opt/flink-table*jar $FLINK_DIR/lib + +start_cluster +$FLINK_DIR/bin/taskmanager.sh start +$FLINK_DIR/bin/taskmanager.sh start +$FLINK_DIR/bin/taskmanager.sh start + +$FLINK_DIR/bin/flink run -p 4 $TEST_PROGRAM_JAR -outputPath $TEST_DATA_DIR/out/result + +stop_cluster +$FLINK_DIR/bin/taskmanager.sh stop-all + +# remove flink-table from lib folder +rm $FLINK_DIR/lib/flink-table*jar + +# collect results from files +cat /tmp/xxx/part-0-0 /tmp/xxx/_part-0-1.pending > $TEST_DATA_DIR/out/result-complete +# check result +check_result_hash "StreamSQL" $TEST_DATA_DIR/out/result-complete "b29f14ed221a936211202ff65b51ee26" From e04a3bf7703544cf5012eb2f55626af8e0a2cb8c Mon Sep 17 00:00:00 2001 From: Timo Walther Date: Tue, 27 Mar 2018 14:20:47 +0200 Subject: [PATCH 0271/2294] [FLINK-9067] [e2eTests] Fix test and simplify code This closes #5759. --- .../flink-stream-sql-test/pom.xml | 21 +--------------- .../flink/sql/tests/StreamSQLTestProgram.java | 12 +++++----- .../test-scripts/test_streaming_sql.sh | 24 ++++++++++++++----- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/flink-end-to-end-tests/flink-stream-sql-test/pom.xml b/flink-end-to-end-tests/flink-stream-sql-test/pom.xml index 860432ce2daac6..1b8e4f7a4d17bf 100644 --- a/flink-end-to-end-tests/flink-stream-sql-test/pom.xml +++ b/flink-end-to-end-tests/flink-stream-sql-test/pom.xml @@ -73,6 +73,7 @@ shade + StreamSQLTestProgram com.google.code.findbugs:jsr305 @@ -99,26 +100,6 @@ - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - rename - package - - run - - - - - - - - - diff --git a/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java b/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java index 91fb4c11acf6ed..e9e79ac2745a9f 100644 --- a/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java +++ b/flink-end-to-end-tests/flink-stream-sql-test/src/main/java/org/apache/flink/sql/tests/StreamSQLTestProgram.java @@ -58,7 +58,7 @@ * *

      The stream is bounded and will complete after about a minute. * The result is always constant. - * The job is killed on the first attemped and restarted. + * The job is killed on the first attempt and restarted. * *

      Parameters: * -outputPath Sets the path to where the result data is written. @@ -249,12 +249,12 @@ public TypeInformation getProducedType() { } @Override - public List snapshotState(long checkpointId, long timestamp) throws Exception { + public List snapshotState(long checkpointId, long timestamp) { return Collections.singletonList(ms); } @Override - public void restoreState(List state) throws Exception { + public void restoreState(List state) { for (Long l : state) { ms += l; } @@ -272,7 +272,7 @@ public static class KillMapper implements MapFunction, ListCheckpointe private int lostRecordCnt = 0; @Override - public Row map(Row value) throws Exception { + public Row map(Row value) { // the both counts are the same only in the first execution attempt if (saveRecordCnt == 1 && lostRecordCnt == 1) { @@ -294,12 +294,12 @@ public TypeInformation getProducedType() { } @Override - public List snapshotState(long checkpointId, long timestamp) throws Exception { + public List snapshotState(long checkpointId, long timestamp) { return Collections.singletonList(saveRecordCnt); } @Override - public void restoreState(List state) throws Exception { + public void restoreState(List state) { for (Integer i : state) { saveRecordCnt += i; } diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh b/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh index ed7b16c852b608..21c64a168c30ef 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh @@ -31,13 +31,25 @@ $FLINK_DIR/bin/taskmanager.sh start $FLINK_DIR/bin/flink run -p 4 $TEST_PROGRAM_JAR -outputPath $TEST_DATA_DIR/out/result -stop_cluster -$FLINK_DIR/bin/taskmanager.sh stop-all +function sql_cleanup() { -# remove flink-table from lib folder -rm $FLINK_DIR/lib/flink-table*jar + stop_cluster + $FLINK_DIR/bin/taskmanager.sh stop-all + + # remove flink-table from lib folder + rm $FLINK_DIR/lib/flink-table*jar + + # make sure to run regular cleanup as well + cleanup +} +trap sql_cleanup INT +trap sql_cleanup EXIT # collect results from files -cat /tmp/xxx/part-0-0 /tmp/xxx/_part-0-1.pending > $TEST_DATA_DIR/out/result-complete -# check result +cat $TEST_DATA_DIR/out/result/part-0-0 $TEST_DATA_DIR/out/result/_part-0-1.pending > $TEST_DATA_DIR/out/result-complete + +# check result: +# 20,1970-01-01 00:00:00.0 +# 20,1970-01-01 00:00:20.0 +# 20,1970-01-01 00:00:40.0 check_result_hash "StreamSQL" $TEST_DATA_DIR/out/result-complete "b29f14ed221a936211202ff65b51ee26" From 3a61ea47922280e15f462ca3cdc0c367047bde24 Mon Sep 17 00:00:00 2001 From: Matrix42 <934336389@qq.com> Date: Tue, 27 Mar 2018 14:28:07 +0800 Subject: [PATCH 0272/2294] [FLINK-9093] [docs] Ship jQuery library without external provider This closes #5770. --- docs/_layouts/base.html | 2 +- docs/page/js/jquery.min.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 docs/page/js/jquery.min.js diff --git a/docs/_layouts/base.html b/docs/_layouts/base.html index 1e360cf5dde1d4..7d86d74f12b067 100644 --- a/docs/_layouts/base.html +++ b/docs/_layouts/base.html @@ -81,7 +81,7 @@ - + diff --git a/docs/page/js/jquery.min.js b/docs/page/js/jquery.min.js new file mode 100644 index 00000000000000..e6a051d0d1d327 --- /dev/null +++ b/docs/page/js/jquery.min.js @@ -0,0 +1,4 @@ +/*! jQuery v1.11.2 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ +!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k={},l="1.11.2",m=function(a,b){return new m.fn.init(a,b)},n=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,o=/^-ms-/,p=/-([\da-z])/gi,q=function(a,b){return b.toUpperCase()};m.fn=m.prototype={jquery:l,constructor:m,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=m.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return m.each(this,a,b)},map:function(a){return this.pushStack(m.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},m.extend=m.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||m.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(m.isPlainObject(c)||(b=m.isArray(c)))?(b?(b=!1,f=a&&m.isArray(a)?a:[]):f=a&&m.isPlainObject(a)?a:{},g[d]=m.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},m.extend({expando:"jQuery"+(l+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===m.type(a)},isArray:Array.isArray||function(a){return"array"===m.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){return!m.isArray(a)&&a-parseFloat(a)+1>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==m.type(a)||a.nodeType||m.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(k.ownLast)for(b in a)return j.call(a,b);for(b in a);return void 0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(b){b&&m.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(o,"ms-").replace(p,q)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=r(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(n,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(r(Object(a))?m.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=r(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(f=a[b],b=a,a=f),m.isFunction(a)?(c=d.call(arguments,2),e=function(){return a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||m.guid++,e):void 0},now:function(){return+new Date},support:k}),m.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function r(a){var b=a.length,c=m.type(a);return"function"===c||m.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var s=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=hb(),z=hb(),A=hb(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N=M.replace("w","w#"),O="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+N+"))|)"+L+"*\\]",P=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+O+")*)|.*)\\)|)",Q=new RegExp(L+"+","g"),R=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),S=new RegExp("^"+L+"*,"+L+"*"),T=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),U=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),V=new RegExp(P),W=new RegExp("^"+N+"$"),X={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M.replace("w","w*")+")"),ATTR:new RegExp("^"+O),PSEUDO:new RegExp("^"+P),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},Y=/^(?:input|select|textarea|button)$/i,Z=/^h\d$/i,$=/^[^{]+\{\s*\[native \w/,_=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ab=/[+~]/,bb=/'|\\/g,cb=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),db=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},eb=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(fb){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function gb(a,b,d,e){var f,h,j,k,l,o,r,s,w,x;if((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,d=d||[],k=b.nodeType,"string"!=typeof a||!a||1!==k&&9!==k&&11!==k)return d;if(!e&&p){if(11!==k&&(f=_.exec(a)))if(j=f[1]){if(9===k){if(h=b.getElementById(j),!h||!h.parentNode)return d;if(h.id===j)return d.push(h),d}else if(b.ownerDocument&&(h=b.ownerDocument.getElementById(j))&&t(b,h)&&h.id===j)return d.push(h),d}else{if(f[2])return H.apply(d,b.getElementsByTagName(a)),d;if((j=f[3])&&c.getElementsByClassName)return H.apply(d,b.getElementsByClassName(j)),d}if(c.qsa&&(!q||!q.test(a))){if(s=r=u,w=b,x=1!==k&&a,1===k&&"object"!==b.nodeName.toLowerCase()){o=g(a),(r=b.getAttribute("id"))?s=r.replace(bb,"\\$&"):b.setAttribute("id",s),s="[id='"+s+"'] ",l=o.length;while(l--)o[l]=s+rb(o[l]);w=ab.test(a)&&pb(b.parentNode)||b,x=o.join(",")}if(x)try{return H.apply(d,w.querySelectorAll(x)),d}catch(y){}finally{r||b.removeAttribute("id")}}}return i(a.replace(R,"$1"),b,d,e)}function hb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ib(a){return a[u]=!0,a}function jb(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function kb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function lb(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function mb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function nb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function ob(a){return ib(function(b){return b=+b,ib(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function pb(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=gb.support={},f=gb.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=gb.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=g.documentElement,e=g.defaultView,e&&e!==e.top&&(e.addEventListener?e.addEventListener("unload",eb,!1):e.attachEvent&&e.attachEvent("onunload",eb)),p=!f(g),c.attributes=jb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=jb(function(a){return a.appendChild(g.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=$.test(g.getElementsByClassName),c.getById=jb(function(a){return o.appendChild(a).id=u,!g.getElementsByName||!g.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(cb,db);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=$.test(g.querySelectorAll))&&(jb(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+L+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+L+"*(?:value|"+K+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),jb(function(a){var b=g.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+L+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=$.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&jb(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",P)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=$.test(o.compareDocumentPosition),t=b||$.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===g||a.ownerDocument===v&&t(v,a)?-1:b===g||b.ownerDocument===v&&t(v,b)?1:k?J(k,a)-J(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,h=[a],i=[b];if(!e||!f)return a===g?-1:b===g?1:e?-1:f?1:k?J(k,a)-J(k,b):0;if(e===f)return lb(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)i.unshift(c);while(h[d]===i[d])d++;return d?lb(h[d],i[d]):h[d]===v?-1:i[d]===v?1:0},g):n},gb.matches=function(a,b){return gb(a,null,null,b)},gb.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(U,"='$1']"),!(!c.matchesSelector||!p||r&&r.test(b)||q&&q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return gb(b,n,null,[a]).length>0},gb.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},gb.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&D.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},gb.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},gb.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=gb.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=gb.selectors={cacheLength:50,createPseudo:ib,match:X,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(cb,db),a[3]=(a[3]||a[4]||a[5]||"").replace(cb,db),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||gb.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&gb.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return X.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&V.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(cb,db).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+L+")"+a+"("+L+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=gb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(Q," ")+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){k=q[u]||(q[u]={}),j=k[a]||[],n=j[0]===w&&j[1],m=j[0]===w&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[w,n,m];break}}else if(s&&(j=(b[u]||(b[u]={}))[a])&&j[0]===w)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(s&&((l[u]||(l[u]={}))[a]=[w,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||gb.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ib(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=J(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ib(function(a){var b=[],c=[],d=h(a.replace(R,"$1"));return d[u]?ib(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ib(function(a){return function(b){return gb(a,b).length>0}}),contains:ib(function(a){return a=a.replace(cb,db),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ib(function(a){return W.test(a||"")||gb.error("unsupported lang: "+a),a=a.replace(cb,db).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Z.test(a.nodeName)},input:function(a){return Y.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:ob(function(){return[0]}),last:ob(function(a,b){return[b-1]}),eq:ob(function(a,b,c){return[0>c?c+b:c]}),even:ob(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:ob(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:ob(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:ob(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function sb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[u]||(b[u]={}),(h=i[d])&&h[0]===w&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function tb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ub(a,b,c){for(var d=0,e=b.length;e>d;d++)gb(a,b[d],c);return c}function vb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function wb(a,b,c,d,e,f){return d&&!d[u]&&(d=wb(d)),e&&!e[u]&&(e=wb(e,f)),ib(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ub(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:vb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=vb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=vb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function xb(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=sb(function(a){return a===b},h,!0),l=sb(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[sb(tb(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return wb(i>1&&tb(m),i>1&&rb(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(R,"$1"),c,e>i&&xb(a.slice(i,e)),f>e&&xb(a=a.slice(e)),f>e&&rb(a))}m.push(c)}return tb(m)}function yb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,m,o,p=0,q="0",r=f&&[],s=[],t=j,u=f||e&&d.find.TAG("*",k),v=w+=null==t?1:Math.random()||.1,x=u.length;for(k&&(j=g!==n&&g);q!==x&&null!=(l=u[q]);q++){if(e&&l){m=0;while(o=a[m++])if(o(l,g,h)){i.push(l);break}k&&(w=v)}c&&((l=!o&&l)&&p--,f&&r.push(l))}if(p+=q,c&&q!==p){m=0;while(o=b[m++])o(r,s,g,h);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=F.call(i));s=vb(s)}H.apply(i,s),k&&!f&&s.length>0&&p+b.length>1&&gb.uniqueSort(i)}return k&&(w=v,j=t),r};return c?ib(f):f}return h=gb.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=xb(b[c]),f[u]?d.push(f):e.push(f);f=A(a,yb(e,d)),f.selector=a}return f},i=gb.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(cb,db),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=X.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(cb,db),ab.test(j[0].type)&&pb(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&rb(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,ab.test(a)&&pb(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=jb(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),jb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||kb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&jb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||kb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),jb(function(a){return null==a.getAttribute("disabled")})||kb(K,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),gb}(a);m.find=s,m.expr=s.selectors,m.expr[":"]=m.expr.pseudos,m.unique=s.uniqueSort,m.text=s.getText,m.isXMLDoc=s.isXML,m.contains=s.contains;var t=m.expr.match.needsContext,u=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,v=/^.[^:#\[\.,]*$/;function w(a,b,c){if(m.isFunction(b))return m.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return m.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(v.test(b))return m.filter(b,a,c);b=m.filter(b,a)}return m.grep(a,function(a){return m.inArray(a,b)>=0!==c})}m.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?m.find.matchesSelector(d,a)?[d]:[]:m.find.matches(a,m.grep(b,function(a){return 1===a.nodeType}))},m.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(m(a).filter(function(){for(b=0;e>b;b++)if(m.contains(d[b],this))return!0}));for(b=0;e>b;b++)m.find(a,d[b],c);return c=this.pushStack(e>1?m.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(w(this,a||[],!1))},not:function(a){return this.pushStack(w(this,a||[],!0))},is:function(a){return!!w(this,"string"==typeof a&&t.test(a)?m(a):a||[],!1).length}});var x,y=a.document,z=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,A=m.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:z.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||x).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof m?b[0]:b,m.merge(this,m.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:y,!0)),u.test(c[1])&&m.isPlainObject(b))for(c in b)m.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}if(d=y.getElementById(c[2]),d&&d.parentNode){if(d.id!==c[2])return x.find(a);this.length=1,this[0]=d}return this.context=y,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):m.isFunction(a)?"undefined"!=typeof x.ready?x.ready(a):a(m):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),m.makeArray(a,this))};A.prototype=m.fn,x=m(y);var B=/^(?:parents|prev(?:Until|All))/,C={children:!0,contents:!0,next:!0,prev:!0};m.extend({dir:function(a,b,c){var d=[],e=a[b];while(e&&9!==e.nodeType&&(void 0===c||1!==e.nodeType||!m(e).is(c)))1===e.nodeType&&d.push(e),e=e[b];return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),m.fn.extend({has:function(a){var b,c=m(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(m.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=t.test(a)||"string"!=typeof a?m(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&m.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?m.unique(f):f)},index:function(a){return a?"string"==typeof a?m.inArray(this[0],m(a)):m.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(m.unique(m.merge(this.get(),m(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function D(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}m.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return m.dir(a,"parentNode")},parentsUntil:function(a,b,c){return m.dir(a,"parentNode",c)},next:function(a){return D(a,"nextSibling")},prev:function(a){return D(a,"previousSibling")},nextAll:function(a){return m.dir(a,"nextSibling")},prevAll:function(a){return m.dir(a,"previousSibling")},nextUntil:function(a,b,c){return m.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return m.dir(a,"previousSibling",c)},siblings:function(a){return m.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return m.sibling(a.firstChild)},contents:function(a){return m.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:m.merge([],a.childNodes)}},function(a,b){m.fn[a]=function(c,d){var e=m.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=m.filter(d,e)),this.length>1&&(C[a]||(e=m.unique(e)),B.test(a)&&(e=e.reverse())),this.pushStack(e)}});var E=/\S+/g,F={};function G(a){var b=F[a]={};return m.each(a.match(E)||[],function(a,c){b[c]=!0}),b}m.Callbacks=function(a){a="string"==typeof a?F[a]||G(a):m.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(c=a.memory&&l,d=!0,f=g||0,g=0,e=h.length,b=!0;h&&e>f;f++)if(h[f].apply(l[0],l[1])===!1&&a.stopOnFalse){c=!1;break}b=!1,h&&(i?i.length&&j(i.shift()):c?h=[]:k.disable())},k={add:function(){if(h){var d=h.length;!function f(b){m.each(b,function(b,c){var d=m.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&f(c)})}(arguments),b?e=h.length:c&&(g=d,j(c))}return this},remove:function(){return h&&m.each(arguments,function(a,c){var d;while((d=m.inArray(c,h,d))>-1)h.splice(d,1),b&&(e>=d&&e--,f>=d&&f--)}),this},has:function(a){return a?m.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],e=0,this},disable:function(){return h=i=c=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,c||k.disable(),this},locked:function(){return!i},fireWith:function(a,c){return!h||d&&!i||(c=c||[],c=[a,c.slice?c.slice():c],b?i.push(c):j(c)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!d}};return k},m.extend({Deferred:function(a){var b=[["resolve","done",m.Callbacks("once memory"),"resolved"],["reject","fail",m.Callbacks("once memory"),"rejected"],["notify","progress",m.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return m.Deferred(function(c){m.each(b,function(b,f){var g=m.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&m.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?m.extend(a,d):d}},e={};return d.pipe=d.then,m.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&m.isFunction(a.promise)?e:0,g=1===f?a:m.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&m.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var H;m.fn.ready=function(a){return m.ready.promise().done(a),this},m.extend({isReady:!1,readyWait:1,holdReady:function(a){a?m.readyWait++:m.ready(!0)},ready:function(a){if(a===!0?!--m.readyWait:!m.isReady){if(!y.body)return setTimeout(m.ready);m.isReady=!0,a!==!0&&--m.readyWait>0||(H.resolveWith(y,[m]),m.fn.triggerHandler&&(m(y).triggerHandler("ready"),m(y).off("ready")))}}});function I(){y.addEventListener?(y.removeEventListener("DOMContentLoaded",J,!1),a.removeEventListener("load",J,!1)):(y.detachEvent("onreadystatechange",J),a.detachEvent("onload",J))}function J(){(y.addEventListener||"load"===event.type||"complete"===y.readyState)&&(I(),m.ready())}m.ready.promise=function(b){if(!H)if(H=m.Deferred(),"complete"===y.readyState)setTimeout(m.ready);else if(y.addEventListener)y.addEventListener("DOMContentLoaded",J,!1),a.addEventListener("load",J,!1);else{y.attachEvent("onreadystatechange",J),a.attachEvent("onload",J);var c=!1;try{c=null==a.frameElement&&y.documentElement}catch(d){}c&&c.doScroll&&!function e(){if(!m.isReady){try{c.doScroll("left")}catch(a){return setTimeout(e,50)}I(),m.ready()}}()}return H.promise(b)};var K="undefined",L;for(L in m(k))break;k.ownLast="0"!==L,k.inlineBlockNeedsLayout=!1,m(function(){var a,b,c,d;c=y.getElementsByTagName("body")[0],c&&c.style&&(b=y.createElement("div"),d=y.createElement("div"),d.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(d).appendChild(b),typeof b.style.zoom!==K&&(b.style.cssText="display:inline;margin:0;border:0;padding:1px;width:1px;zoom:1",k.inlineBlockNeedsLayout=a=3===b.offsetWidth,a&&(c.style.zoom=1)),c.removeChild(d))}),function(){var a=y.createElement("div");if(null==k.deleteExpando){k.deleteExpando=!0;try{delete a.test}catch(b){k.deleteExpando=!1}}a=null}(),m.acceptData=function(a){var b=m.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b};var M=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,N=/([A-Z])/g;function O(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(N,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:M.test(c)?m.parseJSON(c):c}catch(e){}m.data(a,b,c)}else c=void 0}return c}function P(a){var b;for(b in a)if(("data"!==b||!m.isEmptyObject(a[b]))&&"toJSON"!==b)return!1; +return!0}function Q(a,b,d,e){if(m.acceptData(a)){var f,g,h=m.expando,i=a.nodeType,j=i?m.cache:a,k=i?a[h]:a[h]&&h;if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||m.guid++:h),j[k]||(j[k]=i?{}:{toJSON:m.noop}),("object"==typeof b||"function"==typeof b)&&(e?j[k]=m.extend(j[k],b):j[k].data=m.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[m.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[m.camelCase(b)])):f=g,f}}function R(a,b,c){if(m.acceptData(a)){var d,e,f=a.nodeType,g=f?m.cache:a,h=f?a[m.expando]:m.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){m.isArray(b)?b=b.concat(m.map(b,m.camelCase)):b in d?b=[b]:(b=m.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!P(d):!m.isEmptyObject(d))return}(c||(delete g[h].data,P(g[h])))&&(f?m.cleanData([a],!0):k.deleteExpando||g!=g.window?delete g[h]:g[h]=null)}}}m.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?m.cache[a[m.expando]]:a[m.expando],!!a&&!P(a)},data:function(a,b,c){return Q(a,b,c)},removeData:function(a,b){return R(a,b)},_data:function(a,b,c){return Q(a,b,c,!0)},_removeData:function(a,b){return R(a,b,!0)}}),m.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=m.data(f),1===f.nodeType&&!m._data(f,"parsedAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=m.camelCase(d.slice(5)),O(f,d,e[d])));m._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){m.data(this,a)}):arguments.length>1?this.each(function(){m.data(this,a,b)}):f?O(f,a,m.data(f,a)):void 0},removeData:function(a){return this.each(function(){m.removeData(this,a)})}}),m.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=m._data(a,b),c&&(!d||m.isArray(c)?d=m._data(a,b,m.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=m.queue(a,b),d=c.length,e=c.shift(),f=m._queueHooks(a,b),g=function(){m.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return m._data(a,c)||m._data(a,c,{empty:m.Callbacks("once memory").add(function(){m._removeData(a,b+"queue"),m._removeData(a,c)})})}}),m.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},W=/^(?:checkbox|radio)$/i;!function(){var a=y.createElement("input"),b=y.createElement("div"),c=y.createDocumentFragment();if(b.innerHTML="
      a",k.leadingWhitespace=3===b.firstChild.nodeType,k.tbody=!b.getElementsByTagName("tbody").length,k.htmlSerialize=!!b.getElementsByTagName("link").length,k.html5Clone="<:nav>"!==y.createElement("nav").cloneNode(!0).outerHTML,a.type="checkbox",a.checked=!0,c.appendChild(a),k.appendChecked=a.checked,b.innerHTML="",k.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue,c.appendChild(b),b.innerHTML="",k.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,k.noCloneEvent=!0,b.attachEvent&&(b.attachEvent("onclick",function(){k.noCloneEvent=!1}),b.cloneNode(!0).click()),null==k.deleteExpando){k.deleteExpando=!0;try{delete b.test}catch(d){k.deleteExpando=!1}}}(),function(){var b,c,d=y.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(k[b+"Bubbles"]=c in a)||(d.setAttribute(c,"t"),k[b+"Bubbles"]=d.attributes[c].expando===!1);d=null}();var X=/^(?:input|select|textarea)$/i,Y=/^key/,Z=/^(?:mouse|pointer|contextmenu)|click/,$=/^(?:focusinfocus|focusoutblur)$/,_=/^([^.]*)(?:\.(.+)|)$/;function ab(){return!0}function bb(){return!1}function cb(){try{return y.activeElement}catch(a){}}m.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=m.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return typeof m===K||a&&m.event.triggered===a.type?void 0:m.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(E)||[""],h=b.length;while(h--)f=_.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=m.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=m.event.special[o]||{},l=m.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&m.expr.match.needsContext.test(e),namespace:p.join(".")},i),(n=g[o])||(n=g[o]=[],n.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?n.splice(n.delegateCount++,0,l):n.push(l),m.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,n,o,p,q,r=m.hasData(a)&&m._data(a);if(r&&(k=r.events)){b=(b||"").match(E)||[""],j=b.length;while(j--)if(h=_.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=m.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,n=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=n.length;while(f--)g=n[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(n.splice(f,1),g.selector&&n.delegateCount--,l.remove&&l.remove.call(a,g));i&&!n.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||m.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)m.event.remove(a,o+b[j],c,d,!0);m.isEmptyObject(k)&&(delete r.handle,m._removeData(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,l,n,o=[d||y],p=j.call(b,"type")?b.type:b,q=j.call(b,"namespace")?b.namespace.split("."):[];if(h=l=d=d||y,3!==d.nodeType&&8!==d.nodeType&&!$.test(p+m.event.triggered)&&(p.indexOf(".")>=0&&(q=p.split("."),p=q.shift(),q.sort()),g=p.indexOf(":")<0&&"on"+p,b=b[m.expando]?b:new m.Event(p,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=q.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:m.makeArray(c,[b]),k=m.event.special[p]||{},e||!k.trigger||k.trigger.apply(d,c)!==!1)){if(!e&&!k.noBubble&&!m.isWindow(d)){for(i=k.delegateType||p,$.test(i+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),l=h;l===(d.ownerDocument||y)&&o.push(l.defaultView||l.parentWindow||a)}n=0;while((h=o[n++])&&!b.isPropagationStopped())b.type=n>1?i:k.bindType||p,f=(m._data(h,"events")||{})[b.type]&&m._data(h,"handle"),f&&f.apply(h,c),f=g&&h[g],f&&f.apply&&m.acceptData(h)&&(b.result=f.apply(h,c),b.result===!1&&b.preventDefault());if(b.type=p,!e&&!b.isDefaultPrevented()&&(!k._default||k._default.apply(o.pop(),c)===!1)&&m.acceptData(d)&&g&&d[p]&&!m.isWindow(d)){l=d[g],l&&(d[g]=null),m.event.triggered=p;try{d[p]()}catch(r){}m.event.triggered=void 0,l&&(d[g]=l)}return b.result}},dispatch:function(a){a=m.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(m._data(this,"events")||{})[a.type]||[],k=m.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=m.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,g=0;while((e=f.handlers[g++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(e.namespace))&&(a.handleObj=e,a.data=e.data,c=((m.event.special[e.origType]||{}).handle||e.handler).apply(f.elem,i),void 0!==c&&(a.result=c)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(e=[],f=0;h>f;f++)d=b[f],c=d.selector+" ",void 0===e[c]&&(e[c]=d.needsContext?m(c,this).index(i)>=0:m.find(c,this,null,[i]).length),e[c]&&e.push(d);e.length&&g.push({elem:i,handlers:e})}return h]","i"),hb=/^\s+/,ib=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,jb=/<([\w:]+)/,kb=/\s*$/g,rb={option:[1,""],legend:[1,"

      ","
      "],area:[1,"",""],param:[1,"",""],thead:[1,"","
      "],tr:[2,"","
      "],col:[2,"","
      "],td:[3,"","
      "],_default:k.htmlSerialize?[0,"",""]:[1,"X
      ","
      "]},sb=db(y),tb=sb.appendChild(y.createElement("div"));rb.optgroup=rb.option,rb.tbody=rb.tfoot=rb.colgroup=rb.caption=rb.thead,rb.th=rb.td;function ub(a,b){var c,d,e=0,f=typeof a.getElementsByTagName!==K?a.getElementsByTagName(b||"*"):typeof a.querySelectorAll!==K?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||m.nodeName(d,b)?f.push(d):m.merge(f,ub(d,b));return void 0===b||b&&m.nodeName(a,b)?m.merge([a],f):f}function vb(a){W.test(a.type)&&(a.defaultChecked=a.checked)}function wb(a,b){return m.nodeName(a,"table")&&m.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function xb(a){return a.type=(null!==m.find.attr(a,"type"))+"/"+a.type,a}function yb(a){var b=pb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function zb(a,b){for(var c,d=0;null!=(c=a[d]);d++)m._data(c,"globalEval",!b||m._data(b[d],"globalEval"))}function Ab(a,b){if(1===b.nodeType&&m.hasData(a)){var c,d,e,f=m._data(a),g=m._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)m.event.add(b,c,h[c][d])}g.data&&(g.data=m.extend({},g.data))}}function Bb(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!k.noCloneEvent&&b[m.expando]){e=m._data(b);for(d in e.events)m.removeEvent(b,d,e.handle);b.removeAttribute(m.expando)}"script"===c&&b.text!==a.text?(xb(b).text=a.text,yb(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),k.html5Clone&&a.innerHTML&&!m.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&W.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}}m.extend({clone:function(a,b,c){var d,e,f,g,h,i=m.contains(a.ownerDocument,a);if(k.html5Clone||m.isXMLDoc(a)||!gb.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(tb.innerHTML=a.outerHTML,tb.removeChild(f=tb.firstChild)),!(k.noCloneEvent&&k.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||m.isXMLDoc(a)))for(d=ub(f),h=ub(a),g=0;null!=(e=h[g]);++g)d[g]&&Bb(e,d[g]);if(b)if(c)for(h=h||ub(a),d=d||ub(f),g=0;null!=(e=h[g]);g++)Ab(e,d[g]);else Ab(a,f);return d=ub(f,"script"),d.length>0&&zb(d,!i&&ub(a,"script")),d=h=e=null,f},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,l,n=a.length,o=db(b),p=[],q=0;n>q;q++)if(f=a[q],f||0===f)if("object"===m.type(f))m.merge(p,f.nodeType?[f]:f);else if(lb.test(f)){h=h||o.appendChild(b.createElement("div")),i=(jb.exec(f)||["",""])[1].toLowerCase(),l=rb[i]||rb._default,h.innerHTML=l[1]+f.replace(ib,"<$1>")+l[2],e=l[0];while(e--)h=h.lastChild;if(!k.leadingWhitespace&&hb.test(f)&&p.push(b.createTextNode(hb.exec(f)[0])),!k.tbody){f="table"!==i||kb.test(f)?""!==l[1]||kb.test(f)?0:h:h.firstChild,e=f&&f.childNodes.length;while(e--)m.nodeName(j=f.childNodes[e],"tbody")&&!j.childNodes.length&&f.removeChild(j)}m.merge(p,h.childNodes),h.textContent="";while(h.firstChild)h.removeChild(h.firstChild);h=o.lastChild}else p.push(b.createTextNode(f));h&&o.removeChild(h),k.appendChecked||m.grep(ub(p,"input"),vb),q=0;while(f=p[q++])if((!d||-1===m.inArray(f,d))&&(g=m.contains(f.ownerDocument,f),h=ub(o.appendChild(f),"script"),g&&zb(h),c)){e=0;while(f=h[e++])ob.test(f.type||"")&&c.push(f)}return h=null,o},cleanData:function(a,b){for(var d,e,f,g,h=0,i=m.expando,j=m.cache,l=k.deleteExpando,n=m.event.special;null!=(d=a[h]);h++)if((b||m.acceptData(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)n[e]?m.event.remove(d,e):m.removeEvent(d,e,g.handle);j[f]&&(delete j[f],l?delete d[i]:typeof d.removeAttribute!==K?d.removeAttribute(i):d[i]=null,c.push(f))}}}),m.fn.extend({text:function(a){return V(this,function(a){return void 0===a?m.text(this):this.empty().append((this[0]&&this[0].ownerDocument||y).createTextNode(a))},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=wb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?m.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||m.cleanData(ub(c)),c.parentNode&&(b&&m.contains(c.ownerDocument,c)&&zb(ub(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&m.cleanData(ub(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&m.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return m.clone(this,a,b)})},html:function(a){return V(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(fb,""):void 0;if(!("string"!=typeof a||mb.test(a)||!k.htmlSerialize&&gb.test(a)||!k.leadingWhitespace&&hb.test(a)||rb[(jb.exec(a)||["",""])[1].toLowerCase()])){a=a.replace(ib,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(m.cleanData(ub(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,m.cleanData(ub(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,l=this.length,n=this,o=l-1,p=a[0],q=m.isFunction(p);if(q||l>1&&"string"==typeof p&&!k.checkClone&&nb.test(p))return this.each(function(c){var d=n.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(l&&(i=m.buildFragment(a,this[0].ownerDocument,!1,this),c=i.firstChild,1===i.childNodes.length&&(i=c),c)){for(g=m.map(ub(i,"script"),xb),f=g.length;l>j;j++)d=i,j!==o&&(d=m.clone(d,!0,!0),f&&m.merge(g,ub(d,"script"))),b.call(this[j],d,j);if(f)for(h=g[g.length-1].ownerDocument,m.map(g,yb),j=0;f>j;j++)d=g[j],ob.test(d.type||"")&&!m._data(d,"globalEval")&&m.contains(h,d)&&(d.src?m._evalUrl&&m._evalUrl(d.src):m.globalEval((d.text||d.textContent||d.innerHTML||"").replace(qb,"")));i=c=null}return this}}),m.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){m.fn[a]=function(a){for(var c,d=0,e=[],g=m(a),h=g.length-1;h>=d;d++)c=d===h?this:this.clone(!0),m(g[d])[b](c),f.apply(e,c.get());return this.pushStack(e)}});var Cb,Db={};function Eb(b,c){var d,e=m(c.createElement(b)).appendTo(c.body),f=a.getDefaultComputedStyle&&(d=a.getDefaultComputedStyle(e[0]))?d.display:m.css(e[0],"display");return e.detach(),f}function Fb(a){var b=y,c=Db[a];return c||(c=Eb(a,b),"none"!==c&&c||(Cb=(Cb||m("