001/* -*- mode: Java; c-basic-offset: 2; indent-tabs-mode: nil; coding: utf-8-unix -*- 002 * 003 * Copyright © 2017-2018 microBean. 004 * 005 * Licensed under the Apache License, Version 2.0 (the "License"); 006 * you may not use this file except in compliance with the License. 007 * You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 014 * implied. See the License for the specific language governing 015 * permissions and limitations under the License. 016 */ 017package org.microbean.kubernetes.controller; 018 019import java.io.Closeable; 020import java.io.IOException; 021 022import java.time.Duration; 023 024import java.time.temporal.ChronoUnit; 025 026import java.util.ArrayList; 027import java.util.Collection; 028import java.util.Collections; 029import java.util.Objects; 030import java.util.Map; 031 032import java.util.concurrent.Executors; 033import java.util.concurrent.ScheduledExecutorService; 034import java.util.concurrent.ScheduledFuture; 035import java.util.concurrent.TimeUnit; 036 037import java.util.logging.Level; 038import java.util.logging.Logger; 039 040import io.fabric8.kubernetes.client.DefaultKubernetesClient; // for javadoc only 041import io.fabric8.kubernetes.client.KubernetesClientException; 042import io.fabric8.kubernetes.client.Watcher; 043 044import io.fabric8.kubernetes.client.dsl.Listable; 045import io.fabric8.kubernetes.client.dsl.VersionWatchable; 046 047import io.fabric8.kubernetes.api.model.HasMetadata; 048import io.fabric8.kubernetes.api.model.ObjectMeta; 049import io.fabric8.kubernetes.api.model.KubernetesResourceList; 050import io.fabric8.kubernetes.api.model.ListMeta; 051 052import net.jcip.annotations.GuardedBy; 053import net.jcip.annotations.ThreadSafe; 054 055import org.microbean.development.annotation.NonBlocking; 056 057/** 058 * A pump of sorts that continuously "pulls" logical events out of 059 * Kubernetes and {@linkplain EventCache#add(Object, AbstractEvent.Type, 060 * HasMetadata) adds them} to an {@link EventCache} so as to logically 061 * "reflect" the contents of Kubernetes into the cache. 062 * 063 * <h2>Thread Safety</h2> 064 * 065 * <p>Instances of this class are safe for concurrent use by multiple 066 * {@link Thread}s.</p> 067 * 068 * @param <T> a type of Kubernetes resource 069 * 070 * @author <a href="https://about.me/lairdnelson" 071 * target="_parent">Laird Nelson</a> 072 * 073 * @see EventCache 074 */ 075@ThreadSafe 076public class Reflector<T extends HasMetadata> implements Closeable { 077 078 079 /* 080 * Instance fields. 081 */ 082 083 084 /** 085 * The operation that was supplied at construction time. 086 * 087 * <p>This field is never {@code null}.</p> 088 * 089 * <p>It is guaranteed that the value of this field may be 090 * assignable to a reference of type {@link Listable Listable<? 091 * extends KubernetesResourceList>} or to a reference of type 092 * {@link VersionWatchable VersionWatchable<? extends Closeable, 093 * Watcher<T>>}.</p> 094 * 095 * @see Listable 096 * 097 * @see VersionWatchable 098 */ 099 private final Object operation; 100 101 /** 102 * The resource version 103 */ 104 private volatile Object lastSynchronizationResourceVersion; 105 106 private final ScheduledExecutorService synchronizationExecutorService; 107 108 @GuardedBy("this") 109 private ScheduledFuture<?> synchronizationTask; 110 111 private final boolean shutdownSynchronizationExecutorServiceOnClose; 112 113 private final Duration synchronizationInterval; 114 115 @GuardedBy("this") 116 private Closeable watch; 117 118 @GuardedBy("itself") 119 private final EventCache<T> eventCache; 120 121 /** 122 * A {@link Logger} for use by this {@link Reflector}. 123 * 124 * <p>This field is never {@code null}.</p> 125 * 126 * @see #createLogger() 127 */ 128 protected final Logger logger; 129 130 131 /* 132 * Constructors. 133 */ 134 135 136 /** 137 * Creates a new {@link Reflector}. 138 * 139 * @param <X> a type that is both an appropriate kind of {@link 140 * Listable} and {@link VersionWatchable}, such as the kind of 141 * operation returned by {@link 142 * DefaultKubernetesClient#configMaps()} and the like 143 * 144 * @param operation a {@link Listable} and a {@link 145 * VersionWatchable} that can report information from a Kubernetes 146 * cluster; must not be {@code null} 147 * 148 * @param eventCache an {@link EventCache} <strong>that will be 149 * synchronized on</strong> and into which {@link Event}s will be 150 * logically "reflected"; must not be {@code null} 151 * 152 * @exception NullPointerException if {@code operation} or {@code 153 * eventCache} is {@code null} 154 * 155 * @exception IllegalStateException if the {@link #createLogger()} 156 * method returns {@code null} 157 * 158 * @see #Reflector(Listable, EventCache, ScheduledExecutorService, 159 * Duration) 160 * 161 * @see #start() 162 */ 163 @SuppressWarnings("rawtypes") // kubernetes-client's implementations of KubernetesResourceList use raw types 164 public <X extends Listable<? extends KubernetesResourceList> & VersionWatchable<? extends Closeable, Watcher<T>>> Reflector(final X operation, 165 final EventCache<T> eventCache) { 166 this(operation, eventCache, null, null); 167 } 168 169 /** 170 * Creates a new {@link Reflector}. 171 * 172 * @param <X> a type that is both an appropriate kind of {@link 173 * Listable} and {@link VersionWatchable}, such as the kind of 174 * operation returned by {@link 175 * DefaultKubernetesClient#configMaps()} and the like 176 * 177 * @param operation a {@link Listable} and a {@link 178 * VersionWatchable} that can report information from a Kubernetes 179 * cluster; must not be {@code null} 180 * 181 * @param eventCache an {@link EventCache} <strong>that will be 182 * synchronized on</strong> and into which {@link Event}s will be 183 * logically "reflected"; must not be {@code null} 184 * 185 * @param synchronizationInterval a {@link Duration} representing 186 * the time in between one {@linkplain EventCache#synchronize() 187 * synchronization operation} and another; may be {@code null} in 188 * which case no synchronization will occur 189 * 190 * @exception NullPointerException if {@code operation} or {@code 191 * eventCache} is {@code null} 192 * 193 * @exception IllegalStateException if the {@link #createLogger()} 194 * method returns {@code null} 195 * 196 * @see #Reflector(Listable, EventCache, ScheduledExecutorService, 197 * Duration) 198 * 199 * @see #start() 200 */ 201 @SuppressWarnings("rawtypes") // kubernetes-client's implementations of KubernetesResourceList use raw types 202 public <X extends Listable<? extends KubernetesResourceList> & VersionWatchable<? extends Closeable, Watcher<T>>> Reflector(final X operation, 203 final EventCache<T> eventCache, 204 final Duration synchronizationInterval) { 205 this(operation, eventCache, null, synchronizationInterval); 206 } 207 208 /** 209 * Creates a new {@link Reflector}. 210 * 211 * @param <X> a type that is both an appropriate kind of {@link 212 * Listable} and {@link VersionWatchable}, such as the kind of 213 * operation returned by {@link 214 * DefaultKubernetesClient#configMaps()} and the like 215 * 216 * @param operation a {@link Listable} and a {@link 217 * VersionWatchable} that can report information from a Kubernetes 218 * cluster; must not be {@code null} 219 * 220 * @param eventCache an {@link EventCache} <strong>that will be 221 * synchronized on</strong> and into which {@link Event}s will be 222 * logically "reflected"; must not be {@code null} 223 * 224 * @param synchronizationExecutorService a {@link 225 * ScheduledExecutorService} to be used to tell the supplied {@link 226 * EventCache} to {@linkplain EventCache#synchronize() synchronize} 227 * on a schedule; may be {@code null} in which case no 228 * synchronization will occur 229 * 230 * @param synchronizationInterval a {@link Duration} representing 231 * the time in between one {@linkplain EventCache#synchronize() 232 * synchronization operation} and another; may be {@code null} in 233 * which case no synchronization will occur 234 * 235 * @exception NullPointerException if {@code operation} or {@code 236 * eventCache} is {@code null} 237 * 238 * @exception IllegalStateException if the {@link #createLogger()} 239 * method returns {@code null} 240 * 241 * @see #start() 242 */ 243 @SuppressWarnings("rawtypes") // kubernetes-client's implementations of KubernetesResourceList use raw types 244 public <X extends Listable<? extends KubernetesResourceList> & VersionWatchable<? extends Closeable, Watcher<T>>> Reflector(final X operation, 245 final EventCache<T> eventCache, 246 final ScheduledExecutorService synchronizationExecutorService, 247 final Duration synchronizationInterval) { 248 super(); 249 this.logger = this.createLogger(); 250 if (this.logger == null) { 251 throw new IllegalStateException("createLogger() == null"); 252 } 253 final String cn = this.getClass().getName(); 254 final String mn = "<init>"; 255 if (this.logger.isLoggable(Level.FINER)) { 256 this.logger.entering(cn, mn, new Object[] { operation, eventCache, synchronizationExecutorService, synchronizationInterval }); 257 } 258 Objects.requireNonNull(operation); 259 this.eventCache = Objects.requireNonNull(eventCache); 260 // TODO: research: maybe: operation.withField("metadata.resourceVersion", "0")? 261 this.operation = operation.withResourceVersion("0"); 262 this.synchronizationInterval = synchronizationInterval; 263 if (synchronizationExecutorService == null) { 264 if (synchronizationInterval == null) { 265 this.synchronizationExecutorService = null; 266 this.shutdownSynchronizationExecutorServiceOnClose = false; 267 } else { 268 this.synchronizationExecutorService = Executors.newScheduledThreadPool(1); 269 this.shutdownSynchronizationExecutorServiceOnClose = true; 270 } 271 } else { 272 this.synchronizationExecutorService = synchronizationExecutorService; 273 this.shutdownSynchronizationExecutorServiceOnClose = false; 274 } 275 if (this.logger.isLoggable(Level.FINER)) { 276 this.logger.exiting(cn, mn); 277 } 278 } 279 280 281 /* 282 * Instance methods. 283 */ 284 285 286 /** 287 * Returns a {@link Logger} that will be used for this {@link 288 * Reflector}. 289 * 290 * <p>This method never returns {@code null}.</p> 291 * 292 * <p>Overrides of this method must not return {@code null}.</p> 293 * 294 * @return a non-{@code null} {@link Logger} 295 */ 296 protected Logger createLogger() { 297 return Logger.getLogger(this.getClass().getName()); 298 } 299 300 /** 301 * Notionally closes this {@link Reflector} by terminating any 302 * {@link Thread}s that it has started and invoking the {@link 303 * #onClose()} method while holding this {@link Reflector}'s 304 * monitor. 305 * 306 * @exception IOException if an error occurs 307 * 308 * @see #onClose() 309 */ 310 @Override 311 public synchronized final void close() throws IOException { 312 final String cn = this.getClass().getName(); 313 final String mn = "close"; 314 if (this.logger.isLoggable(Level.FINER)) { 315 this.logger.entering(cn, mn); 316 } 317 try { 318 final ScheduledFuture<?> synchronizationTask = this.synchronizationTask; 319 if (synchronizationTask != null) { 320 synchronizationTask.cancel(false); 321 } 322 this.closeSynchronizationExecutorService(); 323 if (this.watch != null) { 324 this.watch.close(); 325 } 326 } finally { 327 this.onClose(); 328 } 329 if (this.logger.isLoggable(Level.FINER)) { 330 this.logger.exiting(cn, mn); 331 } 332 } 333 334 private synchronized final void closeSynchronizationExecutorService() { 335 final String cn = this.getClass().getName(); 336 final String mn = "closeSynchronizationExecutorService"; 337 if (this.logger.isLoggable(Level.FINER)) { 338 this.logger.entering(cn, mn); 339 } 340 if (this.synchronizationExecutorService != null && this.shutdownSynchronizationExecutorServiceOnClose) { 341 this.synchronizationExecutorService.shutdown(); 342 try { 343 if (!this.synchronizationExecutorService.awaitTermination(60L, TimeUnit.SECONDS)) { 344 this.synchronizationExecutorService.shutdownNow(); 345 if (!this.synchronizationExecutorService.awaitTermination(60L, TimeUnit.SECONDS)) { 346 this.synchronizationExecutorService.shutdownNow(); 347 } 348 } 349 } catch (final InterruptedException interruptedException) { 350 this.synchronizationExecutorService.shutdownNow(); 351 Thread.currentThread().interrupt(); 352 } 353 } 354 if (this.logger.isLoggable(Level.FINER)) { 355 this.logger.exiting(cn, mn); 356 } 357 } 358 359 private synchronized final void setUpSynchronization() { 360 final String cn = this.getClass().getName(); 361 final String mn = "setUpSynchronization"; 362 if (this.logger.isLoggable(Level.FINER)) { 363 this.logger.entering(cn, mn); 364 } 365 if (this.synchronizationExecutorService != null) { 366 367 final Duration synchronizationDuration = this.getSynchronizationInterval(); 368 final long seconds; 369 if (synchronizationDuration == null) { 370 seconds = 0L; 371 } else { 372 seconds = synchronizationDuration.get(ChronoUnit.SECONDS); 373 } 374 375 if (seconds > 0L) { 376 if (this.logger.isLoggable(Level.INFO)) { 377 this.logger.logp(Level.INFO, cn, mn, "Scheduling downstream synchronization every {0} seconds", Long.valueOf(seconds)); 378 } 379 final ScheduledFuture<?> job = this.synchronizationExecutorService.scheduleWithFixedDelay(() -> { 380 if (shouldSynchronize()) { 381 if (logger.isLoggable(Level.FINE)) { 382 logger.logp(Level.FINE, cn, mn, "Synchronizing event cache with its downstream consumers"); 383 } 384 synchronized (eventCache) { 385 eventCache.synchronize(); 386 } 387 } 388 }, 0L, seconds, TimeUnit.SECONDS); 389 assert job != null; 390 this.synchronizationTask = job; 391 } 392 393 } 394 if (this.logger.isLoggable(Level.FINER)) { 395 this.logger.exiting(cn, mn); 396 } 397 } 398 399 /** 400 * Returns whether, at any given moment, this {@link Reflector} 401 * should cause its {@link EventCache} to {@linkplain 402 * EventCache#synchronize() synchronize}. 403 * 404 * <h2>Design Notes</h2> 405 * 406 * <p>This code follows the Go code in the Kubernetes {@code 407 * client-go/tools/cache} package. One thing that becomes clear 408 * when looking at all of this through an object-oriented lens is 409 * that it is the {@link EventCache} (the {@code delta_fifo}, in the 410 * Go code) that is ultimately in charge of synchronizing. It is 411 * not clear why this is a function of a reflector. In an 412 * object-oriented world, perhaps the {@link EventCache} itself 413 * should be in charge of resynchronization schedules.</p> 414 * 415 * @return {@code true} if this {@link Reflector} should cause its 416 * {@link EventCache} to {@linkplain EventCache#synchronize() 417 * synchronize}; {@code false} otherwise 418 */ 419 protected boolean shouldSynchronize() { 420 final String cn = this.getClass().getName(); 421 final String mn = "shouldSynchronize"; 422 if (this.logger.isLoggable(Level.FINER)) { 423 this.logger.entering(cn, mn); 424 } 425 final boolean returnValue = this.synchronizationExecutorService != null; 426 if (this.logger.isLoggable(Level.FINER)) { 427 this.logger.exiting(cn, mn, Boolean.valueOf(returnValue)); 428 } 429 return returnValue; 430 } 431 432 private final Duration getSynchronizationInterval() { 433 return this.synchronizationInterval; 434 } 435 436 private final Object getLastSynchronizationResourceVersion() { 437 return this.lastSynchronizationResourceVersion; 438 } 439 440 private final void setLastSynchronizationResourceVersion(final Object resourceVersion) { 441 this.lastSynchronizationResourceVersion = resourceVersion; 442 } 443 444 /** 445 * Using the {@code operation} supplied at construction time, 446 * {@linkplain Listable#list() lists} appropriate Kubernetes 447 * resources, and then, on a separate {@link Thread}, {@linkplain 448 * VersionWatchable sets up a watch} on them, calling {@link 449 * EventCache#replace(Collection, Object)} and {@link 450 * EventCache#add(Object, AbstractEvent.Type, HasMetadata)} methods 451 * as appropriate. 452 * 453 * <p>The calling {@link Thread} is not blocked by invocations of 454 * this method.</p> 455 * 456 * @see #close() 457 */ 458 @NonBlocking 459 public synchronized final void start() { 460 final String cn = this.getClass().getName(); 461 final String mn = "start"; 462 if (this.logger.isLoggable(Level.FINER)) { 463 this.logger.entering(cn, mn); 464 } 465 if (this.watch == null) { 466 467 // Run a list operation, and get the resourceVersion of that list. 468 @SuppressWarnings("unchecked") 469 final KubernetesResourceList<? extends T> list = ((Listable<? extends KubernetesResourceList<? extends T>>)this.operation).list(); 470 assert list != null; 471 final ListMeta metadata = list.getMetadata(); 472 assert metadata != null; 473 final String resourceVersion = metadata.getResourceVersion(); 474 assert resourceVersion != null; 475 476 // Using the results of that list operation, do a full replace 477 // on the EventCache with them. 478 final Collection<? extends T> replacementItems; 479 final Collection<? extends T> items = list.getItems(); 480 if (items == null || items.isEmpty()) { 481 replacementItems = Collections.emptySet(); 482 } else { 483 replacementItems = Collections.unmodifiableCollection(new ArrayList<>(items)); 484 } 485 synchronized (eventCache) { 486 this.eventCache.replace(replacementItems, resourceVersion); 487 } 488 489 // Record the resource version we captured during our list 490 // operation. 491 this.setLastSynchronizationResourceVersion(resourceVersion); 492 493 // Now that we've vetted that our list operation works (i.e. no 494 // syntax errors, no connectivity problems) we can schedule 495 // resynchronizations if necessary. 496 this.setUpSynchronization(); 497 498 // Now that we've taken care of our list() operation, set up our 499 // watch() operation. 500 try { 501 @SuppressWarnings("unchecked") 502 final Closeable temp = ((VersionWatchable<? extends Closeable, Watcher<T>>)operation).withResourceVersion(resourceVersion).watch(new WatchHandler()); 503 assert temp != null; 504 this.watch = temp; 505 } finally { 506 this.closeSynchronizationExecutorService(); 507 } 508 } 509 if (this.logger.isLoggable(Level.FINER)) { 510 this.logger.exiting(cn, mn); 511 } 512 } 513 514 /** 515 * Invoked when {@link #close()} is invoked. 516 * 517 * <p>The default implementation of this method does nothing.</p> 518 */ 519 protected synchronized void onClose() { 520 521 } 522 523 524 /* 525 * Inner and nested classes. 526 */ 527 528 529 /** 530 * A {@link Watcher} of Kubernetes resources. 531 * 532 * @author <a href="https://about.me/lairdnelson" 533 * target="_parent">Laird Nelson</a> 534 * 535 * @see Watcher 536 */ 537 private final class WatchHandler implements Watcher<T> { 538 539 540 /* 541 * Constructors. 542 */ 543 544 545 /** 546 * Creates a new {@link WatchHandler}. 547 */ 548 private WatchHandler() { 549 super(); 550 final String cn = this.getClass().getName(); 551 final String mn = "<init>"; 552 if (logger.isLoggable(Level.FINER)) { 553 logger.entering(cn, mn); 554 logger.exiting(cn, mn); 555 } 556 } 557 558 559 /* 560 * Instance methods. 561 */ 562 563 564 /** 565 * Calls the {@link EventCache#add(Object, AbstractEvent.Type, 566 * HasMetadata)} method on the enclosing {@link Reflector}'s 567 * associated {@link EventCache} with information harvested from 568 * the supplied {@code resource}, and using an {@link Event.Type} 569 * selected appropriately given the supplied {@link 570 * Watcher.Action}. 571 * 572 * @param action the kind of Kubernetes event that happened; must 573 * not be {@code null} 574 * 575 * @param resource the {@link HasMetadata} object that was 576 * affected; must not be {@code null} 577 * 578 * @exception NullPointerException if {@code action} or {@code 579 * resource} was {@code null} 580 * 581 * @exception IllegalStateException if another error occurred 582 */ 583 @Override 584 public final void eventReceived(final Watcher.Action action, final T resource) { 585 final String cn = this.getClass().getName(); 586 final String mn = "eventReceived"; 587 if (logger.isLoggable(Level.FINER)) { 588 logger.entering(cn, mn, new Object[] { action, resource }); 589 } 590 Objects.requireNonNull(action); 591 Objects.requireNonNull(resource); 592 593 final ObjectMeta metadata = resource.getMetadata(); 594 assert metadata != null; 595 596 final Event.Type eventType; 597 598 switch (action) { 599 case ADDED: 600 eventType = Event.Type.ADDITION; 601 break; 602 case MODIFIED: 603 eventType = Event.Type.MODIFICATION; 604 break; 605 case DELETED: 606 eventType = Event.Type.DELETION; 607 break; 608 case ERROR: 609 // Uh...the Go code has: 610 // 611 // if event.Type == watch.Error { 612 // return apierrs.FromObject(event.Object) 613 // } 614 // 615 // Now, apierrs.FromObject is here: 616 // https://github.com/kubernetes/apimachinery/blob/kubernetes-1.9.2/pkg/api/errors/errors.go#L80-L88 617 // This is looking for a Status object. But 618 // WatchConnectionHandler will never forward on such a thing: 619 // https://github.com/fabric8io/kubernetes-client/blob/v3.1.8/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/internal/WatchConnectionManager.java#L246-L258 620 // 621 // So it follows that if by some chance we get here, resource 622 // will definitely be a HasMetadata. We go back to the Go 623 // code again, and remember that if the type is Error, the 624 // equivalent of this watch handler simply returns and goes home. 625 // 626 // Now, if we were to throw a RuntimeException here, which is 627 // the idiomatic equivalent of returning and going home, this 628 // would cause a watch reconnect: 629 // https://github.com/fabric8io/kubernetes-client/blob/v3.1.8/kubernetes-client/src/main/java/io/fabric8/kubernetes/client/dsl/internal/WatchConnectionManager.java#L159-L205 630 // ...up to the reconnect limit. 631 // 632 // ...which is fine, but I'm not sure that in an error case a 633 // WatchEvent will ever HAVE a HasMetadata as its payload. 634 // Which means MAYBE we'll never get here. But if we do, all 635 // we can do is throw a RuntimeException...which ends up 636 // reducing to the same case as the default case below, so we 637 // fall through. 638 default: 639 eventType = null; 640 throw new IllegalStateException(); 641 } 642 643 // Add an Event of the proper kind to our EventCache. 644 if (eventType != null) { 645 if (logger.isLoggable(Level.FINE)) { 646 logger.logp(Level.FINE, cn, mn, "Adding event to cache: {0} {1}", new Object[] { eventType, resource }); 647 } 648 synchronized (eventCache) { 649 eventCache.add(Reflector.this, eventType, resource); 650 } 651 } 652 653 // Record the most recent resource version we're tracking to be 654 // that of this last successful watch() operation. We set it 655 // earlier during a list() operation. 656 setLastSynchronizationResourceVersion(metadata.getResourceVersion()); 657 658 if (logger.isLoggable(Level.FINER)) { 659 logger.exiting(cn, mn); 660 } 661 } 662 663 /** 664 * Invoked when the Kubernetes client connection closes. 665 * 666 * @param exception any {@link KubernetesClientException} that 667 * caused this closing to happen; may be {@code null} 668 */ 669 @Override 670 public final void onClose(final KubernetesClientException exception) { 671 final String cn = this.getClass().getName(); 672 final String mn = "onClose"; 673 if (logger.isLoggable(Level.FINER)) { 674 logger.entering(cn, mn, exception); 675 } 676 if (exception != null && logger.isLoggable(Level.WARNING)) { 677 logger.logp(Level.WARNING, cn, mn, exception.getMessage(), exception); 678 } 679 if (logger.isLoggable(Level.FINER)) { 680 logger.exiting(cn, mn, exception); 681 } 682 } 683 684 } 685 686}