Hystrix源码解析之——Hystrix 各类执行命令_observable.deferdoonterminate

在上一篇文章中最后提到了AbstractCommand是Hystrix中的一个重要的类，于是这篇文章中准备就这个类中关于命令执行的相关方法进行解析。本篇文章将先了解一下observe和toObservable。

一、AbstractCommand#observe 方法

1、com.netflix.hystrix.AbstractCommand#observe

public Observable<R> observe() {
    // us a ReplaySubject to buffer the eagerly subscribed-to Observable
    ReplaySubject<R> subject = ReplaySubject.create();
    // eagerly kick off subscription
    final Subscription sourceSubscription = toObservable().subscribe(subject);
    // return the subject that can be subscribed to later while the execution has already started
    return subject.doOnUnsubscribe(new Action0() {
        @Override
        public void call() {
            sourceSubscription.unsubscribe();
        }
    });
}

在AbstractCommand#observe方法中，首先是创建一个ReplaySubject对象，rx中的Subject既是一个Observable，也是一个Observer。接着调用toObservable()方法来获取以来执行的Observable，然后通过创建的ReplaySubject对象来订阅该Observable，启动Observable中的相关命令，同时返回ReplaySubject给后续观察者，用于订阅获取执行结果（ReplaySubject会推送所有来自原始Observable的时间给观察者，无论它们何时订阅的）。

这个AbstractCommand#observe方法其实主要依赖于toObservable()方法，HystrixExecutable接口中的execute和queue方法的实现则是依赖于AbstractCommand#observe方法，究其根本也是通过toObservable方法。

2、AbstractCommand#toObservable 方法

public Observable<R> toObservable() {
    final AbstractCommand<R> _cmd = this;
 
    //doOnCompleted handler already did all of the SUCCESS work
    //doOnError handler already did all of the FAILURE/TIMEOUT/REJECTION/BAD_REQUEST work
    // 命令结束时的回调方法，主要是命令调用结束后的清理工作
    // 根据 CommandState的执行状态，通过Metrics统计各种状态
    final Action0 terminateCommandCleanup = new Action0() {
 
        @Override
        public void call() {
            if (_cmd.commandState.compareAndSet(CommandState.OBSERVABLE_CHAIN_CREATED, CommandState.TERMINAL)) {
                handleCommandEnd(false); //user code never ran
            } else if (_cmd.commandState.compareAndSet(CommandState.USER_CODE_EXECUTED, CommandState.TERMINAL)) {
                handleCommandEnd(true); //user code did run
            }
        }
    };
 
    //mark the command as CANCELLED and store the latency (in addition to standard cleanup)
    // 命令被取消订阅的清理回调方法
    final Action0 unsubscribeCommandCleanup = new Action0() {
        @Override
        public void call() {
            if (_cmd.commandState.compareAndSet(CommandState.OBSERVABLE_CHAIN_CREATED, CommandState.UNSUBSCRIBED)) {
                if (!_cmd.executionResult.containsTerminalEvent()) {
                    _cmd.eventNotifier.markEvent(HystrixEventType.CANCELLED, _cmd.commandKey);
                    try {
                        executionHook.onUnsubscribe(_cmd);
                    } catch (Throwable hookEx) {
                        logger.warn("Error calling HystrixCommandExecutionHook.onUnsubscribe", hookEx);
                    }
                    _cmd.executionResultAtTimeOfCancellation = _cmd.executionResult
                            .addEvent((int) (System.currentTimeMillis() - _cmd.commandStartTimestamp), HystrixEventType.CANCELLED);
                }
                handleCommandEnd(false); //user code never ran
            } else if (_cmd.commandState.compareAndSet(CommandState.USER_CODE_EXECUTED, CommandState.UNSUBSCRIBED)) {
                if (!_cmd.executionResult.containsTerminalEvent()) {
                    _cmd.eventNotifier.markEvent(HystrixEventType.CANCELLED, _cmd.commandKey);
                    try {
                        executionHook.onUnsubscribe(_cmd);
                    } catch (Throwable hookEx) {
                        logger.warn("Error calling HystrixCommandExecutionHook.onUnsubscribe", hookEx);
                    }
                    _cmd.executionResultAtTimeOfCancellation = _cmd.executionResult
                            .addEvent((int) (System.currentTimeMillis() - _cmd.commandStartTimestamp), HystrixEventType.CANCELLED);
                }
                handleCommandEnd(true); //user code did run
            }
        }
    };
    // 构建执行命令，封装断路器、资源隔离逻辑
    final Func0<Observable<R>> applyHystrixSemantics = new Func0<Observable<R>>() {
        @Override
        public Observable<R> call() {
            if (commandState.get().equals(CommandState.UNSUBSCRIBED)) {
                return Observable.never();
            }
            return applyHystrixSemantics(_cmd);
        }
    };
 
    final Func1<R, R> wrapWithAllOnNextHooks = new Func1<R, R>() {
        @Override
        public R call(R r) {
            R afterFirstApplication = r;
 
            try {
                afterFirstApplication = executionHook.onComplete(_cmd, r);
            } catch (Throwable hookEx) {
                logger.warn("Error calling HystrixCommandExecutionHook.onComplete", hookEx);
            }
 
            try {
                return executionHook.onEmit(_cmd, afterFirstApplication);
            } catch (Throwable hookEx) {
                logger.warn("Error calling HystrixCommandExecutionHook.onEmit", hookEx);
                return afterFirstApplication;
            }
        }
    };
 
    final Action0 fireOnCompletedHook = new Action0() {
        @Override
        public void call() {
            try {
                executionHook.onSuccess(_cmd);
            } catch (Throwable hookEx) {
                logger.warn("Error calling HystrixCommandExecutionHook.onSuccess", hookEx);
            }
        }
    };
 
    return Observable.defer(new Func0<Observable<R>>() {
        @Override
        public Observable<R> call() {
             /* this is a stateful object so can only be used once */
             // 执行状态转换有误时，抛出异常
            if (!commandState.compareAndSet(CommandState.NOT_STARTED, CommandState.OBSERVABLE_CHAIN_CREATED)) {
                IllegalStateException ex = new IllegalStateException("This instance can only be executed once. Please instantiate a new instance.");
                //TODO make a new error type for this
                // 命令多次被执行，抛出异常
                throw new HystrixRuntimeException(FailureType.BAD_REQUEST_EXCEPTION, _cmd.getClass(), getLogMessagePrefix() + " command executed multiple times - this is not permitted.", ex, null);
            }
            // 记录命令开始时间
            commandStartTimestamp = System.currentTimeMillis();
 
            if (properties.requestLogEnabled().get()) {
                // log this command execution regardless of what happened
                if (currentRequestLog != null) {
                    currentRequestLog.addExecutedCommand(_cmd);
                }
            }
 
            final boolean requestCacheEnabled = isRequestCachingEnabled();
            final String cacheKey = getCacheKey();
 
            /* try from cache first */
            // 尝试从环境中获取结果
            if (requestCacheEnabled) {
                HystrixCommandResponseFromCache<R> fromCache = (HystrixCommandResponseFromCache<R>) requestCache.get(cacheKey);
                // 如果缓存不为空直接返回缓存结果
                if (fromCache != null) {
                    isResponseFromCache = true;
                    return handleRequestCacheHitAndEmitValues(fromCache, _cmd);
                }
            }
            // 构建执行命令的Observable
            Observable<R> hystrixObservable =
                    Observable.defer(applyHystrixSemantics)
                            .map(wrapWithAllOnNextHooks);
 
            Observable<R> afterCache;
 
            // put in cache
            // 将Observable封装成HystrixCachedObservable，然后方法哦缓存中
            if (requestCacheEnabled && cacheKey != null) {
                // wrap it for caching
                HystrixCachedObservable<R> toCache = HystrixCachedObservable.from(hystrixObservable, _cmd);
                HystrixCommandResponseFromCache<R> fromCache = (HystrixCommandResponseFromCache<R>) requestCache.putIfAbsent(cacheKey, toCache);
                if (fromCache != null) {
                    // another thread beat us so we'll use the cached value instead
                    toCache.unsubscribe();
                    isResponseFromCache = true;
                    return handleRequestCacheHitAndEmitValues(fromCache, _cmd);
                } else {
                    // we just created an ObservableCommand so we cast and return it
                    afterCache = toCache.toObservable();
                }
            } else {
                afterCache = hystrixObservable;
            }
 
            return afterCache
                    .doOnTerminate(terminateCommandCleanup)     // perform cleanup once (either on normal terminal state (this line), or unsubscribe (next line))
                    .doOnUnsubscribe(unsubscribeCommandCleanup) // perform cleanup once
                    .doOnCompleted(fireOnCompletedHook);
        }
    });
}

AbstractCommand#toObservable 方法的代码还是蛮复杂的，在这里就来对其进行一一解析：

1. 首先通过Observable#defer方法构建并返回Observable。以Observable#defer方式声明的Observable只有当观察者订阅才会真正开始创建，并且为每一个观察创建一个新的Observable，这也就保证了AbstractCommand#toObservable方法返回的Observable对象是纯净的，并没有开始执行命令。
2. 在构建Observable对象的过程中，先通过commandState查看当前的命令执行状态，保证命令未开始并且每条命令只能执行一次。
3. 如果允许请求缓存存在的话，将尝试从缓存中获取对应的执行结果，并将其直接返回。
4. 如果无法获取缓存，将通过applyHystrixSemantics方法构建用于返回的Observable。
5. 如果允许请求缓存，将Observable放置到缓存中用于下一次调用。
6. 最后为返回到Observable对象添加定义好的回调方法。

在以上的流程中需要关注两点，一个是HystrixRequestCache，它里面封装了缓存Obsevable的逻辑，另一个则是applyHystrixSemantics回调方法，其内部封装了断路、资源隔离等核心断路器逻辑。

二、HystrixRequestCache 请求缓存

HystrixRequestCache是对Obsevable进行缓存操作，使用每个命令特有的cacheKey对Obsevable进行缓存，通过ConcurrentHashMap保存缓存结果以保证线程安全。

HystrixRequestCache中所缓存的并不是单纯的Obsevable 对象，而是被封装好的HystrixCachedObservable。在HystrixCachedObservable中，可通过ReplaySubject订阅需要缓存的Obsevable，保证了缓存的Obsevable 能够被多次执行，代码如下：

public class HystrixCachedObservable<R> {
    protected final Subscription originalSubscription;
    protected final Observable<R> cachedObservable;
    private volatile int outstandingSubscriptions = 0;
 
    protected HystrixCachedObservable(final Observable<R> originalObservable) {
        // 使用 ReplaySubject 订阅原始的 Observable，并返回ReplaySubjec
        // 保证其从缓存中取出后，订阅依然能够接收对应事件，命令依然能执行
        ReplaySubject<R> replaySubject = ReplaySubject.create();
        this.originalSubscription = originalObservable
                .subscribe(replaySubject);
 
        this.cachedObservable = replaySubject
                .doOnUnsubscribe(new Action0() {
                    @Override
                    public void call() {
                        outstandingSubscriptions--;
                        if (outstandingSubscriptions == 0) {
                            originalSubscription.unsubscribe();
                        }
                    }
                })
                .doOnSubscribe(new Action0() {
                    @Override
                    public void call() {
                        outstandingSubscriptions++;
                    }
                });
    }
    // 对象转换
    public static <R> HystrixCachedObservable<R> from(Observable<R> o, AbstractCommand<R> originalCommand) {
        return new HystrixCommandResponseFromCache<R>(o, originalCommand);
    }
    // 将Observable对象转换成HystrixCachedObservable
    public static <R> HystrixCachedObservable<R> from(Observable<R> o) {
        return new HystrixCachedObservable<R>(o);
    }
 
    public Observable<R> toObservable() {
        return cachedObservable;
    }
    // 取消订阅
    public void unsubscribe() {
        originalSubscription.unsubscribe();
    }
}

三、applyHystrixSemantics回调方法

在applyHystrixSemantics回调方法中声明了Observable，它主要是用于判断断路器是否打开，以及尝试获取信号量用于执行命令（仅在信号量隔离模式下生效），代码如下：

private Observable<R> applyHystrixSemantics(final AbstractCommand<R> _cmd) {
    // mark that we're starting execution on the ExecutionHook
    // if this hook throws an exception, then a fast-fail occurs with no fallback.  No state is left inconsistent
    // 标记在ExecutionHook中执行
    executionHook.onStart(_cmd);
 
    /* determine if we're allowed to execute */
    // 判断 HystrixCircuitBreaker 命令是否可以执行
    if (circuitBreaker.allowRequest()) {
        // 获取信号量
        final TryableSemaphore executionSemaphore = getExecutionSemaphore();
        final AtomicBoolean semaphoreHasBeenReleased = new AtomicBoolean(false);
        // 释放信号量的回调方法
        final Action0 singleSemaphoreRelease = new Action0() {
            @Override
            public void call() {
                if (semaphoreHasBeenReleased.compareAndSet(false, true)) {
                    executionSemaphore.release();
                }
            }
        };
        // 标记异常的回调方法，对异常进行推送
        final Action1<Throwable> markExceptionThrown = new Action1<Throwable>() {
            @Override
            public void call(Throwable t) {
                eventNotifier.markEvent(HystrixEventType.EXCEPTION_THROWN, commandKey);
            }
        };
        // 尝试获取信号量
        if (executionSemaphore.tryAcquire()) {
            try {
                /* used to track userThreadExecutionTime */
                // 标记 executionResult 开始时间
                executionResult = executionResult.setInvocationStartTime(System.currentTimeMillis());
                // 获取执行命令的 Observable
                return executeCommandAndObserve(_cmd)
                        .doOnError(markExceptionThrown)
                        .doOnTerminate(singleSemaphoreRelease)
                        .doOnUnsubscribe(singleSemaphoreRelease);
            } catch (RuntimeException e) {
                return Observable.error(e);
            }
        } else {
            return handleSemaphoreRejectionViaFallback();
        }
    } else {
        return handleShortCircuitViaFallback();
    }
}

AbstractCommand#applyHystrixSemantics 方法中，首先是通过断路器HystrixCircuitBreaker 检查链路中的断路器是否开启，如果开启的话，执行断路器失败的逻辑 AbstractCommand#handleShortCircuitViaFallback 方法。如果断路器未打开，将会尝试获取信号量。如果不能获取信号量，那么执行信号量失败逻辑 AbstractCommand#handleSemaphoreRejectionViaFallback 方法。当上述校验都通过的时候，才会执行回调操作，该回调操作在命令执行结束后以及取消订阅的时候释放信号量。

在解析 AbstractCommand#executeCommandAndObserve 方法之前，先了解一下 ExecutionResult ，它是用来记录命令执行中各种状态的类，代码如下：

public class ExecutionResult {
    private final EventCounts eventCounts;
    private final Exception failedExecutionException;   // 执行失败的时候抛出的异常
    private final Exception executionException;         // 执行异常
    private final long startTimestamp;                  // 开始执行命令的时间
    private final int executionLatency;                 // run() 方法执行时间，即被 HystrixCommand包装的方法
    private final int userThreadLatency; //time elapsed between caller thread submitting request and response being visible to it
    private final boolean executionOccurred;
    private final boolean isExecutedInThread;
    private final HystrixCollapserKey collapserKey;
 
    private static final HystrixEventType[] ALL_EVENT_TYPES = HystrixEventType.values();
    private static final int NUM_EVENT_TYPES = ALL_EVENT_TYPES.length;
    private static final BitSet EXCEPTION_PRODUCING_EVENTS = new BitSet(NUM_EVENT_TYPES);
    private static final BitSet TERMINAL_EVENTS = new BitSet(NUM_EVENT_TYPES);
 
    static {
        for (HystrixEventType eventType: HystrixEventType.EXCEPTION_PRODUCING_EVENT_TYPES) {
            EXCEPTION_PRODUCING_EVENTS.set(eventType.ordinal());
        }
 
        for (HystrixEventType eventType: HystrixEventType.TERMINAL_EVENT_TYPES) {
            TERMINAL_EVENTS.set(eventType.ordinal());
        }
    }
    // 省略部分代码
}

通过 ExecutionResult来记录 HystrixCommand在不同执行阶段的状态和相关记录，用于后续的统计和分析。AbstractCommand#applyHystrixSemantics 方法最后将执行AbstractCommand#executeCommandAndObserve 方法为命令的配置执行异常回调方法，从而为命令的执行保航护驾。

四、AbstractCommand#executeCommandAndObserve 方法

AbstractCommand#executeCommandAndObserve方法主要用于为执行命令的 Observable 配置执行失败的回调方法，对执行失败的结果进行记录和处理。代码如下：

private Observable<R> executeCommandAndObserve(final AbstractCommand<R> _cmd) {
    final HystrixRequestContext currentRequestContext = HystrixRequestContext.getContextForCurrentThread();
    // 标记命令开始执行的回调方法
    final Action1<R> markEmits = new Action1<R>() {
        @Override
        public void call(R r) {
            if (shouldOutputOnNextEvents()) {
                executionResult = executionResult.addEvent(HystrixEventType.EMIT);
                eventNotifier.markEvent(HystrixEventType.EMIT, commandKey);
            }
            if (commandIsScalar()) {
                long latency = System.currentTimeMillis() - executionResult.getStartTimestamp();
                eventNotifier.markCommandExecution(getCommandKey(), properties.executionIsolationStrategy().get(), (int) latency, executionResult.getOrderedList());
                eventNotifier.markEvent(HystrixEventType.SUCCESS, commandKey);
                executionResult = executionResult.addEvent((int) latency, HystrixEventType.SUCCESS);
                circuitBreaker.markSuccess();
            }
        }
    };
    // 标记命令执行结束的回调方法
    final Action0 markOnCompleted = new Action0() {
        @Override
        public void call() {
            if (!commandIsScalar()) {
                long latency = System.currentTimeMillis() - executionResult.getStartTimestamp();
                eventNotifier.markCommandExecution(getCommandKey(), properties.executionIsolationStrategy().get(), (int) latency, executionResult.getOrderedList());
                eventNotifier.markEvent(HystrixEventType.SUCCESS, commandKey);
                executionResult = executionResult.addEvent((int) latency, HystrixEventType.SUCCESS);
                circuitBreaker.markSuccess();
            }
        }
    };
    // 失败回滚逻辑
    final Func1<Throwable, Observable<R>> handleFallback = new Func1<Throwable, Observable<R>>() {
        @Override
        public Observable<R> call(Throwable t) {
            Exception e = getExceptionFromThrowable(t);
            executionResult = executionResult.setExecutionException(e);
            if (e instanceof RejectedExecutionException) {
                return handleThreadPoolRejectionViaFallback(e);
            } else if (t instanceof HystrixTimeoutException) {
                return handleTimeoutViaFallback();
            } else if (t instanceof HystrixBadRequestException) {
                return handleBadRequestByEmittingError(e);
            } else {
                /*
                 * Treat HystrixBadRequestException from ExecutionHook like a plain HystrixBadRequestException.
                 */
                if (e instanceof HystrixBadRequestException) {
                    eventNotifier.markEvent(HystrixEventType.BAD_REQUEST, commandKey);
                    return Observable.error(e);
                }
 
                return handleFailureViaFallback(e);
            }
        }
    };
    // 设置请求上下文
    final Action1<Notification<? super R>> setRequestContext = new Action1<Notification<? super R>>() {
        @Override
        public void call(Notification<? super R> rNotification) {
            setRequestContextIfNeeded(currentRequestContext);
        }
    };
 
    Observable<R> execution;
    if (properties.executionTimeoutEnabled().get()) {
        execution = executeCommandWithSpecifiedIsolation(_cmd)
                .lift(new HystrixObservableTimeoutOperator<R>(_cmd));
    } else {
        execution = executeCommandWithSpecifiedIsolation(_cmd);
    }
 
    return execution.doOnNext(markEmits)
            .doOnCompleted(markOnCompleted)
            .onErrorResumeNext(handleFallback)
            .doOnEach(setRequestContext);
}

在AbstractCommand#executeCommandAndObserve方法中，handleFallback失败回滚回调方法根据执行过程中抛出异常的不同调用不同的方法进行处理。对线程获取失败处理调用 AbstractCommand#handleThreadPoolRejectionViaFallback 方法超时处理调用 AbstractCommand#handleTimeoutViaFallback 方法，远程调用处理失败调用 AbstractCommand#handleBadRequestByEmittingError 方法。Hystrix 自身异常处理调用AbstractCommand#handleFailureViaFallback 方法。此外在 AbstractCommand#applyHystrixSemantics 方法中还有断路失败处理AbstractCommand#handleShortCircuitViaFallback方法和获取信号量失败处理AbstractCommand#handleSemaphoreRejectionViaFallback 方法。至此已经了解了不同执行失败结果将导致不同的调用链路，这在后续还会进行解析。

AbstractCommand#executeCommandAndObserve方法的最后，有调用 AbstractCommand#executeCommandWithSpecifiedIsolation 方法来为命令的执行配置资源隔离和添加超时控制。

五、executeCommandWithSpecifiedIsolation方法

AbstractCommand#executeCommandWithSpecifiedIsolation 方法为命令构成了隔离的执行环境，提供了两中隔离方式：线程隔离和信号量隔离。如果Hystrix配置中开启了超时配置，还会通过Observable#lift 方法来将现有的Observable 转化成添加了超时的Observable对象。

AbstractCommand#executeCommandWithSpecifiedIsolation 方法根据配置中的隔离策略对命令执行采用了不同的资源隔离方式：ExecutionIsolationStrategy.THREA（线程隔离的方式）、ExecutionIsolationStrategy.SEMAPHORE（信号量隔离方式）。代码如下：

    if (properties.executionIsolationStrategy().get() == ExecutionIsolationStrategy.THREAD) {
        // mark that we are executing in a thread (even if we end up being rejected we still were a THREAD execution and not SEMAPHORE)
        return Observable.defer(new Func0<Observable<R>>() {
            @Override
            public Observable<R> call() {
                executionResult = executionResult.setExecutionOccurred();
                if (!commandState.compareAndSet(CommandState.OBSERVABLE_CHAIN_CREATED, CommandState.USER_CODE_EXECUTED)) {
                    return Observable.error(new IllegalStateException("execution attempted while in state : " + commandState.get().name()));
                }
                // 标记命令通过线程隔离资源执行
                metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.THREAD);
 
                if (isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT) {
                    // the command timed out in the wrapping thread so we will return immediately
                    // and not increment any of the counters below or other such logic
                    return Observable.error(new RuntimeException("timed out before executing run()"));
                }
                if (threadState.compareAndSet(ThreadState.NOT_USING_THREAD, ThreadState.STARTED)) {
                    //we have not been unsubscribed, so should proceed
                    // 标记线程已经执行
                    HystrixCounters.incrementGlobalConcurrentThreads();
                    threadPool.markThreadExecution();
                    // store the command that is being run
                    endCurrentThreadExecutingCommand = Hystrix.startCurrentThreadExecutingCommand(getCommandKey());
                    // 记录是使用线程隔离执行
                    executionResult = executionResult.setExecutedInThread();
                    /**
                     * If any of these hooks throw an exception, then it appears as if the actual execution threw an error
                     */
                    try {
                        executionHook.onThreadStart(_cmd);
                        executionHook.onRunStart(_cmd);
                        executionHook.onExecutionStart(_cmd);
                        return getUserExecutionObservable(_cmd);
                    } catch (Throwable ex) {
                        return Observable.error(ex);
                    }
                } else {
                    //command has already been unsubscribed, so return immediately
                    return Observable.error(new RuntimeException("unsubscribed before executing run()"));
                }
            }
        }).doOnTerminate(new Action0() {
            @Override
            public void call() {
                if (threadState.compareAndSet(ThreadState.STARTED, ThreadState.TERMINAL)) {
                    handleThreadEnd(_cmd);
                }
                if (threadState.compareAndSet(ThreadState.NOT_USING_THREAD, ThreadState.TERMINAL)) {
                    //if it was never started and received terminal, then no need to clean up (I don't think this is possible)
                }
                //if it was unsubscribed, then other cleanup handled it
            }
        // 
        }).doOnUnsubscribe(new Action0() {
            @Override
            public void call() {
                if (threadState.compareAndSet(ThreadState.STARTED, ThreadState.UNSUBSCRIBED)) {
                    handleThreadEnd(_cmd);
                }
                if (threadState.compareAndSet(ThreadState.NOT_USING_THREAD, ThreadState.UNSUBSCRIBED)) {
                    //if it was never started and was cancelled, then no need to clean up
                }
                //if it was terminal, then other cleanup handled it
            }
        // 指定命令在那个线程执行
        }).subscribeOn(threadPool.getScheduler(new Func0<Boolean>() {
            @Override
            public Boolean call() {
                return properties.executionIsolationThreadInterruptOnTimeout().get() && _cmd.isCommandTimedOut.get() == TimedOutStatus.TIMED_OUT;
            }
        }));
    } else {
        return Observable.defer(new Func0<Observable<R>>() {
            @Override
            public Observable<R> call() {
                executionResult = executionResult.setExecutionOccurred();
                if (!commandState.compareAndSet(CommandState.OBSERVABLE_CHAIN_CREATED, CommandState.USER_CODE_EXECUTED)) {
                    return Observable.error(new IllegalStateException("execution attempted while in state : " + commandState.get().name()));
                }
                // 标记命令是通过信号量执行
                metrics.markCommandStart(commandKey, threadPoolKey, ExecutionIsolationStrategy.SEMAPHORE);
                // semaphore isolated
                // store the command that is being run
                endCurrentThreadExecutingCommand = Hystrix.startCurrentThreadExecutingCommand(getCommandKey());
                try {
                    executionHook.onRunStart(_cmd);
                    executionHook.onExecutionStart(_cmd);
                    return getUserExecutionObservable(_cmd);  //the getUserExecutionObservable method already wraps sync exceptions, so this shouldn't throw
                } catch (Throwable ex) {
                    //If the above hooks throw, then use that as the result of the run method
                    return Observable.error(ex);
                }
            }
        });
    }
}

六、AbstractCommand#getExecutionObservable

private Observable<R> getUserExecutionObservable(final AbstractCommand<R> _cmd) {
    Observable<R> userObservable;
 
    try {
        userObservable = getExecutionObservable();
    } catch (Throwable ex) {
        // the run() method is a user provided implementation so can throw instead of using Observable.onError
        // so we catch it here and turn it into Observable.error
        userObservable = Observable.error(ex);
    }
 
    return userObservable
            .lift(new ExecutionHookApplication(_cmd))
            .lift(new DeprecatedOnRunHookApplication(_cmd));
}

在 AbstractCommand#getUserExecutionObservable 方法里会调用 HystrixCommand#getExecutionObservable 方法，HystrixCommand#getExecutionObservable方法代码如下：

@Override
final protected Observable<R> getExecutionObservable() {
    return Observable.defer(new Func0<Observable<R>>() {
        @Override
        public Observable<R> call() {
            try {
                // 执行被包装到 HystrixCommand 的远程调用逻辑中
                return Observable.just(run());
            } catch (Throwable ex) {
                return Observable.error(ex);
            }
        }
    }).doOnSubscribe(new Action0() {
        @Override
        public void call() {
            // Save thread on which we get subscribed so that we can interrupt it later if needed
            executionThread.set(Thread.currentThread());
        }
    });
}

在上面的方法中，run方法也会被延迟到子类中实现，在高级应用中将尝试直接继承HystrixCommand和HystrixObservableCommand来构建对应的HystrixCommand，在HystrixCommand的默认实现GenericCommand中， run方法是通过创建HystrixCommand传入的CommandActions提供具体的实现。CommandActions持有commandAction和fallbackAction分别对应HystrixCommand中的远程调用和失败回滚方法。

七、同步、异步执行命令

在了解了HystrixObservable中的两个关键方法在AbstractCommand中的实现后，接下来就来简单看看HystrixCommand中execute同步执行命令和queue异步执行命令的相关实现，代码如下：

1.com.netflix.hystrix.HystrixCommand#execute

execute方法通过queue方法获取到Future，使用queue().get()方法获取到命令的执行结果，它将一直阻塞线程知道与执行结果返回。

public R execute() {
    try {
        return queue().get();
    } catch (Exception e) {
        throw Exceptions.sneakyThrow(decomposeException(e));
    }
}

2.com.netflix.hystrix.HystrixCommand#queue

queue方法中将 AbstractCommand#toObservable 获取到的Observable通过toBlocking()方法转化为具备阻塞功能的Blocking Observable，再通过toFuture()方法获取到能够执行的run抽象方法Future，最后通过Futere的到正在异步执行的命令执行结果。

 public Future<R> queue() {
 
     final Future<R> delegate = toObservable().toBlocking().toFuture();
   
     final Future<R> f = new Future<R>() {
 
         @Override
         public boolean cancel(boolean mayInterruptIfRunning) {
             if (delegate.isCancelled()) {
                 return false;
             }
 
             if (HystrixCommand.this.getProperties().executionIsolationThreadInterruptOnFutureCancel().get()) {
                 interruptOnFutureCancel.compareAndSet(false, mayInterruptIfRunning);
          }
 
             final boolean res = delegate.cancel(interruptOnFutureCancel.get());
 
             if (!isExecutionComplete() && interruptOnFutureCancel.get()) {
                 final Thread t = executionThread.get();
                 if (t != null && !t.equals(Thread.currentThread())) {
                     t.interrupt();
                 }
             }
 
             return res;
}
 
         @Override
         public boolean isCancelled() {
             return delegate.isCancelled();
}
 
         @Override
         public boolean isDone() {
             return delegate.isDone();
}
 
         @Override
         public R get() throws InterruptedException, ExecutionException {
             return delegate.get();
         }
 
         @Override
         public R get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
             return delegate.get(timeout, unit);
         }
       
     };
 
     /* special handling of error states that throw immediately */
     if (f.isDone()) {
         try {
             f.get();
             return f;
         } catch (Exception e) {
             Throwable t = decomposeException(e);
             if (t instanceof HystrixBadRequestException) {
                 return f;
             } else if (t instanceof HystrixRuntimeException) {
                 HystrixRuntimeException hre = (HystrixRuntimeException) t;
                 switch (hre.getFailureType()) {
      case COMMAND_EXCEPTION:
      case TIMEOUT:
         // we don't throw these types from queue() only from queue().get() as they are execution errors
         return f;
      default:
         // these are errors we throw from queue() as they as rejection type errors
         throw hre;
      }
             } else {
                 throw Exceptions.sneakyThrow(t);
             }
         }
     }
 
     return f;
 }