Резервное отключение - Hystrix

Я использую Hystrix в своей среде, связанной с Liferay и другими внешними службами. Моя схема получает некоторые сбои и открывает схему для определенной службы. Пока все в порядке. Однако цепь больше никогда не становится замкнутой. Мой запасной вариант отключен с помощью этого свойства:

hystrix.command.default.fallback.enabled=false

Что я должен сделать, чтобы проверить, работает ли мой сервис, и снова изменить его на закрытый? Я проверил схемы с помощью Hystrix Dashboard.

com.netflix.hystrix.exception.HystrixRuntimeException: BusinessActivity-FlightAvailabilityV1 timed-out and fallback disabled.
at com.netflix.hystrix.AbstractCommand.getFallbackOrThrowException(AbstractCommand.java:843) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.AbstractCommand.access$200(AbstractCommand.java:59) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.AbstractCommand$9.call(AbstractCommand.java:600) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.AbstractCommand$9.call(AbstractCommand.java:580) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at rx.internal.operators.OperatorOnErrorResumeNextViaFunction$4.onError(OperatorOnErrorResumeNextViaFunction.java:139) ~[rxjava-1.1.5.jar!/:1.1.5]
at rx.internal.operators.OperatorDoOnEach$1.onError(OperatorDoOnEach.java:71) ~[rxjava-1.1.5.jar!/:1.1.5]
at rx.internal.operators.OperatorDoOnEach$1.onError(OperatorDoOnEach.java:71) ~[rxjava-1.1.5.jar!/:1.1.5]
at com.netflix.hystrix.AbstractCommand$HystrixObservableTimeoutOperator$1.run(AbstractCommand.java:953) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable$1.call(HystrixContextRunnable.java:41) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable$1.call(HystrixContextRunnable.java:37) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable.run(HystrixContextRunnable.java:57) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.AbstractCommand$HystrixObservableTimeoutOperator$2.tick(AbstractCommand.java:970) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at com.netflix.hystrix.util.HystrixTimer$1.run(HystrixTimer.java:99) ~[hystrix-core-1.5.2.jar!/:1.5.2]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_111]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) ~[na:1.8.0_111]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) ~[na:1.8.0_111]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) ~[na:1.8.0_111]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[na:1.8.0_111]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[na:1.8.0_111]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_111]
Caused by: java.util.concurrent.TimeoutException: null
    ... 18 common frames omitted

2017-05-11 17:11:14.979  WARN 97456 --- [io-8080-exec-14] o.s.c.n.z.filters.post.SendErrorFilter   : Error during filtering

com.netflix.hystrix.exception.HystrixRuntimeException: BusinessActivity-FlightAvailabilityV1 short-circuited and fallback disabled.
    at com.netflix.hystrix.AbstractCommand.getFallbackOrThrowException(AbstractCommand.java:843) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at com.netflix.hystrix.AbstractCommand.access$200(AbstractCommand.java:59) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at com.netflix.hystrix.AbstractCommand$1.call(AbstractCommand.java:421) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at com.netflix.hystrix.AbstractCommand$1.call(AbstractCommand.java:369) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:50) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:50) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:50) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.Observable.unsafeSubscribe(Observable.java:8460) ~[rxjava-1.1.5.jar!/:1.1.5]
    at com.netflix.hystrix.AbstractCommand$ObservableCommand$1.call(AbstractCommand.java:1133) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at com.netflix.hystrix.AbstractCommand$ObservableCommand$1.call(AbstractCommand.java:1129) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:50) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.Observable.subscribe(Observable.java:8553) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.Observable.subscribe(Observable.java:8520) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.internal.operators.BlockingOperatorToFuture.toFuture(BlockingOperatorToFuture.java:57) ~[rxjava-1.1.5.jar!/:1.1.5]
    at rx.observables.BlockingObservable.toFuture(BlockingObservable.java:401) ~[rxjava-1.1.5.jar!/:1.1.5]
    at com.netflix.hystrix.HystrixCommand.queue(HystrixCommand.java:378) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at com.netflix.hystrix.HystrixCommand.execute(HystrixCommand.java:334) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    at br.com.smiles.hystrix.routes.SmilesHostRoutingFilter.run(SmilesHostRoutingFilter.java:196) ~[smiles-hystrix-server.jar!/:1.0.1-SNAPSHOT]
    at com.netflix.zuul.ZuulFilter.runFilter(ZuulFilter.java:112) ~[zuul-core-1.1.0.jar!/:1.1.0]
    at com.netflix.zuul.FilterProcessor.processZuulFilter(FilterProcessor.java:197) ~[zuul-core-1.1.0.jar!/:1.1.0]
    at com.netflix.zuul.FilterProcessor.runFilters(FilterProcessor.java:161) ~[zuul-core-1.1.0.jar!/:1.1.0]
    at com.netflix.zuul.FilterProcessor.route(FilterProcessor.java:120) ~[zuul-core-1.1.0.jar!/:1.1.0]
    at com.netflix.zuul.ZuulRunner.route(ZuulRunner.java:96) ~[zuul-core-1.1.0.jar!/:1.1.0]
    at com.netflix.zuul.http.ZuulServlet.route(ZuulServlet.java:116) ~[zuul-core-1.1.0.jar!/:1.1.0]
    at com.netflix.zuul.http.ZuulServlet.service(ZuulServlet.java:81) ~[zuul-core-1.1.0.jar!/:1.1.0]
    at org.springframework.web.servlet.mvc.ServletWrappingController.handleRequestInternal(ServletWrappingController.java:158) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.cloud.netflix.zuul.web.ZuulController.handleRequestInternal(ZuulController.java:43) [spring-cloud-netflix-core-1.1.0.RELEASE.jar!/:1.1.0.RELEASE]
    at org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:147) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:50) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:961) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:895) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:967) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:869) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:648) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:843) [spring-webmvc-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:729) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:292) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) [tomcat-embed-websocket-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.boot.actuate.autoconfigure.EndpointWebMvcAutoConfiguration$ApplicationContextHeaderFilter.doFilterInternal(EndpointWebMvcAutoConfiguration.java:261) [spring-boot-actuator-1.3.5.RELEASE.jar!/:1.3.5.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.boot.actuate.trace.WebRequestTraceFilter.doFilterInternal(WebRequestTraceFilter.java:115) [spring-boot-actuator-1.3.5.RELEASE.jar!/:1.3.5.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:87) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:77) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:121) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.springframework.boot.actuate.autoconfigure.MetricsFilter.doFilterInternal(MetricsFilter.java:103) [spring-boot-actuator-1.3.5.RELEASE.jar!/:1.3.5.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-4.2.6.RELEASE.jar!/:4.2.6.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:212) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:522) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1095) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:672) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1502) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1458) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_111]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_111]
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) [tomcat-embed-core-8.0.33.jar!/:8.0.33]
    at java.lang.Thread.run(Thread.java:745) [na:1.8.0_111]
Caused by: java.lang.RuntimeException: Hystrix circuit short-circuited and is OPEN
    at com.netflix.hystrix.AbstractCommand$1.call(AbstractCommand.java:418) ~[hystrix-core-1.5.2.jar!/:1.5.2]
    ... 85 common frames omitted

Вот мои свойства hystrix:

# POOL SIZES
hystrix.threadpool.default.coreSize=70

#QUEUES
hystrix.threadpool.checkout.queueSizeRejectionThreshold=5


#TIMEOUTS
hystrix.command.default.execution.isolation.thread.timeoutInMilliseconds=30000
hystrix.command.getMember.execution.isolation.thread.timeoutInMilliseconds=10000
hystrix.command.checkout.execution.isolation.thread.timeoutInMilliseconds=10000
hystrix.command.joinClub.execution.isolation.thread.timeoutInMilliseconds=10000
hystrix.command.getAvailability.execution.isolation.thread.timeoutInMilliseconds=10000
hystrix.command.myFlights.execution.isolation.thread.timeoutInMilliseconds=10000
hystrix.command.login.execution.isolation.thread.timeoutInMilliseconds=10000

#general HTTP TIMEOUT
zuul.host.socket-timeout-millis=70000
zuul.host.maxTotalConnections=600

#hystrix.command.checkout.circuitBreaker.forceOpen=true
hystrix.threadpool.default.metrics.rollingStats.timeInMilliseconds=10000
hystrix.command.default.circuitBreaker.requestVolumeThreshold=7

# FALLBACKS
hystrix.command.default.fallback.enabled=false

metrics.healthSnapshot.intervalInMilliseconds=500

person cmsantos    schedule 11.05.2017    source источник
comment
можешь поставить свою конфигурацию hystrix   -  person pvpkiran    schedule 11.05.2017
comment
@pvpkiran Я только что обновил свой вопрос, указав некоторые свойства.   -  person cmsantos    schedule 11.05.2017
comment
по умолчанию цепь должна быть разомкнута через 5 секунд. поскольку значение по умолчанию для этого свойства hystrix.command.default.circuitBreaker.sleepWindowInMilliseconds составляет 5000 мс в соответствии с doc(github.com /Netflix/Hystrix/wiki/). Попробуйте установить его вручную, скажем, на 10 секунд и посмотрите, разомкнется ли цепь через 10 секунд.   -  person pvpkiran    schedule 12.05.2017
comment
спасибо! Работает как часы :)   -  person cmsantos    schedule 12.05.2017
comment
хорошо знать. ваше здоровье   -  person pvpkiran    schedule 12.05.2017


Ответы (2)


Как вы знаете, цепь открыта, все запросы в течение sleepWindowInMillisec‌​onds (default 5000ms) будут отклонены. Через sleepWindowInMillisec‌ время ваша цепь становится полуоткрытой, и это позволяет только ОДНОМУ запросу передаваться на ваш внутренний сервер. Если это не удастся, ваша цепь снова откроется, и все последующие запросы будут отклонены в течение следующего sleepWindowInMillisec‌ времени. Если запрос, который был получен в полуоткрытом состоянии, завершится успешно, ваша цепь перейдет в закрытое состояние.

Если ваша схема всегда выглядит открытой, это означает, что запрос в полуоткрытом состоянии снова не удался. Вы можете проверить ситуацию с сообщением об исключении.

person yongsung.yoon    schedule 12.05.2017

Результат моего теста таков:

если hystrix.command.default.fallback.enabled = true, автоматический выключатель может отключиться через указанное время, в противном случае он не может нормально замкнуться

person formerly1    schedule 28.06.2021