diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index eb5d69dfbfde..44eabad70312 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -237,6 +237,11 @@ + + tools.profiler + async-profiler + ${async-profiler.version} + diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index fe2a9a48c210..e139671abca4 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -871,22 +871,16 @@ protected void addDefaultServlets(ContextHandlerCollection contexts, Configurati } else { addUnprivilegedServlet("conf", "/conf", ConfServlet.class); } - final String asyncProfilerHome = ProfileServlet.getAsyncProfilerHome(); - if (asyncProfilerHome != null && !asyncProfilerHome.trim().isEmpty()) { - addPrivilegedServlet("prof", "/prof", ProfileServlet.class); - Path tmpDir = Paths.get(ProfileServlet.OUTPUT_DIR); - if (Files.notExists(tmpDir)) { - Files.createDirectories(tmpDir); - } - ServletContextHandler genCtx = new ServletContextHandler(contexts, "/prof-output-hbase"); - genCtx.addServlet(ProfileOutputServlet.class, "/*"); - genCtx.setResourceBase(tmpDir.toAbsolutePath().toString()); - genCtx.setDisplayName("prof-output-hbase"); - } else { - addUnprivilegedServlet("prof", "/prof", ProfileServlet.DisabledServlet.class); - LOG.info("ASYNC_PROFILER_HOME environment variable and async.profiler.home system property " - + "not specified. Disabling /prof endpoint."); + + addPrivilegedServlet("prof", "/prof", ProfileServlet.class); + Path tmpDir = Paths.get(ProfileServlet.OUTPUT_DIR); + if (Files.notExists(tmpDir)) { + Files.createDirectories(tmpDir); } + ServletContextHandler genCtx = new ServletContextHandler(contexts, "/prof-output-hbase"); + genCtx.addServlet(ProfileOutputServlet.class, "/*"); + genCtx.setResourceBase(tmpDir.toAbsolutePath().toString()); + genCtx.setDisplayName("prof-output-hbase"); /* register metrics servlets */ String[] enabledServlets = conf.getStrings(METRIC_SERVLETS_CONF_KEY, METRICS_SERVLETS_DEFAULT); diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java index 435dde9af496..56105943ab65 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java @@ -19,8 +19,8 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -31,7 +31,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.hbase.util.ProcessUtils; +import one.profiler.AsyncProfiler; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,10 +68,6 @@ public class ProfileServlet extends HttpServlet { private static final String ALLOWED_METHODS = "GET"; private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin"; private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8"; - private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME"; - private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home"; - private static final String OLD_PROFILER_SCRIPT = "profiler.sh"; - private static final String PROFILER_SCRIPT = "asprof"; private static final int DEFAULT_DURATION_SECONDS = 10; private static final AtomicInteger ID_GEN = new AtomicInteger(0); static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output-hbase"; @@ -133,50 +129,110 @@ enum Output { @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "SE_TRANSIENT_FIELD_NOT_RESTORED", justification = "This class is never serialized nor restored.") private transient Lock profilerLock = new ReentrantLock(); - private transient volatile Process process; - private String asyncProfilerHome; - private Integer pid; + private transient volatile boolean profiling; + private final long currentPid = ProcessHandle.current().pid(); + + public static final class ProfileRequest { + private final int duration; + private final Output output; + private final Event event; + private final Long interval; + private final Integer jstackDepth; + private final Long bufsize; + private final boolean thread; + private final boolean simple; + private final Integer width; + private final Integer height; + private final Double minwidth; + private final boolean reverse; + private final int refreshDelay; + private final Integer pid; + + private ProfileRequest(int duration, Output output, Event event, Long interval, + Integer jstackDepth, Long bufsize, boolean thread, boolean simple, Integer width, + Integer height, Double minwidth, boolean reverse, int refreshDelay, Integer pid) { + this.duration = duration; + this.output = output; + this.event = event; + this.interval = interval; + this.jstackDepth = jstackDepth; + this.bufsize = bufsize; + this.thread = thread; + this.simple = simple; + this.width = width; + this.height = height; + this.minwidth = minwidth; + this.reverse = reverse; + this.refreshDelay = refreshDelay; + this.pid = pid; + } - public ProfileServlet() { - this.asyncProfilerHome = getAsyncProfilerHome(); - this.pid = ProcessUtils.getPid(); - LOG.info("Servlet process PID: " + pid + " asyncProfilerHome: " + asyncProfilerHome); - } + public int getDuration() { + return duration; + } - @Override - protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) - throws IOException { - if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) { - resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); - setResponseHeader(resp); - resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!"); - return; + public Output getOutput() { + return output; } - // make sure async profiler home is set - if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - setResponseHeader(resp); - resp.getWriter() - .write("ASYNC_PROFILER_HOME env is not set.\n\n" - + "Please ensure the prerequsites for the Profiler Servlet have been installed and the\n" - + "environment is properly configured. For more information please see\n" - + "http://hbase.apache.org/book.html#profiler\n"); - return; + public Event getEvent() { + return event; + } + + public Long getInterval() { + return interval; } - // if pid is explicitly specified, use it else default to current process - pid = getInteger(req, "pid", pid); + public Integer getJstackDepth() { + return jstackDepth; + } - // if pid is not specified in query param and if current process pid cannot be determined - if (pid == null) { - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - setResponseHeader(resp); - resp.getWriter() - .write("'pid' query parameter unspecified or unable to determine PID of current process."); - return; + public Long getBufsize() { + return bufsize; } + public boolean isThread() { + return thread; + } + + public boolean isSimple() { + return simple; + } + + public Integer getWidth() { + return width; + } + + public Integer getHeight() { + return height; + } + + public Double getMinwidth() { + return minwidth; + } + + public boolean isReverse() { + return reverse; + } + + public int getRefreshDelay() { + return refreshDelay; + } + + public Integer getPid() { + return pid; + } + } + + public ProfileServlet() { + LOG.info("ProfileServlet initialized"); + } + + public ProfileRequest parseProfileRequest(final HttpServletRequest req) { + // Note: when using in-process async-profiler Java API, we can only profile this JVM. + // We keep the pid parameter for API compatibility, but do not support external processes. + Integer requestedPid = getInteger(req, "pid", null); + final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS); final Output output = getOutput(req); final Event event = getEvent(req); @@ -189,109 +245,199 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res final Integer height = getInteger(req, "height", null); final Double minwidth = getMinWidth(req); final boolean reverse = req.getParameterMap().containsKey("reverse"); + int refreshDelay = getInteger(req, "refreshDelay", 0); + + return new ProfileRequest(duration, output, event, interval, jstackDepth, bufsize, thread, + simple, width, height, minwidth, reverse, refreshDelay, requestedPid); + } + + public String buildStartCommand(final ProfileRequest request) { + StringBuilder sb = new StringBuilder("start"); + sb.append(",event=").append(request.getEvent().getInternalName()); + appendOption(sb, "interval", request.getInterval()); + appendOption(sb, "jstackdepth", request.getJstackDepth()); + appendOption(sb, "bufsize", request.getBufsize()); + if (request.isThread()) { + sb.append(",threads"); + } + if (request.isSimple()) { + sb.append(",simple"); + } + return sb.toString(); + } + + public String buildStopCommand(final ProfileRequest request, final File outputFile) { + StringBuilder sb = new StringBuilder("stop"); + sb.append(",file=").append(outputFile.getAbsolutePath()); + sb.append(",format=").append(mapOutputToAsyncProfilerFormat(request.getOutput())); + appendOption(sb, "width", request.getWidth()); + appendOption(sb, "height", request.getHeight()); + appendOption(sb, "minwidth", request.getMinwidth()); + if (request.isReverse()) { + sb.append(",reverse"); + } + return sb.toString(); + } + + private void appendOption(final StringBuilder sb, final String key, final Object value) { + if (value != null) { + sb.append(',').append(key).append('=').append(value); + } + } + + protected String executeProfiler(String command) throws IOException { + return AsyncProfiler.getInstance().execute(command); + } + + @Override + protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) + throws IOException { + if (!checkInstrumentationAccess(req, resp)) { + return; + } + + final ProfileRequest request = parseProfileRequest(req); + + // We keep the pid parameter for backward compatibility but only support profiling this JVM. + if (request.getPid() != null && request.getPid().longValue() != currentPid) { + writeError(resp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "The 'pid' parameter is only supported for the current process when using the " + + "embedded async-profiler library."); + return; + } + + if (profiling) { + writeError(resp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Another instance of profiler is already running."); + return; + } + + int lockTimeoutSecs = 3; + boolean locked = false; + try { + locked = profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS); + if (!locked) { + writeError(resp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Unable to acquire lock. Another instance of profiler might be running."); + LOG.warn("Unable to acquire lock in " + lockTimeoutSecs + + " seconds. Another instance of profiler might be running."); + return; + } - if (process == null || !process.isAlive()) { + File outputFile = createOutputFile(request); + // Ensure the file exists so ProfileOutputServlet can poll until it is complete. + Files.write(outputFile.toPath(), new byte[0]); + + String startCmd = buildStartCommand(request); + executeProfiler(startCmd); + profiling = true; + + String stopCmd = buildStopCommand(request, outputFile); + startStopperThread(request.getDuration(), stopCmd, outputFile); + + List visible = new ArrayList<>(2); + visible.add(startCmd); + visible.add(stopCmd); + writeAcceptedResponse(resp, request, outputFile, visible); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("Interrupted while acquiring profile lock.", e); + writeError(resp, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Interrupted while acquiring profile lock."); + } finally { + if (locked) { + profilerLock.unlock(); + } + } + } + + private void startStopperThread(final int durationSeconds, final String stopCmd, + final File outputFile) { + Thread t = new Thread(() -> { try { - int lockTimeoutSecs = 3; - if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) { - try { - File outputFile = - new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" + event.name().toLowerCase() + "-" - + ID_GEN.incrementAndGet() + "." + output.name().toLowerCase()); - Files.createDirectories(Paths.get(OUTPUT_DIR)); - List cmd = new ArrayList<>(); - Path profilerScriptPath = Paths.get(asyncProfilerHome, "bin", PROFILER_SCRIPT); - if (!Files.exists(profilerScriptPath)) { - LOG.info( - "async-profiler script {} does not exist, fallback to use old script {}(version <= 2.9).", - PROFILER_SCRIPT, OLD_PROFILER_SCRIPT); - profilerScriptPath = Paths.get(asyncProfilerHome, OLD_PROFILER_SCRIPT); - } - cmd.add(profilerScriptPath.toString()); - cmd.add("-e"); - cmd.add(event.getInternalName()); - cmd.add("-d"); - cmd.add("" + duration); - cmd.add("-o"); - cmd.add(output.name().toLowerCase()); - cmd.add("-f"); - cmd.add(outputFile.getAbsolutePath()); - if (interval != null) { - cmd.add("-i"); - cmd.add(interval.toString()); - } - if (jstackDepth != null) { - cmd.add("-j"); - cmd.add(jstackDepth.toString()); - } - if (bufsize != null) { - cmd.add("-b"); - cmd.add(bufsize.toString()); - } - if (thread) { - cmd.add("-t"); - } - if (simple) { - cmd.add("-s"); - } - if (width != null) { - cmd.add("--width"); - cmd.add(width.toString()); - } - if (height != null) { - cmd.add("--height"); - cmd.add(height.toString()); - } - if (minwidth != null) { - cmd.add("--minwidth"); - cmd.add(minwidth.toString()); - } - if (reverse) { - cmd.add("--reverse"); - } - cmd.add(pid.toString()); - process = ProcessUtils.runCmdAsync(cmd); - - // set response and set refresh header to output location - setResponseHeader(resp); - resp.setStatus(HttpServletResponse.SC_ACCEPTED); - String relativeUrl = "/prof-output-hbase/" + outputFile.getName(); - resp.getWriter() - .write("Started [" + event.getInternalName() - + "] profiling. This page will automatically redirect to " + relativeUrl + " after " - + duration + " seconds. " - + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async " - + "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler." - + "\n\nCommand:\n" + Joiner.on(" ").join(cmd)); - - // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified - // via url param - int refreshDelay = getInteger(req, "refreshDelay", 0); - - // instead of sending redirect, set auto-refresh so that browsers will refresh - // with redirected url - resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl); - resp.getWriter().flush(); - } finally { - profilerLock.unlock(); - } - } else { - setResponseHeader(resp); - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - resp.getWriter() - .write("Unable to acquire lock. Another instance of profiler might be running."); - LOG.warn("Unable to acquire lock in " + lockTimeoutSecs - + " seconds. Another instance of profiler might be running."); + TimeUnit.SECONDS.sleep(durationSeconds); + executeProfiler(stopCmd); + } catch (Exception e) { + try { + Files.write(outputFile.toPath(), + ("Profiler failed: " + e.getMessage()).getBytes(StandardCharsets.UTF_8)); + } catch (IOException ioe) { + LOG.warn("Unable to write profiler error to output file", ioe); } - } catch (InterruptedException e) { - LOG.warn("Interrupted while acquiring profile lock.", e); - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + LOG.warn("Profiler stop/dump failed", e); + } finally { + profiling = false; } - } else { + }, "ProfileServlet-stopper"); + t.setDaemon(true); + t.start(); + } + + private boolean checkInstrumentationAccess(final HttpServletRequest req, + final HttpServletResponse resp) throws IOException { + if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), req, resp)) { + resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); setResponseHeader(resp); - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - resp.getWriter().write("Another instance of profiler is already running."); + resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!"); + return false; } + return true; + } + + private String mapOutputToAsyncProfilerFormat(Output output) { + switch (output) { + case SUMMARY: + return "summary"; + case TRACES: + return "traces"; + case FLAT: + return "flat"; + case COLLAPSED: + return "collapsed"; + case TREE: + return "tree"; + case JFR: + return "jfr"; + case SVG: + return "svg"; + case HTML: + default: + return "html"; + } + } + + private void writeError(final HttpServletResponse resp, final int status, final String message) + throws IOException { + resp.setStatus(status); + setResponseHeader(resp); + resp.getWriter().write(message); + } + + private File createOutputFile(final ProfileRequest request) throws IOException { + final long pid = request.getPid() != null ? request.getPid().longValue() : currentPid; + File outputFile = + new File(OUTPUT_DIR, "async-prof-pid-" + pid + "-" + request.getEvent().name().toLowerCase() + + "-" + ID_GEN.incrementAndGet() + "." + request.getOutput().name().toLowerCase()); + Files.createDirectories(Paths.get(OUTPUT_DIR)); + return outputFile; + } + + private void writeAcceptedResponse(final HttpServletResponse resp, final ProfileRequest request, + final File outputFile, final List cmd) throws IOException { + setResponseHeader(resp); + resp.setStatus(HttpServletResponse.SC_ACCEPTED); + String relativeUrl = "/prof-output-hbase/" + outputFile.getName(); + resp.getWriter() + .write("Started [" + request.getEvent().getInternalName() + + "] profiling. This page will automatically redirect to " + relativeUrl + " after " + + request.getDuration() + " seconds. " + + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async " + + "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler." + + "\n\nCommand:\n" + Joiner.on(" ").join(cmd)); + + resp.setHeader("Refresh", + (request.getDuration() + request.getRefreshDelay()) + ";" + relativeUrl); + resp.getWriter().flush(); } private Integer getInteger(final HttpServletRequest req, final String param, @@ -358,16 +504,6 @@ static void setResponseHeader(final HttpServletResponse response) { response.setContentType(CONTENT_TYPE_TEXT); } - static String getAsyncProfilerHome() { - String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV); - // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set - if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) { - asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY); - } - - return asyncProfilerHome; - } - public static class DisabledServlet extends HttpServlet { private static final long serialVersionUID = 1L; diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml index 0ce3c3ebc648..fd049b5b8d88 100644 --- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml +++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml @@ -55,6 +55,20 @@ under the License. + + + tools.profiler + async-profiler + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + diff --git a/pom.xml b/pom.xml index 13062ead5920..38acd4ad6baa 100644 --- a/pom.xml +++ b/pom.xml @@ -1098,6 +1098,7 @@ 5.32.0 6.29.0 5.23.0 + 2.9