diff --git a/.sonarcloud.properties b/.sonarcloud.properties deleted file mode 100644 index 23a51a2894..0000000000 --- a/.sonarcloud.properties +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SonarCloud automatic analysis reads this file (not sonar-project.properties). -sonar.cpd.exclusions=**/streampark-flink-shims/**,**/streampark-spark-shims/**,**/streampark-flink-shims-base/**,**/streampark-flink-shims-base-v2/**,**/streampark-flink-connector/**,**/streampark-flink-kubernetes/**,**/streampark-flink-packer/** - -sonar.issue.ignore.multicriteria=s2083safepath,s100shims,s1948connector,s6355shims,s1123deprecated,s1133deprecated,s6355shimsall,s8786sqlconvert,s112client,s1186silent -sonar.issue.ignore.multicriteria.s2083safepath.ruleKey=javasecurity:S2083 -sonar.issue.ignore.multicriteria.s2083safepath.resourceKey=**/SafePathUtils.java -sonar.issue.ignore.multicriteria.s100shims.ruleKey=java:S100 -sonar.issue.ignore.multicriteria.s100shims.resourceKey=**/streampark-flink-shims*/** -sonar.issue.ignore.multicriteria.s1948connector.ruleKey=java:S1948 -sonar.issue.ignore.multicriteria.s1948connector.resourceKey=**/streampark-flink-connector/** -sonar.issue.ignore.multicriteria.s6355shims.ruleKey=java:S6355 -sonar.issue.ignore.multicriteria.s6355shims.resourceKey=**/FlinkStreamTableTrait*.java -sonar.issue.ignore.multicriteria.s1123deprecated.ruleKey=java:S1123 -sonar.issue.ignore.multicriteria.s1123deprecated.resourceKey=**/FlinkStreamTableTrait*.java -sonar.issue.ignore.multicriteria.s1133deprecated.ruleKey=java:S1133 -sonar.issue.ignore.multicriteria.s1133deprecated.resourceKey=**/FlinkStreamTableTrait*.java -sonar.issue.ignore.multicriteria.s6355shimsall.ruleKey=java:S6355 -sonar.issue.ignore.multicriteria.s6355shimsall.resourceKey=**/streampark-flink-shims*/** -sonar.issue.ignore.multicriteria.s8786sqlconvert.ruleKey=java:S8786 -sonar.issue.ignore.multicriteria.s8786sqlconvert.resourceKey=**/SqlConvertUtils.java -sonar.issue.ignore.multicriteria.s112client.ruleKey=java:S112 -sonar.issue.ignore.multicriteria.s112client.resourceKey=**/streampark-*-client*/** -sonar.issue.ignore.multicriteria.s1186silent.ruleKey=java:S1186 -sonar.issue.ignore.multicriteria.s1186silent.resourceKey=**/Silent*.java diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 31410c1b0a..0000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -sonar.cpd.exclusions=**/streampark-flink-shims/**,**/streampark-spark-shims/**,**/streampark-flink-shims-base/**,**/streampark-flink-shims-base-v2/**,**/streampark-flink-connector/**,**/streampark-flink-kubernetes/**,**/streampark-flink-packer/** - -sonar.issue.ignore.multicriteria=s2083safepath,s100shims,s1948connector,s6355shims,s1123deprecated,s1133deprecated,s6355shimsall,s8786sqlconvert,s112client,s1186silent -sonar.issue.ignore.multicriteria.s2083safepath.ruleKey=javasecurity:S2083 -sonar.issue.ignore.multicriteria.s2083safepath.resourceKey=**/SafePathUtils.java -sonar.issue.ignore.multicriteria.s100shims.ruleKey=java:S100 -sonar.issue.ignore.multicriteria.s100shims.resourceKey=**/streampark-flink-shims*/** -sonar.issue.ignore.multicriteria.s1948connector.ruleKey=java:S1948 -sonar.issue.ignore.multicriteria.s1948connector.resourceKey=**/streampark-flink-connector/** -sonar.issue.ignore.multicriteria.s6355shims.ruleKey=java:S6355 -sonar.issue.ignore.multicriteria.s6355shims.resourceKey=**/FlinkStreamTableTrait*.java -sonar.issue.ignore.multicriteria.s1123deprecated.ruleKey=java:S1123 -sonar.issue.ignore.multicriteria.s1123deprecated.resourceKey=**/FlinkStreamTableTrait*.java -sonar.issue.ignore.multicriteria.s1133deprecated.ruleKey=java:S1133 -sonar.issue.ignore.multicriteria.s1133deprecated.resourceKey=**/FlinkStreamTableTrait*.java -sonar.issue.ignore.multicriteria.s6355shimsall.ruleKey=java:S6355 -sonar.issue.ignore.multicriteria.s6355shimsall.resourceKey=**/streampark-flink-shims*/** -sonar.issue.ignore.multicriteria.s8786sqlconvert.ruleKey=java:S8786 -sonar.issue.ignore.multicriteria.s8786sqlconvert.resourceKey=**/SqlConvertUtils.java -sonar.issue.ignore.multicriteria.s112client.ruleKey=java:S112 -sonar.issue.ignore.multicriteria.s112client.resourceKey=**/streampark-*-client*/** -sonar.issue.ignore.multicriteria.s1186silent.ruleKey=java:S1186 -sonar.issue.ignore.multicriteria.s1186silent.resourceKey=**/Silent*.java diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/FlinkVersion.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/FlinkVersion.java index 9112c7ae8b..394a8004ea 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/FlinkVersion.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/conf/FlinkVersion.java @@ -23,16 +23,15 @@ import java.io.File; import java.io.Serializable; +import java.net.MalformedURLException; import java.net.URL; import java.util.Arrays; +import java.util.Collections; import java.util.List; -import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import scala.collection.JavaConverters; - /** @param flinkHome actual flink home that must be a readable local path */ public class FlinkVersion implements Serializable { @@ -85,11 +84,6 @@ public String version() { return getVersion(); } - /** Scala API alias for {@link #getFlinkLibs()}. */ - public scala.collection.immutable.List flinkLibs() throws Exception { - return JavaConverters.asScalaIteratorConverter(getFlinkLibs().iterator()).asScala().toList(); - } - public String getScalaVersion() { if (scalaVersion == null) { Matcher matcher = FLINK_SCALA_VERSION_PATTERN.matcher(getFlinkDistJar().getName()); @@ -123,13 +117,13 @@ public File getFlinkLib() { public List getFlinkLibs() throws Exception { File[] files = getFlinkLib().listFiles(); if (files == null) { - return Arrays.asList(); + return Collections.emptyList(); } return Arrays.stream(files).map(f -> { try { return f.toURI().toURL(); - } catch (Exception e) { - throw new RuntimeException(e); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Invalid Flink lib URL: " + f, e); } }).collect(Collectors.toList()); } @@ -147,22 +141,18 @@ public String getVersion() { CommandUtils.execute( getFlinkLib().getAbsolutePath(), cmd, - new Consumer() { - - @Override - public void accept(String out) { - buffer.append(out).append("\n"); - Matcher matcher = FLINK_VERSION_PATTERN.matcher(out); - if (matcher.find()) { - String ver = matcher.group(1); - Matcher m1 = APACHE_FLINK_VERSION_PATTERN.matcher(ver); - if (m1.find()) { + out -> { + buffer.append(out).append("\n"); + Matcher matcher = FLINK_VERSION_PATTERN.matcher(out); + if (matcher.find()) { + String ver = matcher.group(1); + Matcher m1 = APACHE_FLINK_VERSION_PATTERN.matcher(ver); + if (m1.find()) { + flinkVersion[0] = ver; + } else { + Matcher m2 = OTHER_FLINK_VERSION_PATTERN.matcher(ver); + if (m2.find()) { flinkVersion[0] = ver; - } else { - Matcher m2 = OTHER_FLINK_VERSION_PATTERN.matcher(ver); - if (m2.find()) { - flinkVersion[0] = ver; - } } } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/SparkVersion.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/SparkVersion.java index 095f9bbd2c..f57aecd0ab 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/SparkVersion.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/conf/SparkVersion.java @@ -66,7 +66,7 @@ private static String[] parseVersion(String sparkHome) { sparkVersion[1] = m1.group(1); }); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to parse Spark version from " + sparkHome, e); } LOG.info("[StreamPark] {}", buffer); if (sparkVersion[0] == null || sparkVersion[1] == null) { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/AutoCloseUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/AutoCloseUtils.java index 0f24d5a0a5..db46760839 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/AutoCloseUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/AutoCloseUtils.java @@ -44,7 +44,7 @@ public static R using( if (e instanceof Error) { throw (Error) e; } - throw new RuntimeException(e); + throw new IllegalStateException(e); } } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java index 6e126bf946..98080ab4b1 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java @@ -68,7 +68,7 @@ public static void loadJar(String jarFilePath) { try { loadPath(f.getAbsolutePath()); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to load jar: " + jarFilePath, e); } } @@ -91,7 +91,7 @@ public static void loadJars(String path) { try { loadPath(x.getAbsolutePath()); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to load jar: " + x.getAbsolutePath(), e); } } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/CommandUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/CommandUtils.java index 12684915e2..fd6da82665 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/CommandUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/CommandUtils.java @@ -59,9 +59,9 @@ public static CommandResult execute(String command) { return new CommandResult(code, buffer.toString()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw new IllegalStateException(e); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException(e); } } @@ -96,9 +96,9 @@ public static int execute(String directory, Iterable commands, Consumer< return waitFor(process); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw new IllegalStateException(e); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException(e); } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/ExceptionUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/ExceptionUtils.java index 59652e2fca..c674e67a4a 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/ExceptionUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/ExceptionUtils.java @@ -62,9 +62,9 @@ public static O wrapRuntimeException(I input, WrapperRuntimeExceptionHand return handler.handle(input); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw new IllegalStateException(e); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException(e); } } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java index 1193332ff7..44b5f6445b 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java @@ -85,7 +85,7 @@ private static String bytesToHexString(byte[] src) { public static boolean isJarFileType(InputStream input) { if (input == null) { - throw new RuntimeException("The inputStream can not be null"); + throw new IllegalArgumentException("The inputStream can not be null"); } return AutoCloseUtils.using( input, @@ -94,7 +94,7 @@ public static boolean isJarFileType(InputStream input) { try { in.read(b, 0, b.length); } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to read jar file header", e); } return bytesToHexString(b); }) @@ -103,7 +103,7 @@ public static boolean isJarFileType(InputStream input) { public static boolean isJarFileType(File file) throws IOException { if (!file.exists() || !file.isFile()) { - throw new RuntimeException("The file does not exist or the path is a directory"); + throw new IllegalArgumentException("The file does not exist or the path is a directory"); } return isJarFileType(new FileInputStream(file)); } @@ -174,7 +174,7 @@ public static List listFileAsURL(String dirPath) { try { urls.add(f.toURI().toURL()); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to convert file to URL: " + f, e); } } return urls; @@ -206,7 +206,7 @@ public static boolean equals(File file1, File file2) { try { return equalsInternal(file1, file2); } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to compare files", e); } } @@ -310,7 +310,7 @@ public static String tailOf(String path, int offset, int limit) { } return null; } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to read tail of file: " + path, e); } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java index 751f6073c7..91e55d0340 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java @@ -61,7 +61,7 @@ public static Map loadFlinkConf(File file) { return loadFlinkConf( org.apache.commons.io.FileUtils.readFileToString(file, StandardCharsets.UTF_8)); } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to read Flink configuration file: " + file, e); } } @@ -79,7 +79,7 @@ public static Map loadLegacyFlinkConf(File file) { return loadLegacyFlinkConf( org.apache.commons.io.FileUtils.readFileToString(file, StandardCharsets.UTF_8)); } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to read legacy Flink configuration file: " + file, e); } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java index 08c59793b4..2232dc49a6 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java @@ -76,7 +76,7 @@ public static HBaseClient apply(Properties prop) { })); return connection; } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to create HBase connection", e); } }); } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java index 62dfd119ac..409a5b13d7 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java @@ -179,7 +179,7 @@ private static void rewriteHostIpMapper(File configFile, Map hos } org.apache.commons.io.FileUtils.writeLines(configFile, lines); } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to rewrite Hadoop configuration file: " + configFile, e); } } @@ -194,67 +194,47 @@ private static Map.Entry findHostMatch(String line, Map readSystemHadoopConf() { return getSystemHadoopConfDirOptional() - .map( - confDir -> { - Map map = new LinkedHashMap<>(); - File[] files = LfsOperator.listDir(confDir); - if (files != null) { - for (File f : files) { - boolean matched = false; - for (String name : HADOOP_CLIENT_CONF_FILES) { - if (name.equals(f.getName())) { - matched = true; - break; - } - } - if (!matched) { - continue; - } - try { - map.put( - f.getName(), - org.apache.commons.io.FileUtils.readFileToString( - f, StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - return map; - }) + .map(dir -> readSystemConfFiles(dir, HADOOP_CLIENT_CONF_FILES, "Hadoop")) .orElse(Collections.emptyMap()); } public static Map readSystemHiveConf() { return getSystemHiveConfDirOptional() - .map( - confDir -> { - Map map = new LinkedHashMap<>(); - File[] files = LfsOperator.listDir(confDir); - if (files != null) { - for (File f : files) { - boolean matched = false; - for (String name : HIVE_CLIENT_CONF_FILES) { - if (name.equals(f.getName())) { - matched = true; - break; - } - } - if (!matched) { - continue; - } - try { - map.put( - f.getName(), - org.apache.commons.io.FileUtils.readFileToString( - f, StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - return map; - }) + .map(dir -> readSystemConfFiles(dir, HIVE_CLIENT_CONF_FILES, "Hive")) .orElse(Collections.emptyMap()); } + + private static Map readSystemConfFiles( + String confDir, + String[] confFileNames, + String confLabel) { + Map map = new LinkedHashMap<>(); + File[] files = LfsOperator.listDir(confDir); + if (files == null) { + return map; + } + for (File f : files) { + if (!matchesConfFile(f.getName(), confFileNames)) { + continue; + } + try { + map.put( + f.getName(), + org.apache.commons.io.FileUtils.readFileToString(f, StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to read " + confLabel + " configuration file: " + f.getAbsolutePath(), e); + } + } + return map; + } + + private static boolean matchesConfFile(String fileName, String[] confFileNames) { + for (String name : confFileNames) { + if (name.equals(fileName)) { + return true; + } + } + return false; + } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java index c19871f99a..6e2ba78c2c 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java @@ -138,7 +138,7 @@ private static long getTgtRefreshTime() { } } } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to resolve kerberos TTL configuration", e); } } return tgtRefreshTime; @@ -179,7 +179,7 @@ public static Configuration hadoopConf() { try { ClassLoaderUtils.loadResource(getHadoopConfDir()); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to load Hadoop resources from " + getHadoopConfDir(), e); } if (StringUtils.isBlank(reusableConf.get("hadoop.tmp.dir"))) { @@ -260,7 +260,7 @@ private static UserGroupInformation getKerberosUGI() { LOG.info("[StreamPark] kerberos authentication successful"); return kerberosUgi; } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Kerberos authentication failed", e); } } @@ -274,7 +274,7 @@ public static FileSystem hdfs() { try { return FileSystem.get(hadoopConf()); } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to obtain HDFS FileSystem", e); } }); if (HadoopConfigUtils.KERBEROS_ENABLE) { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HttpClientUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HttpClientUtils.java index a66f8ec049..d97597de7d 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HttpClientUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HttpClientUtils.java @@ -78,7 +78,7 @@ private static HttpGet getHttpGet(String url, Map params, Reques try { httpGet = new HttpGet(uriBuilder.build()); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to build HTTP GET URI for " + url, e); } } if (config != null) { @@ -195,7 +195,7 @@ private static String getHttpResult(HttpUriRequestBase request, CloseableHttpCli return null; }); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("HTTP request failed", e); } } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java index ea02881af7..0af294fd8d 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java @@ -264,7 +264,7 @@ public static Connection getConnection(Properties prop) { } return ds.getConnection(); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to obtain JDBC connection for alias: " + alias, e); } finally { lock.unlock(); } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/LoggerSupport.java b/streampark-common/src/main/java/org/apache/streampark/common/util/LoggerSupport.java index 6421ae1eeb..e7b45e49c4 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/LoggerSupport.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/LoggerSupport.java @@ -18,8 +18,7 @@ package org.apache.streampark.common.util; /** - * Java logging support using {@link StreamParkLoggerFactory}. Scala modules mix in {@code Logger} - * from streampark-common-scala-bridge instead. + * Java logging support using {@link StreamParkLoggerFactory}. */ public abstract class LoggerSupport { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java b/streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java index 04fed6b431..00b60b4047 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java @@ -97,7 +97,7 @@ public static Jedis connect(RedisEndpoint endpoint) { Thread.sleep(sleepTime); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); - throw new RuntimeException(ie); + throw new IllegalStateException("Interrupted while waiting for Redis connection", ie); } } else { throw e; diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java index d88359eaa7..55e24ee6e1 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java @@ -25,24 +25,34 @@ private StringCastUtils() { @SuppressWarnings("unchecked") public static T cast(String v, Class classType) { + return (T) classType.cast(parseValue(v, classType)); + } + + private static Object parseValue(String v, Class classType) { if (classType == String.class) { - return (T) v; - } else if (classType == byte.class || classType == Byte.class) { - return (T) (classType == byte.class ? (Object) Byte.parseByte(v) : Byte.valueOf(v)); - } else if (classType == int.class || classType == Integer.class) { - return (T) (classType == int.class ? (Object) Integer.parseInt(v) : Integer.valueOf(v)); - } else if (classType == long.class || classType == Long.class) { - return (T) (classType == long.class ? (Object) Long.parseLong(v) : Long.valueOf(v)); - } else if (classType == float.class || classType == Float.class) { - return (T) (classType == float.class ? (Object) Float.parseFloat(v) : Float.valueOf(v)); - } else if (classType == double.class || classType == Double.class) { - return (T) (classType == double.class ? (Object) Double.parseDouble(v) : Double.valueOf(v)); - } else if (classType == short.class || classType == Short.class) { - return (T) (classType == short.class ? (Object) Short.parseShort(v) : Short.valueOf(v)); - } else if (classType == boolean.class || classType == Boolean.class) { - return (T) (classType == boolean.class ? (Object) Boolean.parseBoolean(v) : Boolean.valueOf(v)); - } else { - throw new IllegalArgumentException("Unsupported type: " + classType); + return v; + } + if (classType == Byte.class || classType == byte.class) { + return Byte.parseByte(v); + } + if (classType == Integer.class || classType == int.class) { + return Integer.parseInt(v); + } + if (classType == Long.class || classType == long.class) { + return Long.parseLong(v); + } + if (classType == Float.class || classType == float.class) { + return Float.parseFloat(v); + } + if (classType == Double.class || classType == double.class) { + return Double.parseDouble(v); + } + if (classType == Short.class || classType == short.class) { + return Short.parseShort(v); + } + if (classType == Boolean.class || classType == boolean.class) { + return Boolean.parseBoolean(v); } + throw new IllegalArgumentException("Unsupported type: " + classType); } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/SystemPropertyUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/SystemPropertyUtils.java index c35d9954bb..d126aac1c3 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/SystemPropertyUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/SystemPropertyUtils.java @@ -129,9 +129,11 @@ public static void setEnv(String name, String value) throws Exception { Class clazz = Class.forName("java.util.Collections$UnmodifiableMap"); java.lang.reflect.Field field = clazz.getDeclaredField("m"); field.setAccessible(true); - @SuppressWarnings("unchecked") - Map map = (Map) field.get(unmodifiableEnvironment); - map.put(name, value); + Object envMap = field.get(unmodifiableEnvironment); + if (!(envMap instanceof Map)) { + throw new IllegalStateException("Process environment is not a map"); + } + Map.class.getMethod("put", Object.class, Object.class).invoke(envMap, name, value); } public static String getOrElseUpdate(String key, String defaultValue) { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java index 1dfa2c9f51..0fb5e294a2 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java @@ -110,7 +110,7 @@ public static String getJarManClass(File jarFile) { } return mainClass; } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to read jar manifest: " + jarFile, e); } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java index 62fed8ef93..574528a1e0 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java @@ -249,7 +249,7 @@ public static String getYarnAppTrackingUrl(ApplicationId applicationId) { try { return HadoopUtils.yarnClient().getApplicationReport(applicationId).getTrackingUrl(); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to get Yarn application tracking URL for " + applicationId, e); } } diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/AutoCloseUtilsTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/AutoCloseUtilsTest.java new file mode 100644 index 0000000000..788dbcf910 --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/AutoCloseUtilsTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AutoCloseUtilsTest { + + @Test + void usingShouldReturnResultAndCloseResource() throws Exception { + TestResource resource = new TestResource(); + String result = AutoCloseUtils.using(resource, r -> "ok-" + r.name); + assertThat(result).isEqualTo("ok-test"); + assertThat(resource.closed).isTrue(); + } + + @Test + void usingShouldInvokeExceptionHandler() { + TestResource resource = new TestResource(); + String result = + AutoCloseUtils.using( + resource, + r -> { + throw new IllegalStateException("boom"); + }, + e -> "handled"); + assertThat(result).isEqualTo("handled"); + assertThat(resource.closed).isTrue(); + } + + @Test + void usingShouldRethrowRuntimeExceptionWhenNoHandler() { + TestResource resource = new TestResource(); + assertThatThrownBy( + () -> AutoCloseUtils.using( + resource, + r -> { + throw new IllegalArgumentException("bad"); + })) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("bad"); + assertThat(resource.closed).isTrue(); + } + + private static final class TestResource implements AutoCloseable { + + private final String name = "test"; + private boolean closed; + + @Override + public void close() { + closed = true; + } + } +} diff --git a/streampark-console/streampark-console-service/pom.xml b/streampark-console/streampark-console-service/pom.xml index d94e90864e..3d8431b4d8 100644 --- a/streampark-console/streampark-console-service/pom.xml +++ b/streampark-console/streampark-console-service/pom.xml @@ -643,26 +643,10 @@ - - net.alchim31.maven - scala-maven-plugin - - JavaThenScala - - - com.diffplug.spotless spotless-maven-plugin ${maven-spotless-plugin.version} - - - - ${spotless.scalafmt.version} - ${maven.multiModuleProjectDirectory}/tools/checkstyle/.scalafmt.conf - - - spotless-check diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/Throws.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/Throws.java index e5434cf47d..9a6afd1e7e 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/Throws.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/Throws.java @@ -159,7 +159,7 @@ private static T createRuntimeException( try { return createRuntimeExceptionInternal(exceptionClass, errorMsgs); } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to instantiate exception type: " + exceptionClass.getName(), e); } } @@ -176,7 +176,8 @@ private static T createRuntimeExceptionInternal( ? exceptionClass.getConstructor(String.class) : exceptionClass.getConstructor(); } catch (NoSuchMethodException e) { - throw new RuntimeException(e); + throw new IllegalStateException( + "No suitable constructor for exception type: " + exceptionClass.getName(), e); } }); AssertUtils.notNull( diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/AppControl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/AppControl.java index d85db72a96..6795204926 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/AppControl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/AppControl.java @@ -17,6 +17,8 @@ package org.apache.streampark.console.core.bean; +import org.apache.streampark.flink.packer.pipeline.PipelineStatusEnum; + import lombok.Getter; import lombok.Setter; import lombok.experimental.Accessors; @@ -36,4 +38,18 @@ public class AppControl { private boolean allowBuild; private boolean allowView; + + public static AppControl fromPipelineBuildStatus( + Integer buildStatus, + boolean shouldTracking, + boolean running) { + return new AppControl() + .setAllowBuild( + buildStatus == null + || !PipelineStatusEnum.RUNNING.getCode().equals(buildStatus)) + .setAllowStart( + !shouldTracking && PipelineStatusEnum.SUCCESS.getCode().equals(buildStatus)) + .setAllowStop(running) + .setAllowView(shouldTracking); + } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/FlinkCheckpointProcessor.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/FlinkCheckpointProcessor.java index b2419be39d..2651f50056 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/FlinkCheckpointProcessor.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/FlinkCheckpointProcessor.java @@ -143,7 +143,7 @@ private void processFailoverStrategy( try { applicationActionService.restart(application); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to restart application after checkpoint failure", e); } break; default: diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationBuildPipeline.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationBuildPipeline.java index a8bd2df79d..de55d92b5b 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationBuildPipeline.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationBuildPipeline.java @@ -304,7 +304,7 @@ public static View of(@Nonnull ApplicationBuildPipeline pipe) { Step step = new Step() .setSeq(i) .setDesc(stepDesc.getOrDefault(i, "unknown step")) - .setStatus(stepStatus.getOrDefault(i, PipelineStepStatusEnum.unknown).getCode()); + .setStatus(stepStatus.getOrDefault(i, PipelineStepStatusEnum.UNKNOWN).getCode()); Long st = stepTs.get(i); if (st != null) { step.setTs(new Date(st)); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationEntitySupport.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationEntitySupport.java new file mode 100644 index 0000000000..2e2b87280d --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationEntitySupport.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.entity; + +import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.enums.StorageType; +import org.apache.streampark.common.fs.FsOperator; +import org.apache.streampark.console.core.util.ApplicationEntityUtils; +import org.apache.streampark.flink.packer.maven.DependencyInfo; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public interface ApplicationEntitySupport { + + Long getProjectId(); + + Long getId(); + + String getModule(); + + String getDependency(); + + Integer getRelease(); + + Integer getRestartSize(); + + Integer getRestartCount(); + + @JsonIgnore + StorageType getStorageType(); + + @JsonIgnore + default String getDistHome() { + return ApplicationEntityUtils.distHome(getProjectId(), getModule()); + } + + @JsonIgnore + default String getLocalAppHome() { + return ApplicationEntityUtils.localAppHome(getId()); + } + + @JsonIgnore + default String getRemoteAppHome() { + return ApplicationEntityUtils.remoteAppHome(getId()); + } + + @JsonIgnore + default DependencyInfo getDependencyInfo() { + return ApplicationEntityUtils.dependencyInfo(getDependency()); + } + + @JsonIgnore + default boolean isNeedRollback() { + return ApplicationEntityUtils.needRollback(getRelease()); + } + + @JsonIgnore + default boolean isNeedRestartOnFailed() { + return ApplicationEntityUtils.needRestartOnFailed(getRestartSize(), getRestartCount()); + } + + @JsonIgnore + default FsOperator getFsOperator() { + return ApplicationEntityUtils.fsOperator(getStorageType()); + } + + @JsonIgnore + default Workspace getWorkspace() { + return ApplicationEntityUtils.workspace(getStorageType()); + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java index 021612f1d5..f16dfc675f 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java @@ -18,14 +18,12 @@ package org.apache.streampark.console.core.entity; import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.Workspace; import org.apache.streampark.common.constants.Constants; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.common.enums.FlinkK8sRestExposedType; import org.apache.streampark.common.enums.StorageType; -import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.console.base.mybatis.entity.BaseEntity; import org.apache.streampark.console.base.util.JacksonUtils; import org.apache.streampark.console.core.bean.AppControl; @@ -36,7 +34,6 @@ import org.apache.streampark.console.core.metrics.flink.JobsOverview; import org.apache.streampark.console.core.util.YarnQueueLabelExpression; import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates; -import org.apache.streampark.flink.packer.maven.DependencyInfo; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -47,6 +44,7 @@ import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.type.TypeReference; import lombok.Getter; import lombok.Setter; import lombok.SneakyThrows; @@ -63,7 +61,11 @@ @Setter @TableName("t_flink_app") @Slf4j -public class FlinkApplication extends BaseEntity { +public class FlinkApplication extends BaseEntity implements ApplicationEntitySupport, ReleaseOutcomeTarget { + + private static final TypeReference> STRING_OBJECT_MAP = + new TypeReference>() { + }; @TableId(type = IdType.AUTO) private Long id; @@ -420,30 +422,6 @@ public boolean cpFailedTrigger() { && this.cpFailureAction != null; } - /** - * Local compilation and packaging working directory - */ - @JsonIgnore - public String getDistHome() { - String path = String.format("%s/%s/%s", Workspace.APP_LOCAL_DIST(), projectId.toString(), getModule()); - log.info("local distHome:{}", path); - return path; - } - - @JsonIgnore - public String getLocalAppHome() { - String path = String.format("%s/%s", Workspace.local().APP_WORKSPACE(), id.toString()); - log.info("local appHome:{}", path); - return path; - } - - @JsonIgnore - public String getRemoteAppHome() { - String path = String.format("%s/%s", Workspace.remote().APP_WORKSPACE(), id.toString()); - log.info("remote appHome:{}", path); - return path; - } - /** * Automatically identify remoteAppHome or localAppHome based on app FlinkDeployMode */ @@ -492,12 +470,11 @@ public ApplicationType getApplicationType() { @JsonIgnore @SneakyThrows - @SuppressWarnings("unchecked") public Map getOptionMap() { if (StringUtils.isBlank(this.options)) { return new HashMap<>(); } - Map optionMap = JacksonUtils.read(this.options, Map.class); + Map optionMap = JacksonUtils.read(this.options, STRING_OBJECT_MAP); optionMap.entrySet().removeIf(entry -> entry.getValue() == null); return optionMap; } @@ -540,29 +517,11 @@ public Dependency getDependencyObject() { return Dependency.toDependency(this.dependency); } - @JsonIgnore - public DependencyInfo getDependencyInfo() { - return Dependency.toDependency(getDependency()).toJarPackDeps(); - } - @JsonIgnore public boolean isRunning() { return FlinkAppStateEnum.RUNNING.getValue() == this.getState(); } - @JsonIgnore - public boolean isNeedRollback() { - return ReleaseStateEnum.NEED_ROLLBACK.get() == this.getRelease(); - } - - @JsonIgnore - public boolean isNeedRestartOnFailed() { - if (this.restartSize != null && this.restartCount != null) { - return this.restartSize > 0 && this.restartCount <= this.restartSize; - } - return false; - } - @JsonIgnore public StorageType getStorageType() { return getStorageType(getDeployMode()); @@ -584,22 +543,11 @@ public static StorageType getStorageType(Integer deployMode) { } } - @JsonIgnore - public FsOperator getFsOperator() { - return FsOperator.of(getStorageType()); - } - - @JsonIgnore - public Workspace getWorkspace() { - return Workspace.of(getStorageType()); - } - @JsonIgnore @SneakyThrows - @SuppressWarnings("unchecked") public Map getHotParamsMap() { if (this.hotParams != null) { - Map map = JacksonUtils.read(this.hotParams, Map.class); + Map map = JacksonUtils.read(this.hotParams, STRING_OBJECT_MAP); map.entrySet().removeIf(entry -> entry.getValue() == null); return map; } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java index 7b13727a93..66942473b5 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java @@ -177,7 +177,7 @@ private Float getVersionNumber() { if (StringUtils.isNotBlank(this.version)) { return Float.parseFloat(getVersionOfFirst() + "." + getVersionOfMiddle()); } - throw new RuntimeException("Flink version is null"); + throw new IllegalStateException("Flink version is null"); } public boolean isLegacyFlinkConf() { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ReleaseOutcomeTarget.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ReleaseOutcomeTarget.java new file mode 100644 index 0000000000..eb48a42466 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ReleaseOutcomeTarget.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.entity; + +public interface ReleaseOutcomeTarget { + + boolean isRunning(); + + void setRelease(Integer release); + + void setOptionState(Integer optionState); + + void setBuild(Boolean build); +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java index 01815b7658..2f59d1cd17 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java @@ -18,13 +18,11 @@ package org.apache.streampark.console.core.entity; import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.Workspace; import org.apache.streampark.common.constants.Constants; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.SparkDeployMode; import org.apache.streampark.common.enums.SparkJobType; import org.apache.streampark.common.enums.StorageType; -import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.console.base.mybatis.entity.BaseEntity; import org.apache.streampark.console.base.util.JacksonUtils; import org.apache.streampark.console.core.bean.AppControl; @@ -34,7 +32,6 @@ import org.apache.streampark.console.core.enums.SparkAppStateEnum; import org.apache.streampark.console.core.metrics.spark.SparkApplicationSummary; import org.apache.streampark.console.core.util.YarnQueueLabelExpression; -import org.apache.streampark.flink.packer.maven.DependencyInfo; import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates; import org.apache.commons.lang3.StringUtils; @@ -59,7 +56,7 @@ @Setter @TableName("t_spark_app") @Slf4j -public class SparkApplication extends BaseEntity { +public class SparkApplication extends BaseEntity implements ApplicationEntitySupport, ReleaseOutcomeTarget { @TableId(type = IdType.AUTO) private Long id; @@ -337,27 +334,6 @@ public SparkDeployMode getDeployModeEnum() { } /** Local compilation and packaging working directory */ - @JsonIgnore - public String getDistHome() { - String path = String.format("%s/%s/%s", Workspace.APP_LOCAL_DIST(), projectId.toString(), getModule()); - log.info("local distHome:{}", path); - return path; - } - - @JsonIgnore - public String getLocalAppHome() { - String path = String.format("%s/%s", Workspace.local().APP_WORKSPACE(), id.toString()); - log.info("local appHome:{}", path); - return path; - } - - @JsonIgnore - public String getRemoteAppHome() { - String path = String.format("%s/%s", Workspace.remote().APP_WORKSPACE(), id.toString()); - log.info("remote appHome:{}", path); - return path; - } - /** Automatically identify remoteAppHome or localAppHome based on app SparkDeployMode */ @JsonIgnore public String getAppHome() { @@ -446,29 +422,11 @@ public Dependency getDependencyObject() { return Dependency.toDependency(this.dependency); } - @JsonIgnore - public DependencyInfo getDependencyInfo() { - return Dependency.toDependency(getDependency()).toJarPackDeps(); - } - @JsonIgnore public boolean isRunning() { return SparkAppStateEnum.RUNNING.getValue() == this.getState(); } - @JsonIgnore - public boolean isNeedRollback() { - return ReleaseStateEnum.NEED_ROLLBACK.get() == this.getRelease(); - } - - @JsonIgnore - public boolean isNeedRestartOnFailed() { - if (this.restartSize != null && this.restartCount != null) { - return this.restartSize > 0 && this.restartCount <= this.restartSize; - } - return false; - } - @JsonIgnore public StorageType getStorageType() { return getStorageType(getDeployMode()); @@ -489,16 +447,6 @@ public static StorageType getStorageType(Integer deployMode) { } } - @JsonIgnore - public FsOperator getFsOperator() { - return FsOperator.of(getStorageType()); - } - - @JsonIgnore - public Workspace getWorkspace() { - return Workspace.of(getStorageType()); - } - public void fillRunningMetrics(SparkApplicationSummary summary) { this.setNumTasks(summary.getNumTasks()); this.setNumCompletedTasks(summary.getNumCompletedTasks()); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/AbstractApplicationBuildPipelineService.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/AbstractApplicationBuildPipelineService.java new file mode 100644 index 0000000000..252dc7e510 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/AbstractApplicationBuildPipelineService.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.service.application.impl; + +import org.apache.streampark.console.core.entity.ApplicationBuildPipeline; +import org.apache.streampark.console.core.mapper.ApplicationBuildPipelineMapper; +import org.apache.streampark.flink.packer.pipeline.PipelineStatusEnum; + +import org.apache.commons.collections.CollectionUtils; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +import javax.annotation.Nonnull; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +public abstract class AbstractApplicationBuildPipelineService + extends + ServiceImpl { + + public Optional getCurrentBuildPipeline(@Nonnull Long appId) { + return Optional.ofNullable(getById(appId)); + } + + public boolean allowToBuildNow(@Nonnull Long appId) { + return getCurrentBuildPipeline(appId) + .map(pipeline -> PipelineStatusEnum.RUNNING != pipeline.getPipelineStatus()) + .orElse(true); + } + + public Map listAppIdPipelineStatusMap(List appIds) { + if (CollectionUtils.isEmpty(appIds)) { + return new HashMap<>(); + } + List appBuildPipelines = + this.lambdaQuery().in(ApplicationBuildPipeline::getAppId, appIds).list(); + if (CollectionUtils.isEmpty(appBuildPipelines)) { + return new HashMap<>(); + } + return appBuildPipelines.stream() + .collect(Collectors.toMap(ApplicationBuildPipeline::getAppId, ApplicationBuildPipeline::getPipelineStatus)); + } + + public void removeByAppId(Long appId) { + this.lambdaUpdate().eq(ApplicationBuildPipeline::getAppId, appId).remove(); + } + + public boolean saveEntity(ApplicationBuildPipeline pipe) { + ApplicationBuildPipeline old = getById(pipe.getAppId()); + if (old == null) { + return save(pipe); + } + return updateById(pipe); + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java index 299f68ab4f..f6570c4920 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java @@ -626,7 +626,8 @@ private boolean checkAppRepeatInYarn(String jobName) { } return false; } catch (Exception e) { - throw new RuntimeException("The yarn api is abnormal. Ensure that yarn is running properly."); + throw new IllegalStateException( + "The yarn api is abnormal. Ensure that yarn is running properly.", e); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java index bd7e24647b..af204ce492 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java @@ -21,14 +21,9 @@ import org.apache.streampark.common.constants.Constants; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.ExceptionUtils; import org.apache.streampark.common.util.FileUtils; import org.apache.streampark.console.base.exception.ApiAlertException; -import org.apache.streampark.console.base.util.JacksonUtils; -import org.apache.streampark.console.base.util.WebUtils; -import org.apache.streampark.console.core.bean.Dependency; import org.apache.streampark.console.core.bean.DockerConfig; import org.apache.streampark.console.core.entity.ApplicationBuildPipeline; import org.apache.streampark.console.core.entity.ApplicationLog; @@ -36,14 +31,10 @@ import org.apache.streampark.console.core.entity.FlinkApplicationConfig; import org.apache.streampark.console.core.entity.FlinkEnv; import org.apache.streampark.console.core.entity.FlinkSql; -import org.apache.streampark.console.core.entity.Message; import org.apache.streampark.console.core.entity.Resource; import org.apache.streampark.console.core.enums.CandidateTypeEnum; -import org.apache.streampark.console.core.enums.NoticeTypeEnum; import org.apache.streampark.console.core.enums.OptionStateEnum; import org.apache.streampark.console.core.enums.ReleaseStateEnum; -import org.apache.streampark.console.core.enums.ResourceTypeEnum; -import org.apache.streampark.console.core.mapper.ApplicationBuildPipelineMapper; import org.apache.streampark.console.core.service.FlinkEnvService; import org.apache.streampark.console.core.service.FlinkSqlService; import org.apache.streampark.console.core.service.MessageService; @@ -56,10 +47,10 @@ import org.apache.streampark.console.core.service.application.FlinkApplicationConfigService; import org.apache.streampark.console.core.service.application.FlinkApplicationInfoService; import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; +import org.apache.streampark.console.core.util.ApplicationBuildPipelineUtils; import org.apache.streampark.console.core.util.ServiceHelper; import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; import org.apache.streampark.flink.packer.docker.DockerConf; -import org.apache.streampark.flink.packer.maven.Artifact; import org.apache.streampark.flink.packer.maven.DependencyInfo; import org.apache.streampark.flink.packer.pipeline.BuildPipeline; import org.apache.streampark.flink.packer.pipeline.BuildResult; @@ -74,18 +65,14 @@ import org.apache.streampark.flink.packer.pipeline.FlinkYarnApplicationBuildRequest; import org.apache.streampark.flink.packer.pipeline.PipeWatcher; import org.apache.streampark.flink.packer.pipeline.PipelineSnapshot; -import org.apache.streampark.flink.packer.pipeline.PipelineStatusEnum; import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum; import org.apache.streampark.flink.packer.pipeline.impl.FlinkK8sApplicationBuildPipeline; import org.apache.streampark.flink.packer.pipeline.impl.FlinkK8sSessionBuildPipeline; import org.apache.streampark.flink.packer.pipeline.impl.FlinkRemoteBuildPipeline; import org.apache.streampark.flink.packer.pipeline.impl.FlinkYarnApplicationBuildPipeline; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.fasterxml.jackson.core.JsonProcessingException; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import lombok.extern.slf4j.Slf4j; @@ -97,27 +84,17 @@ import javax.annotation.Nonnull; -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.apache.streampark.common.enums.ApplicationType.APACHE_FLINK; -import static org.apache.streampark.console.core.enums.OperationEnum.RELEASE; @Service @Slf4j @Transactional(propagation = Propagation.SUPPORTS, rollbackFor = Exception.class) public class FlinkApplicationBuildPipelineServiceImpl extends - ServiceImpl + AbstractApplicationBuildPipelineService implements FlinkApplicationBuildPipelineService { @@ -180,7 +157,7 @@ public boolean buildApplication(@Nonnull Long appId, boolean forceBuild) { checkBuildEnv(appId, forceBuild); FlinkApplication app = applicationManageService.getById(appId); - ApplicationLog applicationLog = getApplicationLog(app); + ApplicationLog applicationLog = ApplicationBuildPipelineUtils.createReleaseLog(app.getId(), null); // check if you need to go through the build process (if the jar and pom have changed, // you need to go through the build process, if other common parameters are modified, @@ -219,9 +196,7 @@ public boolean buildApplication(@Nonnull Long appId, boolean forceBuild) { @Override public void onStart(PipelineSnapshot snapshot) { - ApplicationBuildPipeline buildPipeline = ApplicationBuildPipeline.fromPipeSnapshot(snapshot) - .setAppId(app.getId()); - saveEntity(buildPipeline); + saveEntity(ApplicationBuildPipelineUtils.fromSnapshot(snapshot, app.getId())); app.setRelease(ReleaseStateEnum.RELEASING.get()); applicationManageService.updateRelease(app); @@ -230,116 +205,51 @@ public void onStart(PipelineSnapshot snapshot) { flinkAppHttpWatcher.init(); } - // 1) checkEnv applicationInfoService.checkEnv(app); - // 2) some preparatory work String appUploads = app.getWorkspace().APP_UPLOADS(); - - if (app.isFlinkJarOrPyFlink()) { - // flinkJar upload jar to appHome... - String appHome = app.getAppHome(); - FsOperator fsOperator = app.getFsOperator(); - fsOperator.delete(appHome); - if (app.isUploadResource()) { - String uploadJar = appUploads.concat("/").concat(app.getJar()); - File localJar = new File( - String.format( - "%s/%d/%s", - Workspace.local().APP_UPLOADS(), - app.getTeamId(), - app.getJar())); - if (!localJar.exists()) { - Resource resource = resourceService.findByResourceName(app.getTeamId(), - app.getJar()); - if (resource != null && StringUtils.isNotBlank(resource.getFilePath())) { - localJar = new File(resource.getFilePath()); - uploadJar = appUploads.concat("/").concat(localJar.getName()); - } else { - localJar = - new File(WebUtils.getAppTempDir(), app.getJar()); - uploadJar = appUploads.concat("/").concat(localJar.getName()); - } - } - // upload jar copy to appHome - checkOrElseUploadJar(app.getFsOperator(), localJar, uploadJar, appUploads); - - switch (app.getApplicationType()) { - case STREAMPARK_FLINK: - fsOperator.mkdirs(app.getAppLib()); - fsOperator.copy(uploadJar, app.getAppLib(), false, true); - break; - case APACHE_FLINK: - fsOperator.mkdirs(appHome); - fsOperator.copy(uploadJar, appHome, false, true); - break; - default: - throw new IllegalArgumentException( - "[StreamPark] unsupported ApplicationType of FlinkJar: " - + app.getApplicationType()); - } - } else { - fsOperator.upload(app.getDistHome(), appHome); - } - } else { - if (!app.getDependencyObject().getJar().isEmpty()) { - String localUploads = Workspace.local().APP_UPLOADS(); - // copy jar to local upload dir - for (String jar : app.getDependencyObject().getJar()) { - File localJar = new File(WebUtils.getAppTempDir(), jar); - File uploadJar = new File(localUploads, jar); - if (!localJar.exists() && !uploadJar.exists()) { - throw new ApiAlertException( - "Missing file: " + jar + ", please upload again"); - } - if (localJar.exists()) { - checkOrElseUploadJar( - FsOperator.lfs(), localJar, uploadJar.getAbsolutePath(), - localUploads); - } - } - } - } + ApplicationBuildPipelineUtils.prepareBuildResources( + app.isFlinkJarOrPyFlink(), + () -> ApplicationBuildPipelineUtils.prepareJarJobHome( + app.getTeamId(), + app.getJar(), + app.getAppHome(), + app.getAppLib(), + app.getDistHome(), + app.getFsOperator(), + appUploads, + app.isUploadResource(), + app.getApplicationType(), + resourceService, + true), + app.getDependencyObject(), + resourceService); } @Override public void onStepStateChange(PipelineSnapshot snapshot) { - ApplicationBuildPipeline buildPipeline = ApplicationBuildPipeline.fromPipeSnapshot(snapshot) - .setAppId(app.getId()); - saveEntity(buildPipeline); + saveEntity(ApplicationBuildPipelineUtils.fromSnapshot(snapshot, app.getId())); } @Override public void onFinish(PipelineSnapshot snapshot, BuildResult result) { - ApplicationBuildPipeline buildPipeline = ApplicationBuildPipeline.fromPipeSnapshot(snapshot) - .setAppId(app.getId()) - .setBuildResult(result); - saveEntity(buildPipeline); + saveEntity(ApplicationBuildPipelineUtils.finishedSnapshot(snapshot, result, app.getId())); if (result.pass()) { - // running job ... - if (app.isRunning()) { - app.setRelease(ReleaseStateEnum.NEED_RESTART.get()); - } else { - app.setOptionState(OptionStateEnum.NONE.getValue()); - app.setRelease(ReleaseStateEnum.DONE.get()); - // If the current task is not running, or the task has just been added, directly - // set - // the candidate version to the official version - if (app.isFlinkSql()) { - applicationManageService.toEffective(app); - } else { - if (app.isStreamParkType()) { + ApplicationBuildPipelineUtils.applySuccessfulRelease( + app, + () -> { + if (app.isFlinkSql()) { + applicationManageService.toEffective(app); + } else if (app.isStreamParkType()) { FlinkApplicationConfig config = applicationConfigService.getLatest(app.getId()); if (config != null) { config.setToApplication(app); - applicationConfigService.toEffective(app.getId(), - app.getConfigId()); + applicationConfigService.toEffective( + app.getId(), app.getConfigId()); } } - } - } - // backup. + }); if (!app.isNeedRollback()) { if (app.isFlinkSql() && newFlinkSql != null) { backUpService.backup(app, newFlinkSql); @@ -348,28 +258,22 @@ public void onFinish(PipelineSnapshot snapshot, BuildResult result) { } } applicationLog.setSuccess(true); - app.setBuild(false); - } else { - Message message = new Message( - ServiceHelper.getUserId(), - app.getId(), - app.getJobName().concat(" release failed"), - ExceptionUtils.stringifyException(snapshot.error().exception()), - NoticeTypeEnum.EXCEPTION); - messageService.push(message); + ApplicationBuildPipelineUtils.recordReleaseFailure( + app.getId(), app.getJobName(), snapshot, applicationLog, messageService); app.setRelease(ReleaseStateEnum.FAILED.get()); app.setOptionState(OptionStateEnum.NONE.getValue()); app.setBuild(true); - applicationLog.setException( - ExceptionUtils.stringifyException(snapshot.error().exception())); - applicationLog.setSuccess(false); - } - applicationManageService.updateRelease(app); - applicationLogService.save(applicationLog); - if (flinkAppHttpWatcher.isWatchingApp(app.getId())) { - flinkAppHttpWatcher.init(); } + ApplicationBuildPipelineUtils.finalizeRelease( + () -> applicationManageService.updateRelease(app), + applicationLog, + applicationLogService, + () -> { + if (flinkAppHttpWatcher.isWatchingApp(app.getId())) { + flinkAppHttpWatcher.init(); + } + }); } }); // save docker resolve progress detail to cache, only for flink-k8s application mode. @@ -411,16 +315,6 @@ public void onDockerPushProgressChange(DockerPushSnapshot snapshot) { }); } - @Nonnull - private ApplicationLog getApplicationLog(FlinkApplication app) { - ApplicationLog applicationLog = new ApplicationLog(); - applicationLog.setOptionName(RELEASE.getValue()); - applicationLog.setAppId(app.getId()); - applicationLog.setCreateTime(new Date()); - applicationLog.setUserId(ServiceHelper.getUserId()); - return applicationLog; - } - /** * check the build environment * @@ -614,11 +508,6 @@ private String retrieveFlinkUserJar(FlinkEnv flinkEnv, FlinkApplication app) { } } - @Override - public Optional getCurrentBuildPipeline(@Nonnull Long appId) { - return Optional.ofNullable(getById(appId)); - } - @Override public DockerResolvedSnapshot getDockerProgressDetailSnapshot(@Nonnull Long appId) { return new DockerResolvedSnapshot( @@ -627,138 +516,13 @@ public DockerResolvedSnapshot getDockerProgressDetailSnapshot(@Nonnull Long appI DOCKER_PUSH_PG_SNAPSHOTS.getIfPresent(appId)); } - @Override - public boolean allowToBuildNow(@Nonnull Long appId) { - return getCurrentBuildPipeline(appId) - .map(pipeline -> PipelineStatusEnum.running != pipeline.getPipelineStatus()) - .orElse(true); - } - - @Override - public Map listAppIdPipelineStatusMap(List appIds) { - if (CollectionUtils.isEmpty(appIds)) { - return new HashMap<>(); - } - List appBuildPipelines = - this.lambdaQuery().in(ApplicationBuildPipeline::getAppId, appIds).list(); - if (CollectionUtils.isEmpty(appBuildPipelines)) { - return new HashMap<>(); - } - return appBuildPipelines.stream() - .collect(Collectors.toMap(ApplicationBuildPipeline::getAppId, ApplicationBuildPipeline::getPipelineStatus)); - } - - @Override - public void removeByAppId(Long appId) { - this.lambdaUpdate().eq(ApplicationBuildPipeline::getAppId, appId).remove(); - } - - /** - * save or update build pipeline - * - * @param pipe application build pipeline - * @return value after the save or update - */ - public boolean saveEntity(ApplicationBuildPipeline pipe) { - ApplicationBuildPipeline old = getById(pipe.getAppId()); - if (old == null) { - return save(pipe); - } - return updateById(pipe); - } - - /** - * Check if the jar exists, and upload a copy if it does not exist - * - * @param fsOperator - * @param localJar - * @param targetJar - * @param targetDir - */ - private void checkOrElseUploadJar( - FsOperator fsOperator, File localJar, String targetJar, String targetDir) { - if (!fsOperator.exists(targetJar)) { - fsOperator.upload(localJar.getAbsolutePath(), targetDir, false, true); - } else { - // The file exists to check whether it is consistent, and if it is inconsistent, re-upload it - if (!FileUtils.equals(localJar, new File(targetJar))) { - fsOperator.upload(localJar.getAbsolutePath(), targetDir, false, true); - } - } - } - - /** - * Gets and parses dependencies on the application - * - * @param application - * @return DependencyInfo - */ private DependencyInfo getMergedDependencyInfo(FlinkApplication application) { - DependencyInfo dependencyInfo = application.getDependencyInfo(); - if (StringUtils.isBlank(application.getTeamResource())) { - return dependencyInfo; - } - - try { - String[] resourceIds = JacksonUtils.read(application.getTeamResource(), String[].class); - - List mvnArtifacts = new ArrayList(); - List jarLibs = new ArrayList(); - - Arrays.stream(resourceIds) - .forEach( - resourceId -> { - Resource resource = resourceService.getById(resourceId); - - if (resource.getResourceType() != ResourceTypeEnum.GROUP) { - mergeDependency(application, mvnArtifacts, jarLibs, resource); - } else { - try { - String[] groupElements = - JacksonUtils.read(resource.getResource(), - String[].class); - Arrays.stream(groupElements) - .forEach( - resourceIdInGroup -> mergeDependency( - application, - mvnArtifacts, - jarLibs, - resourceService.getById( - resourceIdInGroup))); - } catch (JsonProcessingException e) { - throw new ApiAlertException("Parse resource group failed.", e); - } - } - }); - return dependencyInfo.merge(mvnArtifacts, jarLibs); - } catch (Exception e) { - log.error("Merge team dependency failed.", e); - return dependencyInfo; - } - } - - private static void mergeDependency( - FlinkApplication application, - List mvnArtifacts, - List jarLibs, - Resource resource) { - Dependency dependency = Dependency.toDependency(resource.getResource()); - dependency - .getPom() - .forEach( - pom -> mvnArtifacts.add( - new Artifact( - pom.getGroupId(), - pom.getArtifactId(), - pom.getVersion(), - pom.getClassifier()))); - dependency - .getJar() - .forEach( - jar -> jarLibs.add( - String.format( - "%s/%d/%s", - Workspace.local().APP_UPLOADS(), - application.getTeamId(), jar))); + return ApplicationBuildPipelineUtils.getMergedDependencyInfo( + application.getDependencyInfo(), + application.getTeamResource(), + application.getTeamId(), + resourceService, + log, + true); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java index c33e1e89bb..153498dccc 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java @@ -355,8 +355,8 @@ public List getYarnAppReport(String appName) { .filter(report -> report.getName().equals(appName)) .collect(Collectors.toList()); } catch (Exception e) { - throw new RuntimeException( - "getYarnAppReport failed. Ensure that yarn is running properly. ", e); + throw new IllegalStateException( + "getYarnAppReport failed. Ensure that yarn is running properly.", e); } } @@ -377,7 +377,7 @@ public String k8sStartLog(Long id, Integer offset, Integer limit) throws Excepti application.getJobName(), application.getJobId()); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to watch Kubernetes deployment log", e); } }); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationManageServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationManageServiceImpl.java index 0d18aab3d2..5d6a489e0b 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationManageServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationManageServiceImpl.java @@ -289,7 +289,8 @@ record -> { if (pipeStates.containsKey(record.getId())) { record.setBuildStatus(pipeStates.get(record.getId()).getCode()); } - AppControl appControl = getAppControl(record); + AppControl appControl = AppControl.fromPipelineBuildStatus( + record.getBuildStatus(), record.shouldTracking(), record.isRunning()); record.setAppControl(appControl); }) .collect(Collectors.toList()); @@ -305,20 +306,6 @@ private void setAppDurationIfNeeded(FlinkApplication record, long now) { } } - private AppControl getAppControl(FlinkApplication record) { - return new AppControl() - .setAllowBuild( - record.getBuildStatus() == null - || !PipelineStatusEnum.running.getCode() - .equals(record.getBuildStatus())) - .setAllowStart( - !record.shouldTracking() - && PipelineStatusEnum.success.getCode() - .equals(record.getBuildStatus())) - .setAllowStop(record.isRunning()) - .setAllowView(record.shouldTracking()); - } - @Override public void changeOwnership(Long userId, Long targetUserId) { this.lambdaUpdate().eq(FlinkApplication::getUserId, userId) @@ -507,7 +494,7 @@ public boolean update(FlinkApplication appParam) { } } catch (IOException e) { log.error("Error in checksumCRC32 for {}.", jarFile); - throw new RuntimeException(e); + throw new IllegalStateException("Failed to compute jar checksum for " + jarFile, e); } } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java index 2a690908ab..db0268170e 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java @@ -21,18 +21,12 @@ import org.apache.streampark.common.constants.Constants; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.SparkDeployMode; -import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.ExceptionUtils; import org.apache.streampark.common.util.FileUtils; import org.apache.streampark.console.base.exception.ApiAlertException; -import org.apache.streampark.console.base.util.JacksonUtils; -import org.apache.streampark.console.base.util.WebUtils; -import org.apache.streampark.console.core.bean.Dependency; import org.apache.streampark.console.core.bean.DockerConfig; import org.apache.streampark.console.core.entity.ApplicationBuildPipeline; import org.apache.streampark.console.core.entity.ApplicationLog; -import org.apache.streampark.console.core.entity.Message; import org.apache.streampark.console.core.entity.Resource; import org.apache.streampark.console.core.entity.SparkApplication; import org.apache.streampark.console.core.entity.SparkApplicationConfig; @@ -40,11 +34,8 @@ import org.apache.streampark.console.core.entity.SparkSql; import org.apache.streampark.console.core.enums.CandidateTypeEnum; import org.apache.streampark.console.core.enums.EngineTypeEnum; -import org.apache.streampark.console.core.enums.NoticeTypeEnum; import org.apache.streampark.console.core.enums.OptionStateEnum; import org.apache.streampark.console.core.enums.ReleaseStateEnum; -import org.apache.streampark.console.core.enums.ResourceTypeEnum; -import org.apache.streampark.console.core.mapper.ApplicationBuildPipelineMapper; import org.apache.streampark.console.core.service.MessageService; import org.apache.streampark.console.core.service.ResourceService; import org.apache.streampark.console.core.service.SettingService; @@ -55,26 +46,22 @@ import org.apache.streampark.console.core.service.application.SparkApplicationConfigService; import org.apache.streampark.console.core.service.application.SparkApplicationInfoService; import org.apache.streampark.console.core.service.application.SparkApplicationManageService; +import org.apache.streampark.console.core.util.ApplicationBuildPipelineUtils; import org.apache.streampark.console.core.util.ServiceHelper; import org.apache.streampark.console.core.watcher.SparkAppHttpWatcher; import org.apache.streampark.flink.packer.docker.DockerConf; -import org.apache.streampark.flink.packer.maven.Artifact; import org.apache.streampark.flink.packer.maven.DependencyInfo; import org.apache.streampark.flink.packer.pipeline.BuildPipeline; import org.apache.streampark.flink.packer.pipeline.BuildResult; import org.apache.streampark.flink.packer.pipeline.PipeWatcher; import org.apache.streampark.flink.packer.pipeline.PipelineSnapshot; -import org.apache.streampark.flink.packer.pipeline.PipelineStatusEnum; import org.apache.streampark.flink.packer.pipeline.SparkK8sApplicationBuildRequest; import org.apache.streampark.flink.packer.pipeline.SparkYarnBuildRequest; import org.apache.streampark.flink.packer.pipeline.impl.SparkK8sApplicationBuildPipeline; import org.apache.streampark.flink.packer.pipeline.impl.SparkYarnBuildPipeline; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.fasterxml.jackson.core.JsonProcessingException; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -84,25 +71,14 @@ import javax.annotation.Nonnull; -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.concurrent.ExecutorService; -import java.util.stream.Collectors; - -import static org.apache.streampark.console.core.enums.OperationEnum.RELEASE; @Service @Slf4j @Transactional(propagation = Propagation.SUPPORTS, rollbackFor = Exception.class) public class SparkApplicationBuildPipelineServiceImpl extends - ServiceImpl + AbstractApplicationBuildPipelineService implements SparkAplicationBuildPipelineService { @@ -153,12 +129,8 @@ public boolean buildApplication(@Nonnull Long appId, boolean forceBuild) { checkBuildEnv(appId, forceBuild); SparkApplication app = applicationManageService.getById(appId); - ApplicationLog applicationLog = new ApplicationLog(); - applicationLog.setJobType(EngineTypeEnum.SPARK.getCode()); - applicationLog.setOptionName(RELEASE.getValue()); - applicationLog.setAppId(app.getId()); - applicationLog.setCreateTime(new Date()); - applicationLog.setUserId(ServiceHelper.getUserId()); + ApplicationLog applicationLog = + ApplicationBuildPipelineUtils.createReleaseLog(app.getId(), EngineTypeEnum.SPARK.getCode()); // check if you need to go through the build process (if the jar and pom have changed, // you need to go through the build process, if other common parameters are modified, @@ -192,9 +164,7 @@ public boolean buildApplication(@Nonnull Long appId, boolean forceBuild) { @Override public void onStart(PipelineSnapshot snapshot) { - ApplicationBuildPipeline buildPipeline = ApplicationBuildPipeline.fromPipeSnapshot(snapshot) - .setAppId(app.getId()); - saveEntity(buildPipeline); + saveEntity(ApplicationBuildPipelineUtils.fromSnapshot(snapshot, app.getId())); app.setRelease(ReleaseStateEnum.RELEASING.get()); applicationManageService.updateRelease(app); @@ -203,134 +173,68 @@ public void onStart(PipelineSnapshot snapshot) { sparkAppHttpWatcher.init(); } - // 1) checkEnv applicationInfoService.checkEnv(app); - // 2) some preparatory work String appUploads = app.getWorkspace().APP_UPLOADS(); - - if (app.isSparkJarOrPySparkJob()) { - // spark jar and pyspark upload resource to appHome... - String appHome = app.getAppHome(); - FsOperator fsOperator = app.getFsOperator(); - fsOperator.delete(appHome); - if (app.isFromUploadJob()) { - String uploadJar = appUploads.concat("/").concat(app.getJar()); - File localJar = new File( - String.format( - "%s/%d/%s", - Workspace.local().APP_UPLOADS(), - app.getTeamId(), - app.getJar())); - if (!localJar.exists()) { - Resource resource = resourceService.findByResourceName(app.getTeamId(), - app.getJar()); - if (resource != null && StringUtils.isNotBlank(resource.getFilePath())) { - localJar = new File(resource.getFilePath()); - uploadJar = appUploads.concat("/").concat(localJar.getName()); - } - } - // upload jar copy to appHome - checkOrElseUploadJar(app.getFsOperator(), localJar, uploadJar, appUploads); - - switch (app.getApplicationType()) { - case STREAMPARK_SPARK: - fsOperator.mkdirs(app.getAppLib()); - fsOperator.copy(uploadJar, app.getAppLib(), false, true); - break; - case APACHE_SPARK: - fsOperator.mkdirs(appHome); - fsOperator.copy(uploadJar, appHome, false, true); - break; - default: - throw new IllegalArgumentException( - "[StreamPark] unsupported ApplicationType of FlinkJar: " - + app.getApplicationType()); - } - } else { - fsOperator.upload(app.getDistHome(), appHome); - } - } else { - if (!app.getDependencyObject().getJar().isEmpty()) { - String localUploads = Workspace.local().APP_UPLOADS(); - // copy jar to local upload dir - for (String jar : app.getDependencyObject().getJar()) { - File localJar = new File(WebUtils.getAppTempDir(), jar); - File uploadJar = new File(localUploads, jar); - if (!localJar.exists() && !uploadJar.exists()) { - throw new ApiAlertException( - "Missing file: " + jar + ", please upload again"); - } - if (localJar.exists()) { - checkOrElseUploadJar( - FsOperator.lfs(), localJar, uploadJar.getAbsolutePath(), - localUploads); - } - } - } - } + ApplicationBuildPipelineUtils.prepareBuildResources( + app.isSparkJarOrPySparkJob(), + () -> ApplicationBuildPipelineUtils.prepareJarJobHome( + app.getTeamId(), + app.getJar(), + app.getAppHome(), + app.getAppLib(), + app.getDistHome(), + app.getFsOperator(), + appUploads, + app.isFromUploadJob(), + app.getApplicationType(), + resourceService, + false), + app.getDependencyObject(), + resourceService); } @Override public void onStepStateChange(PipelineSnapshot snapshot) { - ApplicationBuildPipeline buildPipeline = ApplicationBuildPipeline.fromPipeSnapshot(snapshot) - .setAppId(app.getId()); - saveEntity(buildPipeline); + saveEntity(ApplicationBuildPipelineUtils.fromSnapshot(snapshot, app.getId())); } @Override public void onFinish(PipelineSnapshot snapshot, BuildResult result) { - ApplicationBuildPipeline buildPipeline = ApplicationBuildPipeline.fromPipeSnapshot(snapshot) - .setAppId(app.getId()) - .setBuildResult(result); - saveEntity(buildPipeline); + saveEntity(ApplicationBuildPipelineUtils.finishedSnapshot(snapshot, result, app.getId())); if (result.pass()) { - // running job ... - if (app.isRunning()) { - app.setRelease(ReleaseStateEnum.NEED_RESTART.get()); - } else { - app.setOptionState(OptionStateEnum.NONE.getValue()); - app.setRelease(ReleaseStateEnum.DONE.get()); - // If the current task is not running, or the task has just been added, directly - // set - // the candidate version to the official version - if (app.isSparkOnYarnJob()) { - applicationManageService.toEffective(app); - } else { - if (app.isStreamParkJob()) { + ApplicationBuildPipelineUtils.applySuccessfulRelease( + app, + () -> { + if (app.isSparkOnYarnJob()) { + applicationManageService.toEffective(app); + } else if (app.isStreamParkJob()) { SparkApplicationConfig config = applicationConfigService.getLatest(app.getId()); if (config != null) { config.setToApplication(app); - applicationConfigService.toEffective(app.getId(), - app.getConfigId()); + applicationConfigService.toEffective( + app.getId(), app.getConfigId()); } } - } - } + }); applicationLog.setSuccess(true); - app.setBuild(false); - } else { - Message message = new Message( - ServiceHelper.getUserId(), - app.getId(), - app.getAppName().concat(" release failed"), - ExceptionUtils.stringifyException(snapshot.error().exception()), - NoticeTypeEnum.EXCEPTION); - messageService.push(message); + ApplicationBuildPipelineUtils.recordReleaseFailure( + app.getId(), app.getAppName(), snapshot, applicationLog, messageService); app.setRelease(ReleaseStateEnum.FAILED.get()); app.setOptionState(OptionStateEnum.NONE.getValue()); app.setBuild(true); - applicationLog.setException( - ExceptionUtils.stringifyException(snapshot.error().exception())); - applicationLog.setSuccess(false); - } - applicationManageService.updateRelease(app); - applicationLogService.save(applicationLog); - if (sparkAppHttpWatcher.isWatchingApp(app.getId())) { - sparkAppHttpWatcher.init(); } + ApplicationBuildPipelineUtils.finalizeRelease( + () -> applicationManageService.updateRelease(app), + applicationLog, + applicationLogService, + () -> { + if (sparkAppHttpWatcher.isWatchingApp(app.getId())) { + sparkAppHttpWatcher.init(); + } + }); } }); // save pipeline instance snapshot to db before release it. @@ -471,143 +375,12 @@ private String retrieveSparkUserJar(SparkEnv sparkEnv, SparkApplication app) { } } - @Override - public Optional getCurrentBuildPipeline(@Nonnull Long appId) { - return Optional.ofNullable(getById(appId)); - } - - @Override - public boolean allowToBuildNow(@Nonnull Long appId) { - return getCurrentBuildPipeline(appId) - .map(pipeline -> PipelineStatusEnum.running != pipeline.getPipelineStatus()) - .orElse(true); - } - - @Override - public Map listAppIdPipelineStatusMap(List appIds) { - if (CollectionUtils.isEmpty(appIds)) { - return new HashMap<>(); - } - List appBuildPipelines = - this.lambdaQuery().in(ApplicationBuildPipeline::getAppId, appIds).list(); - if (CollectionUtils.isEmpty(appBuildPipelines)) { - return new HashMap<>(); - } - return appBuildPipelines.stream() - .collect(Collectors.toMap(ApplicationBuildPipeline::getAppId, ApplicationBuildPipeline::getPipelineStatus)); - } - - @Override - public void removeByAppId(Long appId) { - this.lambdaUpdate().eq(ApplicationBuildPipeline::getAppId, appId).remove(); - } - - /** - * save or update build pipeline - * - * @param pipe application build pipeline - * @return value after the save or update - */ - public boolean saveEntity(ApplicationBuildPipeline pipe) { - ApplicationBuildPipeline old = getById(pipe.getAppId()); - if (old == null) { - return save(pipe); - } - return updateById(pipe); - } - - /** - * Check if the jar exists, and upload a copy if it does not exist - * - * @param fsOperator - * @param localJar - * @param targetJar - * @param targetDir - */ - private void checkOrElseUploadJar( - FsOperator fsOperator, File localJar, String targetJar, String targetDir) { - if (!fsOperator.exists(targetJar)) { - fsOperator.upload(localJar.getAbsolutePath(), targetDir, false, true); - } else { - // The file exists to check whether it is consistent, and if it is inconsistent, re-upload it - if (!FileUtils.equals(localJar, new File(targetJar))) { - fsOperator.upload(localJar.getAbsolutePath(), targetDir, false, true); - } - } - } - - /** - * Gets and parses dependencies on the application - * - * @param application - * @return DependencyInfo - */ private DependencyInfo getMergedDependencyInfo(SparkApplication application) { - DependencyInfo dependencyInfo = application.getDependencyInfo(); - if (StringUtils.isBlank(application.getTeamResource())) { - return dependencyInfo; - } - - try { - String[] resourceIds = JacksonUtils.read(application.getTeamResource(), String[].class); - - List mvnArtifacts = new ArrayList(); - List jarLibs = new ArrayList(); - - Arrays.stream(resourceIds) - .forEach( - resourceId -> { - Resource resource = resourceService.getById(resourceId); - - if (resource.getResourceType() != ResourceTypeEnum.GROUP) { - mergeDependency(application, mvnArtifacts, jarLibs, resource); - } else { - try { - String[] groupElements = - JacksonUtils.read(resource.getResource(), - String[].class); - Arrays.stream(groupElements) - .forEach( - resourceIdInGroup -> mergeDependency( - application, - mvnArtifacts, - jarLibs, - resourceService.getById( - resourceIdInGroup))); - } catch (JsonProcessingException e) { - throw new ApiAlertException("Parse resource group failed.", e); - } - } - }); - return dependencyInfo.merge(mvnArtifacts, jarLibs); - } catch (Exception e) { - log.warn("Merge team dependency failed.", e); - return dependencyInfo; - } - } - - private static void mergeDependency( - SparkApplication application, - List mvnArtifacts, - List jarLibs, - Resource resource) { - Dependency dependency = Dependency.toDependency(resource.getResource()); - dependency - .getPom() - .forEach( - pom -> mvnArtifacts.add( - new Artifact( - pom.getGroupId(), - pom.getArtifactId(), - pom.getVersion(), - pom.getClassifier()))); - dependency - .getJar() - .forEach( - jar -> jarLibs.add( - String.format( - "%s/%d/%s", - Workspace.local().APP_UPLOADS(), - application.getTeamId(), jar))); + return ApplicationBuildPipelineUtils.getMergedDependencyInfo( + application.getDependencyInfo(), + application.getTeamResource(), + application.getTeamId(), + resourceService, + log); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java index b81d43f03c..b0cf67c829 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java @@ -223,19 +223,9 @@ record -> { record.setBuildStatus(pipeStates.get(record.getId()).getCode()); } - AppControl appControl = new AppControl() - .setAllowBuild( - record.getBuildStatus() == null - || !PipelineStatusEnum.running - .getCode() - .equals(record.getBuildStatus())) - .setAllowStart( - !record.shouldTracking() - && PipelineStatusEnum.success - .getCode() - .equals(record.getBuildStatus())) - .setAllowStop(record.isRunning()) - .setAllowView(record.shouldTracking()); + AppControl appControl = + AppControl.fromPipelineBuildStatus( + record.getBuildStatus(), record.shouldTracking(), record.isRunning()); record.setAppControl(appControl); }) .collect(Collectors.toList()); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java index b45c8a9519..a3b60414d3 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java @@ -175,7 +175,7 @@ public void rollback(FlinkApplication application) { backUpService.rollbackFlinkSql(application, sql); } catch (Exception e) { log.error("Backup and Roll back FlinkSql before start failed."); - throw new RuntimeException(e.getMessage()); + throw new IllegalStateException("Backup and rollback FlinkSql failed", e); } } @@ -193,7 +193,8 @@ public FlinkSqlValidationResult verifySql(String sql, Long versionId) { if (result == null) { return null; } - return FlinkShimsProxy.getObject(this.getClass().getClassLoader(), result); + return FlinkShimsProxy.getObject( + this.getClass().getClassLoader(), result, FlinkSqlValidationResult.class); } catch (Throwable e) { log.error( "verifySql invocationTargetException: {}", diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java index 3571cbe27b..9570394fb1 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java @@ -154,7 +154,7 @@ private void deploy(Project project) throws Exception { // project) under the project path findTarOrJar(apps, path); if (apps.isEmpty()) { - throw new RuntimeException( + throw new IllegalStateException( "[StreamPark] can't find tar.gz or jar in " + path.getAbsolutePath()); } for (File app : apps) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationBuildPipelineUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationBuildPipelineUtils.java new file mode 100644 index 0000000000..1011bb150e --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationBuildPipelineUtils.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.util; + +import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.enums.ApplicationType; +import org.apache.streampark.common.fs.FsOperator; +import org.apache.streampark.common.util.ExceptionUtils; +import org.apache.streampark.common.util.FileUtils; +import org.apache.streampark.console.base.exception.ApiAlertException; +import org.apache.streampark.console.base.util.JacksonUtils; +import org.apache.streampark.console.base.util.WebUtils; +import org.apache.streampark.console.core.bean.Dependency; +import org.apache.streampark.console.core.entity.ApplicationBuildPipeline; +import org.apache.streampark.console.core.entity.ApplicationLog; +import org.apache.streampark.console.core.entity.Message; +import org.apache.streampark.console.core.entity.ReleaseOutcomeTarget; +import org.apache.streampark.console.core.entity.Resource; +import org.apache.streampark.console.core.enums.NoticeTypeEnum; +import org.apache.streampark.console.core.enums.OptionStateEnum; +import org.apache.streampark.console.core.enums.ReleaseStateEnum; +import org.apache.streampark.console.core.enums.ResourceTypeEnum; +import org.apache.streampark.console.core.service.MessageService; +import org.apache.streampark.console.core.service.ResourceService; +import org.apache.streampark.console.core.service.application.ApplicationLogService; +import org.apache.streampark.flink.packer.maven.Artifact; +import org.apache.streampark.flink.packer.maven.DependencyInfo; +import org.apache.streampark.flink.packer.pipeline.BuildResult; +import org.apache.streampark.flink.packer.pipeline.PipelineSnapshot; + +import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.slf4j.Logger; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +public final class ApplicationBuildPipelineUtils { + + private ApplicationBuildPipelineUtils() { + } + + public static ApplicationBuildPipeline finishedSnapshot( + PipelineSnapshot snapshot, + BuildResult result, + Long appId) { + return ApplicationBuildPipeline.fromPipeSnapshot(snapshot).setAppId(appId).setBuildResult(result); + } + + public static void applySuccessfulRelease(ReleaseOutcomeTarget app, Runnable promoteEffectiveVersion) { + if (app.isRunning()) { + app.setRelease(ReleaseStateEnum.NEED_RESTART.get()); + } else { + app.setOptionState(OptionStateEnum.NONE.getValue()); + app.setRelease(ReleaseStateEnum.DONE.get()); + promoteEffectiveVersion.run(); + } + app.setBuild(false); + } + + public static void finalizeRelease( + Runnable updateRelease, + ApplicationLog applicationLog, + ApplicationLogService applicationLogService, + Runnable refreshWatcherIfWatching) { + updateRelease.run(); + applicationLogService.save(applicationLog); + refreshWatcherIfWatching.run(); + } + + public static void prepareBuildResources( + boolean jarJob, + Runnable jarJobPreparer, + Dependency dependencyObject, + ResourceService resourceService) { + if (jarJob) { + jarJobPreparer.run(); + } else { + uploadSqlJobDependencies(dependencyObject, resourceService); + } + } + + public static ApplicationLog createReleaseLog(Long appId, Integer jobTypeCode) { + ApplicationLog applicationLog = new ApplicationLog(); + if (jobTypeCode != null) { + applicationLog.setJobType(jobTypeCode); + } + applicationLog.setOptionName(org.apache.streampark.console.core.enums.OperationEnum.RELEASE.getValue()); + applicationLog.setAppId(appId); + applicationLog.setCreateTime(new Date()); + applicationLog.setUserId(ServiceHelper.getUserId()); + return applicationLog; + } + + public static ApplicationBuildPipeline fromSnapshot(PipelineSnapshot snapshot, Long appId) { + return ApplicationBuildPipeline.fromPipeSnapshot(snapshot).setAppId(appId); + } + + public static void checkOrElseUploadJar( + FsOperator fsOperator, File localJar, String targetJar, String targetDir) { + if (!fsOperator.exists(targetJar)) { + fsOperator.upload(localJar.getAbsolutePath(), targetDir, false, true); + } else if (!FileUtils.equals(localJar, new File(targetJar))) { + fsOperator.upload(localJar.getAbsolutePath(), targetDir, false, true); + } + } + + public static void uploadSqlJobDependencies(Dependency dependencyObject, ResourceService resourceService) { + if (dependencyObject.getJar().isEmpty()) { + return; + } + String localUploads = Workspace.local().APP_UPLOADS(); + for (String jar : dependencyObject.getJar()) { + File localJar = new File(WebUtils.getAppTempDir(), jar); + File uploadJar = new File(localUploads, jar); + if (!localJar.exists() && !uploadJar.exists()) { + throw new ApiAlertException("Missing file: " + jar + ", please upload again"); + } + if (localJar.exists()) { + checkOrElseUploadJar(FsOperator.lfs(), localJar, uploadJar.getAbsolutePath(), localUploads); + } + } + } + + public static void prepareJarJobHome( + Long teamId, + String jar, + String appHome, + String appLib, + String distHome, + FsOperator fsOperator, + String appUploads, + boolean fromUpload, + ApplicationType applicationType, + ResourceService resourceService, + boolean tempDirFallback) { + fsOperator.delete(appHome); + if (!fromUpload) { + fsOperator.upload(distHome, appHome); + return; + } + + String uploadJar = appUploads.concat("/").concat(jar); + File localJar = new File(String.format("%s/%d/%s", Workspace.local().APP_UPLOADS(), teamId, jar)); + if (!localJar.exists()) { + Resource resource = resourceService.findByResourceName(teamId, jar); + if (resource != null && StringUtils.isNotBlank(resource.getFilePath())) { + localJar = new File(resource.getFilePath()); + uploadJar = appUploads.concat("/").concat(localJar.getName()); + } else if (tempDirFallback) { + localJar = new File(WebUtils.getAppTempDir(), jar); + uploadJar = appUploads.concat("/").concat(localJar.getName()); + } + } + checkOrElseUploadJar(fsOperator, localJar, uploadJar, appUploads); + + switch (applicationType) { + case STREAMPARK_FLINK: + case STREAMPARK_SPARK: + fsOperator.mkdirs(appLib); + fsOperator.copy(uploadJar, appLib, false, true); + break; + case APACHE_FLINK: + case APACHE_SPARK: + fsOperator.mkdirs(appHome); + fsOperator.copy(uploadJar, appHome, false, true); + break; + default: + throw new IllegalArgumentException( + "[StreamPark] unsupported ApplicationType: " + applicationType); + } + } + + public static DependencyInfo getMergedDependencyInfo( + DependencyInfo dependencyInfo, + String teamResource, + Long teamId, + ResourceService resourceService, + Logger log) { + return getMergedDependencyInfo(dependencyInfo, teamResource, teamId, resourceService, log, false); + } + + public static DependencyInfo getMergedDependencyInfo( + DependencyInfo dependencyInfo, + String teamResource, + Long teamId, + ResourceService resourceService, + Logger log, + boolean errorOnFailure) { + if (StringUtils.isBlank(teamResource)) { + return dependencyInfo; + } + + try { + String[] resourceIds = JacksonUtils.read(teamResource, String[].class); + + List mvnArtifacts = new ArrayList<>(); + List jarLibs = new ArrayList<>(); + + Arrays.stream(resourceIds) + .forEach( + resourceId -> { + Resource resource = resourceService.getById(resourceId); + + if (resource.getResourceType() != ResourceTypeEnum.GROUP) { + mergeDependency(teamId, mvnArtifacts, jarLibs, resource); + } else { + try { + String[] groupElements = + JacksonUtils.read(resource.getResource(), String[].class); + Arrays.stream(groupElements) + .forEach( + resourceIdInGroup -> mergeDependency( + teamId, + mvnArtifacts, + jarLibs, + resourceService.getById(resourceIdInGroup))); + } catch (JsonProcessingException e) { + throw new ApiAlertException("Parse resource group failed.", e); + } + } + }); + return dependencyInfo.merge(mvnArtifacts, jarLibs); + } catch (Exception e) { + if (errorOnFailure) { + log.error("Merge team dependency failed.", e); + } else { + log.warn("Merge team dependency failed.", e); + } + return dependencyInfo; + } + } + + public static void recordReleaseFailure( + Long appId, + String jobDisplayName, + PipelineSnapshot snapshot, + ApplicationLog applicationLog, + MessageService messageService) { + Message message = new Message( + ServiceHelper.getUserId(), + appId, + jobDisplayName.concat(" release failed"), + ExceptionUtils.stringifyException(snapshot.error().exception()), + NoticeTypeEnum.EXCEPTION); + messageService.push(message); + applicationLog.setException(ExceptionUtils.stringifyException(snapshot.error().exception())); + applicationLog.setSuccess(false); + } + + private static void mergeDependency( + Long teamId, + List mvnArtifacts, + List jarLibs, + Resource resource) { + Dependency dependency = Dependency.toDependency(resource.getResource()); + dependency + .getPom() + .forEach( + pom -> mvnArtifacts.add( + new Artifact( + pom.getGroupId(), + pom.getArtifactId(), + pom.getVersion(), + pom.getClassifier()))); + dependency + .getJar() + .forEach( + jar -> jarLibs.add( + String.format("%s/%d/%s", Workspace.local().APP_UPLOADS(), teamId, jar))); + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntityUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntityUtils.java new file mode 100644 index 0000000000..bc964d5e9a --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntityUtils.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.util; + +import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.enums.StorageType; +import org.apache.streampark.common.fs.FsOperator; +import org.apache.streampark.console.core.bean.Dependency; +import org.apache.streampark.console.core.enums.ReleaseStateEnum; +import org.apache.streampark.flink.packer.maven.DependencyInfo; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public final class ApplicationEntityUtils { + + private ApplicationEntityUtils() { + } + + public static String distHome(Long projectId, String module) { + String path = String.format("%s/%s/%s", Workspace.APP_LOCAL_DIST(), projectId.toString(), module); + log.info("local distHome:{}", path); + return path; + } + + public static String localAppHome(Long id) { + String path = String.format("%s/%s", Workspace.local().APP_WORKSPACE(), id.toString()); + log.info("local appHome:{}", path); + return path; + } + + public static String remoteAppHome(Long id) { + String path = String.format("%s/%s", Workspace.remote().APP_WORKSPACE(), id.toString()); + log.info("remote appHome:{}", path); + return path; + } + + public static DependencyInfo dependencyInfo(String dependency) { + return Dependency.toDependency(dependency).toJarPackDeps(); + } + + public static boolean needRollback(Integer release) { + return ReleaseStateEnum.NEED_ROLLBACK.get() == release; + } + + public static boolean needRestartOnFailed(Integer restartSize, Integer restartCount) { + if (restartSize != null && restartCount != null) { + return restartSize > 0 && restartCount <= restartSize; + } + return false; + } + + public static FsOperator fsOperator(StorageType storageType) { + return FsOperator.of(storageType); + } + + public static Workspace workspace(StorageType storageType) { + return Workspace.of(storageType); + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationWatcherUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationWatcherUtils.java new file mode 100644 index 0000000000..51c7ad53f4 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationWatcherUtils.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.util; + +import com.github.benmanes.caffeine.cache.Cache; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public final class ApplicationWatcherUtils { + + private ApplicationWatcherUtils() { + } + + public static void initWatchingApps( + Map watchingApps, + Cache startingCache, + List applications, + Function idExtractor, + Byte startingFlag) { + watchingApps.clear(); + applications.forEach( + app -> { + Long appId = idExtractor.apply(app); + watchingApps.put(appId, app); + startingCache.put(appId, startingFlag); + }); + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkAppHttpWatcher.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkAppHttpWatcher.java index aeec8cd8bd..845b215e8b 100755 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkAppHttpWatcher.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkAppHttpWatcher.java @@ -42,6 +42,7 @@ import org.apache.streampark.console.core.service.application.FlinkApplicationInfoService; import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; import org.apache.streampark.console.core.util.AlertTemplateUtils; +import org.apache.streampark.console.core.util.ApplicationWatcherUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hc.client5.http.config.RequestConfig; @@ -175,16 +176,15 @@ public class FlinkAppHttpWatcher { @PostConstruct public void init() { - WATCHING_APPS.clear(); - List applications = applicationManageService.list( - new LambdaQueryWrapper() - .eq(FlinkApplication::getTracking, 1) - .notIn(FlinkApplication::getDeployMode, FlinkDeployMode.getKubernetesMode())); - applications.forEach(app -> { - Long appId = app.getId(); - WATCHING_APPS.put(appId, app); - STARTING_CACHE.put(appId, DEFAULT_FLAG_BYTE); - }); + ApplicationWatcherUtils.initWatchingApps( + WATCHING_APPS, + STARTING_CACHE, + applicationManageService.list( + new LambdaQueryWrapper() + .eq(FlinkApplication::getTracking, 1) + .notIn(FlinkApplication::getDeployMode, FlinkDeployMode.getKubernetesMode())), + FlinkApplication::getId, + DEFAULT_FLAG_BYTE); } @PreDestroy @@ -589,7 +589,7 @@ private void getStateFromYarn(FlinkApplication application) throws Exception { YarnAppInfo yarnAppInfo = httpYarnAppInfo(application); if (yarnAppInfo == null) { if (!FlinkDeployMode.REMOTE.equals(application.getDeployModeEnum())) { - throw new RuntimeException( + throw new IllegalStateException( "[StreamPark][FlinkAppHttpWatcher] getFromYarnRestApi failed "); } } else { @@ -628,8 +628,8 @@ private void getStateFromYarn(FlinkApplication application) throws Exception { } } catch (Exception e) { if (!FlinkDeployMode.REMOTE.equals(application.getDeployModeEnum())) { - throw new RuntimeException( - "[StreamPark][FlinkAppHttpWatcher] getFromYarnRestApi error,", e); + throw new IllegalStateException( + "[StreamPark][FlinkAppHttpWatcher] getFromYarnRestApi error", e); } } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/SparkAppHttpWatcher.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/SparkAppHttpWatcher.java index 5383629d7e..c6a12118e5 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/SparkAppHttpWatcher.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/SparkAppHttpWatcher.java @@ -33,6 +33,7 @@ import org.apache.streampark.console.core.service.application.SparkApplicationInfoService; import org.apache.streampark.console.core.service.application.SparkApplicationManageService; import org.apache.streampark.console.core.util.AlertTemplateUtils; +import org.apache.streampark.console.core.util.ApplicationWatcherUtils; import org.apache.flink.annotation.VisibleForTesting; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -57,7 +58,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Date; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; @@ -125,17 +125,15 @@ public class SparkAppHttpWatcher { @PostConstruct public void init() { - WATCHING_APPS.clear(); - List applications = applicationManageService.list( - new LambdaQueryWrapper() - .eq(SparkApplication::getTracking, 1) - .ne(SparkApplication::getState, SparkAppStateEnum.LOST.getValue())); - - applications.forEach(app -> { - Long appId = app.getId(); - WATCHING_APPS.put(appId, app); - STARTING_CACHE.put(appId, DEFAULT_FLAG_BYTE); - }); + ApplicationWatcherUtils.initWatchingApps( + WATCHING_APPS, + STARTING_CACHE, + applicationManageService.list( + new LambdaQueryWrapper() + .eq(SparkApplication::getTracking, 1) + .ne(SparkApplication::getState, SparkAppStateEnum.LOST.getValue())), + SparkApplication::getId, + DEFAULT_FLAG_BYTE); } @PreDestroy diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/ShiroService.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/ShiroService.java index 8f212c4be1..cfcb943a70 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/ShiroService.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/ShiroService.java @@ -50,7 +50,7 @@ public void init() { try { shiroFilter = shiroFilterFactoryBean.getObject(); } catch (Exception e) { - throw new RuntimeException("Fail to get ShiroFilter from shiroFilterFactoryBean!"); + throw new IllegalStateException("Fail to get ShiroFilter from shiroFilterFactoryBean!", e); } securityManager = (DefaultWebSecurityManager) shiroFilter.getSecurityManager(); PathMatchingFilterChainResolver filterChainResolver = (PathMatchingFilterChainResolver) shiroFilter diff --git a/streampark-flink/pom.xml b/streampark-flink/pom.xml index beac918830..e4f4ffef66 100644 --- a/streampark-flink/pom.xml +++ b/streampark-flink/pom.xml @@ -40,23 +40,6 @@ - - org.apache.streampark - streampark-common-scala-bridge_${scala.binary.version} - - - - com.beachape - enumeratum_${scala.binary.version} - - - - org.scalatest - scalatest_${scala.binary.version} - ${scalatest.version} - test - - org.junit.jupiter junit-jupiter-engine @@ -82,13 +65,4 @@ - - - - net.alchim31.maven - scala-maven-plugin - - - - diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml index be2413c38e..bb2f5c371d 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml @@ -78,6 +78,18 @@ lombok provided + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.assertj + assertj-core + test + diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java index e1784690b5..71efdaf96e 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java @@ -20,7 +20,7 @@ import java.security.Permission; /** Used to mask JVM requests for external operations. */ -public class ExitSecurityManager extends SecurityManager { +public final class ExitSecurityManager extends SecurityManager { @Override public void checkExit(int status) { diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java index 73c0e89e21..516e8f4286 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java @@ -18,7 +18,6 @@ package org.apache.streampark.flink.client; import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.util.LoggerSupport; import org.apache.streampark.flink.client.bean.CancelRequest; import org.apache.streampark.flink.client.bean.CancelResponse; import org.apache.streampark.flink.client.bean.DeployRequest; @@ -31,9 +30,11 @@ import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; import org.apache.streampark.flink.proxy.FlinkShimsProxy; -import java.util.function.Function; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; -public final class FlinkClient extends LoggerSupport { +public final class FlinkClient { private static final String FLINK_CLIENT_ENTRYPOINT_CLASS = "org.apache.streampark.flink.client.FlinkClientEntrypoint"; @@ -60,57 +61,85 @@ public static SubmitResponse submit(SubmitRequest submitRequest) { SecurityManager securityManager = System.getSecurityManager(); try { System.setSecurityManager(new ExitSecurityManager()); - return proxy(submitRequest, submitRequest.flinkVersion(), SUBMIT_REQUEST, "submit"); + return invokeClient( + submitRequest, + submitRequest.flinkVersion(), + SUBMIT_REQUEST, + "submit", + SubmitResponse.class); } finally { System.setSecurityManager(securityManager); } } public static CancelResponse cancel(CancelRequest stopRequest) { - return proxy(stopRequest, stopRequest.flinkVersion(), CANCEL_REQUEST, "cancel"); + return invokeClient( + stopRequest, stopRequest.flinkVersion(), CANCEL_REQUEST, "cancel", CancelResponse.class); } public static DeployResponse deploy(DeployRequest deployRequest) { - return proxy(deployRequest, deployRequest.flinkVersion(), DEPLOY_REQUEST, "deploy"); + return invokeClient( + deployRequest, deployRequest.flinkVersion(), DEPLOY_REQUEST, "deploy", DeployResponse.class); } public static ShutDownResponse shutdown(ShutDownRequest shutDownRequest) { - return proxy(shutDownRequest, shutDownRequest.flinkVersion(), SHUTDOWN_REQUEST, "shutdown"); + return invokeClient( + shutDownRequest, + shutDownRequest.flinkVersion(), + SHUTDOWN_REQUEST, + "shutdown", + ShutDownResponse.class); } public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) { - return proxy( - savepointRequest, savepointRequest.flinkVersion(), SAVEPOINT_REQUEST, "triggerSavepoint"); + return invokeClient( + savepointRequest, + savepointRequest.flinkVersion(), + SAVEPOINT_REQUEST, + "triggerSavepoint", + SavepointResponse.class); } - @SuppressWarnings("unchecked") - private static T proxy( - Object request, - FlinkVersion flinkVersion, - String requestClassName, - String methodName) { + private static R invokeClient( + Object request, + FlinkVersion flinkVersion, + String requestClassName, + String methodName, + Class responseType) { flinkVersion.checkVersion(); return FlinkShimsProxy.proxy( flinkVersion, - (Function) classLoader -> { + classLoader -> { try { - Class submitClass = classLoader.loadClass(FLINK_CLIENT_ENTRYPOINT_CLASS); + Class entrypointClass = classLoader.loadClass(FLINK_CLIENT_ENTRYPOINT_CLASS); Class requestClass = classLoader.loadClass(requestClassName); - java.lang.reflect.Method method = - submitClass.getDeclaredMethod(methodName, requestClass); + Method method = entrypointClass.getDeclaredMethod(methodName, requestClass); method.setAccessible(true); - Object obj = - method.invoke( - null, FlinkShimsProxy.getObject(classLoader, request)); - if (obj == null) { + Object shimsRequest = + FlinkShimsProxy.getObject(classLoader, request, requestClass); + Object result = method.invoke(null, shimsRequest); + if (result == null) { return null; } - return FlinkShimsProxy.getObject(FlinkClient.class.getClassLoader(), obj); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); + return FlinkShimsProxy.getObject( + FlinkClient.class.getClassLoader(), result, responseType); + } catch (InvocationTargetException e) { + throw unwrapInvocationTarget(e); + } catch (ReflectiveOperationException | IOException e) { + throw new IllegalStateException( + "Failed to invoke Flink client via shims proxy: " + methodName, e); } }); } + + private static RuntimeException unwrapInvocationTarget(InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + return (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + return new IllegalStateException("Failed to invoke Flink client method", e); + } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java index 625a917daf..e8ad5eb252 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java @@ -26,7 +26,9 @@ import javax.annotation.Nullable; import java.io.File; +import java.io.IOException; import java.io.Serializable; +import java.io.UncheckedIOException; import java.util.HashMap; import java.util.Map; @@ -45,8 +47,9 @@ static HdfsWorkspace createHdfsWorkspace(FlinkVersion flinkVersion) { FileUtils.isSymlink(flinkHomeDir) ? flinkHomeDir.getCanonicalFile().getName() : flinkHomeDir.getName(); - } catch (Exception e) { - throw new RuntimeException(e); + } catch (IOException e) { + throw new UncheckedIOException( + "Failed to resolve Flink home directory name: " + flinkHome, e); } String flinkHdfsHome = workspace.APP_FLINK() + "/" + flinkName; return new HdfsWorkspace( diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java index 3bca741e91..350e9205c4 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java @@ -44,6 +44,7 @@ import java.io.File; import java.io.Serializable; +import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Collections; @@ -225,8 +226,8 @@ public List libs() { for (File file : files) { try { urls.add(file.toURI().toURL()); - } catch (Exception e) { - throw new RuntimeException(e); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Invalid lib file URL: " + file, e); } } libs = urls; @@ -242,7 +243,7 @@ public List classPaths() { combined.addAll(libs()); classPaths = combined; } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to build classpath", e); } } return classPaths; @@ -250,7 +251,11 @@ public List classPaths() { public String flinkSQL() { if (flinkSQL == null) { - flinkSQL = extraParameter.get(ConfigKeys.KEY_FLINK_SQL()).toString(); + if (extraParameter == null) { + return null; + } + Object sql = extraParameter.get(ConfigKeys.KEY_FLINK_SQL()); + flinkSQL = sql != null ? sql.toString() : null; } return flinkSQL; } @@ -336,7 +341,7 @@ public boolean hasExtra(String key) { } public Object getExtra(String key) { - return extraParameter.get(key); + return extraParameter != null ? extraParameter.get(key) : null; } public HdfsWorkspace hdfsWorkspace() { @@ -419,14 +424,15 @@ private Map parseJsonAppConf() { .filter(e -> e.getValue() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalArgumentException("Failed to parse JSON app config", e); } } private Map parseHdfsAppConf() { try { String text = HdfsUtils.read(appConf()); - String extension = appConf().split("\\.")[appConf().split("\\.").length - 1].toLowerCase(); + String[] parts = appConf().split("\\."); + String extension = parts[parts.length - 1].toLowerCase(); switch (extension) { case "yml": case "yaml": @@ -439,10 +445,8 @@ private Map parseHdfsAppConf() { throw new IllegalArgumentException( "[StreamPark] Usage: application config format error,must be [yaml|conf|properties]"); } - } catch (RuntimeException e) { - throw e; } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to parse HDFS app config: " + appConf(), e); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/bean/ClientBeanUtilsTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/bean/ClientBeanUtilsTest.java new file mode 100644 index 0000000000..af4b5c1fd5 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/bean/ClientBeanUtilsTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.bean; + +import org.junit.jupiter.api.Test; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class ClientBeanUtilsTest { + + @Test + void toSerializableMapShouldConvertNonSerializableValues() { + Map input = new HashMap<>(); + input.put("serializable", "value"); + input.put("custom", new Object()); + + Map result = ClientBeanUtils.toSerializableMap(input); + + assertThat(result).containsEntry("serializable", "value"); + assertThat(result.get("custom")).isNotNull(); + } + + @Test + void copyPropertiesMapShouldReturnNullForNullInput() { + assertThat(ClientBeanUtils.copyPropertiesMap(null)).isNull(); + } + + @Test + void toSerializableMapShouldReturnNullForNullInput() { + assertThat(ClientBeanUtils.toSerializableMap(null)).isNull(); + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml index 6d5edbb876..1b50fc9e00 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml @@ -101,6 +101,12 @@ junit-jupiter-engine test + + + org.assertj + assertj-core + test + diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java index 36d12ee6bc..552a4e683f 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java @@ -77,7 +77,7 @@ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { providedLibs.add(jobLib); } } catch (Exception e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to check Flink SQL job lib path: " + jobLib, e); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java index 0d88a992e8..23744254be 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java @@ -185,7 +185,7 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { return new DeployResponse( null, null, - new RuntimeException("get the cluster getWebInterfaceURL failed.")); + new IllegalStateException("Failed to get YARN session cluster WebInterfaceURL")); } catch (Exception e) { logError("start flink session fail in " + deployRequest.deployMode() + " mode"); throw e; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ExitSecurityManagerTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ExitSecurityManagerTest.java new file mode 100644 index 0000000000..b55d5ca1ba --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ExitSecurityManagerTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.test; + +import org.apache.streampark.flink.client.ExitSecurityManager; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ExitSecurityManagerTest { + + @Test + void checkExitShouldThrowSecurityException() { + ExitSecurityManager securityManager = new ExitSecurityManager(); + assertThatThrownBy(() -> securityManager.checkExit(0)) + .isInstanceOf(SecurityException.class) + .hasMessageContaining("System.exit(0)"); + } + + @Test + void checkPermissionShouldAllowAllPermissions() { + ExitSecurityManager securityManager = new ExitSecurityManager(); + assertThatCode(() -> securityManager.checkPermission(null)).doesNotThrowAnyException(); + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestTest.java new file mode 100644 index 0000000000..441b213f04 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.test; + +import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.enums.ApplicationType; +import org.apache.streampark.common.enums.FlinkDeployMode; +import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.flink.client.bean.SubmitApplicationSpec; +import org.apache.streampark.flink.client.bean.SubmitRequest; + +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class SubmitRequestTest { + + private static final FlinkVersion FLINK_VERSION = new FlinkVersion("/tmp/flink-home"); + + @Test + void appMainForFlinkSqlJob() { + SubmitRequest request = createRequest(FlinkJobType.FLINK_SQL, null); + assertThat(request.appMain()).isEqualTo(Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS); + } + + @Test + void appMainForPyFlinkJob() { + SubmitRequest request = createRequest(FlinkJobType.PYFLINK, null); + assertThat(request.appMain()).isEqualTo(Constants.PYTHON_FLINK_DRIVER_CLASS_NAME); + } + + @Test + void savepointRestoreSettingsWithoutSavepoint() { + SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); + assertThat(request.savepointRestoreSettings()).isEqualTo(SavepointRestoreSettings.none()); + } + + @Test + void appPropertiesFromJsonConf() { + String propertyKey = ConfigKeys.KEY_FLINK_PROPERTY_PREFIX() + "parallelism.default"; + String appConf = "json://{\"" + propertyKey + "\":\"8\"}"; + SubmitApplicationSpec application = + SubmitApplicationSpec.builder() + .jobType(FlinkJobType.FLINK_JAR) + .appConf(appConf) + .build(); + SubmitRequest request = + new SubmitRequest( + FLINK_VERSION, + FlinkDeployMode.YARN_APPLICATION, + Collections.emptyMap(), + application, + null, + null, + null); + + assertThat(request.appProperties()).containsEntry("parallelism.default", "8"); + } + + @Test + void propertiesMapCopiesSerializableIntegerValues() { + Map properties = new HashMap<>(); + properties.put("key", 42); + SubmitApplicationSpec application = + SubmitApplicationSpec.builder().jobType(FlinkJobType.FLINK_JAR).build(); + SubmitRequest request = + new SubmitRequest( + FLINK_VERSION, + FlinkDeployMode.YARN_APPLICATION, + properties, + application, + null, + null, + null); + + assertThat(request.getProp("key")).isEqualTo(42); + } + + @Test + void flinkSqlShouldReturnNullWhenExtraParameterMissing() { + SubmitRequest request = createRequest(FlinkJobType.FLINK_SQL, null); + assertThat(request.flinkSQL()).isNull(); + } + + @Test + void flinkSqlShouldReadFromExtraParameter() { + Map extra = new HashMap<>(); + extra.put(ConfigKeys.KEY_FLINK_SQL(), "select 1"); + SubmitRequest request = + new SubmitRequest( + FLINK_VERSION, + FlinkDeployMode.YARN_APPLICATION, + Collections.emptyMap(), + SubmitApplicationSpec.builder().jobType(FlinkJobType.FLINK_SQL).build(), + null, + null, + extra); + assertThat(request.flinkSQL()).isEqualTo("select 1"); + } + + @Test + void getExtraShouldBeNullSafe() { + SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); + assertThat(request.getExtra("missing")).isNull(); + assertThat(request.hasExtra("missing")).isFalse(); + } + + @Test + void allowNonRestoredStateShouldDefaultToFalse() { + SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); + assertThat(request.allowNonRestoredState()).isFalse(); + } + + private static SubmitRequest createRequest(FlinkJobType jobType, String appConf) { + SubmitApplicationSpec application = + SubmitApplicationSpec.builder() + .jobType(jobType) + .applicationType(ApplicationType.APACHE_FLINK) + .appConf(appConf) + .build(); + return new SubmitRequest( + FLINK_VERSION, + FlinkDeployMode.YARN_APPLICATION, + Collections.emptyMap(), + application, + null, + null, + null); + } +} diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateParser.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateParser.java index d1f36ae867..2cdaba57b0 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateParser.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateParser.java @@ -23,6 +23,7 @@ import org.yaml.snakeyaml.Yaml; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -54,13 +55,11 @@ public static String getInitPodTemplateContent() { * @param podTemplateContent original pod template * @return complemented pod template */ - @SuppressWarnings("unchecked") public static String completeInitPodTemplate(String podTemplateContent) { if (podTemplateContent == null || podTemplateContent.trim().isEmpty()) { return POD_TEMPLATE_INIT_CONTENT; } - Yaml yaml = new Yaml(); - Map root = yaml.load(podTemplateContent); + Map root = loadYamlMap(podTemplateContent); Map res = new LinkedHashMap<>(); res.put("apiVersion", root.getOrDefault("apiVersion", "v1")); @@ -68,13 +67,11 @@ public static String completeInitPodTemplate(String podTemplateContent) { Object metadata = root.getOrDefault("metadata", defaultMetadata()); res.put("metadata", metadata); - if (root.containsKey("spec")) { - Object spec = root.get("spec"); - if (spec instanceof Map && !((Map) spec).isEmpty()) { - res.put("spec", spec); - } + Map spec = getStringObjectMap(root.get(KEY_SPEC)); + if (!spec.isEmpty()) { + res.put(KEY_SPEC, spec); } - return yaml.dumpAsMap(res); + return new Yaml().dumpAsMap(res); } private static Map defaultMetadata() { @@ -91,7 +88,6 @@ private static Map defaultMetadata() { * @param podTemplateContent pod template content * @return pod template content */ - @SuppressWarnings("unchecked") public static String completeHostAliasSpec(Map hosts, String podTemplateContent) { if (hosts.isEmpty()) { return podTemplateContent; @@ -99,17 +95,17 @@ public static String completeHostAliasSpec(Map hosts, String pod try { String content = completeInitPodTemplate(podTemplateContent); List> hostAlias = covertHostsMapToHostAliasNode(hosts); - Yaml yaml = new Yaml(); - Map root = yaml.load(content); - if (!root.containsKey("spec")) { + Map root = loadYamlMap(content); + if (!root.containsKey(KEY_SPEC)) { Map spec = new LinkedHashMap<>(); spec.put(KEY_HOST_ALIASES, hostAlias); root.put(KEY_SPEC, spec); - return yaml.dumpAsMap(root); + return new Yaml().dumpAsMap(root); } - Map spec = (Map) root.get(KEY_SPEC); + Map spec = getStringObjectMap(root.get(KEY_SPEC)); spec.put(KEY_HOST_ALIASES, hostAlias); - return yaml.dumpAsMap(root); + root.put(KEY_SPEC, spec); + return new Yaml().dumpAsMap(root); } catch (Throwable e) { return podTemplateContent; } @@ -140,23 +136,21 @@ private static List> covertHostsMapToHostAliasNode(Map ipv4 */ - @SuppressWarnings("unchecked") public static Map extractHostAliasMap(String podTemplateContent) { Map hosts = new LinkedHashMap<>(0); if (podTemplateContent == null || podTemplateContent.isEmpty()) { return hosts; } try { - Yaml yaml = new Yaml(); - Map root = yaml.load(podTemplateContent); + Map root = loadYamlMap(podTemplateContent); if (!root.containsKey(KEY_SPEC)) { return hosts; } - Map spec = (Map) root.get(KEY_SPEC); + Map spec = getStringObjectMap(root.get(KEY_SPEC)); if (!spec.containsKey(KEY_HOST_ALIASES)) { return hosts; } - List> hostAliases = (List>) spec.get(KEY_HOST_ALIASES); + List> hostAliases = getMapList(spec.get(KEY_HOST_ALIASES)); if (CollectionUtils.isEmpty(hostAliases)) { return hosts; } @@ -169,20 +163,16 @@ public static Map extractHostAliasMap(String podTemplateContent) return hosts; } - @SuppressWarnings("unchecked") private static void collectHostAliasEntry(Map hosts, Map hostAlias) { if (!hostAlias.containsKey(KEY_IP) || !hostAlias.containsKey(KEY_HOSTNAMES)) { return; } - String ip = (String) hostAlias.get(KEY_IP); - if (StringUtils.isBlank(ip)) { - return; - } - List hostnames = (List) hostAlias.get(KEY_HOSTNAMES); - if (hostnames == null) { + Object ipValue = hostAlias.get(KEY_IP); + if (!(ipValue instanceof String) || StringUtils.isBlank((String) ipValue)) { return; } - for (String hostname : hostnames) { + String ip = (String) ipValue; + for (String hostname : getStringList(hostAlias.get(KEY_HOSTNAMES))) { if (StringUtils.isNotBlank(hostname)) { hosts.put(hostname, ip); } @@ -201,4 +191,57 @@ public static String previewHostAliasSpec(Map hosts) { root.put(KEY_HOST_ALIASES, hostAlias); return new Yaml().dumpAsMap(root); } + + private static Map loadYamlMap(String content) { + Object loaded = new Yaml().load(content); + if (!(loaded instanceof Map)) { + return new LinkedHashMap<>(); + } + return toStringObjectMap((Map) loaded); + } + + private static Map getStringObjectMap(Object value) { + if (!(value instanceof Map)) { + return new LinkedHashMap<>(); + } + return toStringObjectMap((Map) value); + } + + private static Map toStringObjectMap(Map source) { + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : source.entrySet()) { + if (entry.getKey() instanceof String) { + result.put((String) entry.getKey(), entry.getValue()); + } + } + return result; + } + + private static List getStringList(Object value) { + if (!(value instanceof List)) { + return Collections.emptyList(); + } + List raw = (List) value; + List result = new ArrayList<>(raw.size()); + for (Object item : raw) { + if (item instanceof String) { + result.add((String) item); + } + } + return result; + } + + private static List> getMapList(Object value) { + if (!(value instanceof List)) { + return Collections.emptyList(); + } + List raw = (List) value; + List> result = new ArrayList<>(raw.size()); + for (Object item : raw) { + if (item instanceof Map) { + result.add(toStringObjectMap((Map) item)); + } + } + return result; + } } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkJobState.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkJobState.java index a6a9651240..3df0a101af 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkJobState.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkJobState.java @@ -24,34 +24,40 @@ import java.util.Set; /** flink job status on kubernetes */ -@SuppressWarnings("AlibabaEnumConstantsMustHaveComment") public enum FlinkJobState { - // flink job has been submit by the streampark. + /** Flink job has been submitted by StreamPark. */ STARTING, - // flink k8s resources are being initialized. + /** Flink Kubernetes resources are being initialized. */ K8S_INITIALIZING, - // lost track of flink job temporarily. + /** Lost track of Flink job temporarily. */ SILENT, - // flink job has terminated positively (maybe FINISHED or CANCELED) + /** Flink job terminated positively (FINISHED or CANCELED). */ POS_TERMINATED, - // flink job has terminated (maybe FINISHED, CANCELED or FAILED) + /** Flink job terminated (FINISHED, CANCELED, or FAILED). */ TERMINATED, - // lost track of flink job completely. + /** Lost track of Flink job completely. */ LOST, - // other flink state + /** Other Flink job state. */ OTHER, - // the following enum have the same meaning as the native flink state enum. - // @see org.apache.flink.api.common.JobStatus + /** Native Flink state: initializing. */ INITIALIZING, + /** Native Flink state: created. */ CREATED, + /** Native Flink state: running. */ RUNNING, + /** Native Flink state: failing. */ FAILING, + /** Native Flink state: failed. */ FAILED, + /** Native Flink state: cancelling. */ CANCELLING, + /** Native Flink state: canceled. */ CANCELED, + /** Native Flink state: finished. */ FINISHED, + /** Native Flink state: restarting. */ RESTARTING; private static final Set ENDING_STATES = diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesDeploymentHelper.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesDeploymentHelper.java index 0c0453e207..5305c10128 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesDeploymentHelper.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesDeploymentHelper.java @@ -134,7 +134,7 @@ public static String watchDeploymentLog(String nameSpace, String jobName, String Files.asCharSink(file, Charsets.UTF_8).write(log); return path; } catch (IOException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to write deployment log for job " + jobId, e); } }); } @@ -161,7 +161,7 @@ public static String watchPodTerminatedLog(String nameSpace, String jobName, Str } }, error -> { - throw new RuntimeException(error); + throw new IllegalStateException("Failed to read terminated pod log for job " + jobId, error); }); } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1.java index 006f2cc01b..0478f0934c 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1.java @@ -39,7 +39,7 @@ public String getIngressUrl(String nameSpace, String clusterId, ClusterClient } return clusterClient.getWebInterfaceURL(); } catch (Exception e) { - throw new RuntimeException("[StreamPark] get ingressUrlAddress error: " + e, e); + throw new IllegalStateException("[StreamPark] get ingressUrlAddress error: " + e, e); } } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1beta1.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1beta1.java index 8cd8cedb02..37d7cc8024 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1beta1.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategyV1beta1.java @@ -51,7 +51,7 @@ private String resolveIngressUrl( } return clusterClient.getWebInterfaceURL(); } catch (Exception e) { - throw new RuntimeException("[StreamPark] get ingressUrlAddress error: " + e, e); + throw new IllegalStateException("[StreamPark] get ingressUrlAddress error: " + e, e); } } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java index 129063b7c9..b4ab58980c 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java @@ -128,11 +128,7 @@ public static TrackId onApplication( } private static boolean isNotEmpty(String value) { - try { - return value != null && !value.isEmpty(); - } catch (Exception e) { - return false; - } + return value != null && !value.isEmpty(); } @Override diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/rest/FlinkHistoryArchives.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/rest/FlinkHistoryArchives.java index 4a6038ec48..5d6b7f23fc 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/rest/FlinkHistoryArchives.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/rest/FlinkHistoryArchives.java @@ -79,8 +79,7 @@ public static String getJobStateFromArchiveFile(TrackId trackId) { private static void writeExceptionLog(String json, String jobId) { try { - @SuppressWarnings("unchecked") - Map root = JsonUtils.read(json, Map.class); + Map root = JsonUtils.read(json, Map.class); Object log = root.get("root-exception"); if (log != null) { String path = KubernetesDeploymentHelper.getJobErrorLog(jobId); diff --git a/streampark-flink/streampark-flink-kubernetes/src/test/java/org/apache/streampark/flink/kubernetes/model/TrackIdTest.java b/streampark-flink/streampark-flink-kubernetes/src/test/java/org/apache/streampark/flink/kubernetes/model/TrackIdTest.java new file mode 100644 index 0000000000..f5d4fb8119 --- /dev/null +++ b/streampark-flink/streampark-flink-kubernetes/src/test/java/org/apache/streampark/flink/kubernetes/model/TrackIdTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.kubernetes.model; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TrackIdTest { + + @Test + void applicationTrackShouldBeLegalWithNamespaceAndClusterId() { + TrackId trackId = + TrackId.onApplication("ns1", "cluster-a", 1L, null, null, null); + assertThat(trackId.isLegal()).isTrue(); + assertThat(trackId.isActive()).isFalse(); + } + + @Test + void sessionTrackShouldRequireJobIdToBeActive() { + TrackId trackId = TrackId.onSession("ns1", "cluster-a", 1L, "job-1", null, null); + assertThat(trackId.isLegal()).isTrue(); + assertThat(trackId.isActive()).isTrue(); + } + + @Test + void incompleteSessionTrackShouldBeIllegal() { + TrackId trackId = TrackId.onSession("ns1", "cluster-a", 1L, null, null, null); + assertThat(trackId.isLegal()).isFalse(); + } + + @Test + void equalsShouldCompareIdentityFields() { + TrackId left = TrackId.onApplication("ns1", "cluster-a", 1L, "job-1", "g1", null); + TrackId right = TrackId.onApplication("ns1", "cluster-a", 1L, "job-1", "g1", null); + assertThat(left).isEqualTo(right); + assertThat(left.hashCode()).isEqualTo(right.hashCode()); + } +} diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerImageExist.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerImageExist.java index fb3f0deddd..637362a28d 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerImageExist.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerImageExist.java @@ -33,7 +33,7 @@ public boolean doesDockerImageExist(String imageName) { } }, err -> { - throw new RuntimeException( + throw new IllegalStateException( "Check docker image failed, imageName=" + imageName, err); }); } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipeline.java index f0bb8a4f71..781f1e0eab 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipeline.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipeline.java @@ -17,11 +17,19 @@ package org.apache.streampark.flink.packer.pipeline; +import org.apache.streampark.common.fs.FsOperator; +import org.apache.streampark.common.fs.HdfsOperator; +import org.apache.streampark.common.fs.LfsOperator; import org.apache.streampark.common.util.LoggerSupport; import org.apache.streampark.common.util.ThreadUtils; +import org.apache.streampark.flink.packer.maven.DependencyInfo; +import org.apache.streampark.flink.packer.maven.MavenTool; +import java.io.File; import java.util.AbstractMap; +import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -29,6 +37,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; /** Building pipeline abstract class. */ public abstract class BuildPipeline extends LoggerSupport @@ -46,7 +55,7 @@ public abstract class BuildPipeline extends LoggerSupport ThreadUtils.threadFactory("streampark-pipeline-watcher-executor"), new ThreadPoolExecutor.AbortPolicy()); - protected PipelineStatusEnum pipeStatus = PipelineStatusEnum.pending; + protected PipelineStatusEnum pipeStatus = PipelineStatusEnum.PENDING; protected PipeError error = PipeError.empty(); @@ -64,7 +73,7 @@ protected BuildPipeline() { (seq, desc) -> stepsStatus.put( seq, new AbstractMap.SimpleEntry<>( - PipelineStepStatusEnum.waiting, System.currentTimeMillis()))); + PipelineStepStatusEnum.WAITING, System.currentTimeMillis()))); } /** use to identify the log record that belongs to which pipeline instance */ @@ -107,7 +116,7 @@ protected java.util.Optional execStep(int seq, Callable process) { stepsStatus.put( seq, new AbstractMap.SimpleEntry<>( - PipelineStepStatusEnum.running, System.currentTimeMillis())); + PipelineStepStatusEnum.RUNNING, System.currentTimeMillis())); logInfo( "Building pipeline step[" + seq @@ -120,7 +129,7 @@ protected java.util.Optional execStep(int seq, Callable process) { stepsStatus.put( seq, new AbstractMap.SimpleEntry<>( - PipelineStepStatusEnum.success, System.currentTimeMillis())); + PipelineStepStatusEnum.SUCCESS, System.currentTimeMillis())); logInfo("Building pipeline step[" + seq + "/" + allSteps() + "] success"); notifyStepChange(); return java.util.Optional.of(result); @@ -128,8 +137,8 @@ protected java.util.Optional execStep(int seq, Callable process) { stepsStatus.put( seq, new AbstractMap.SimpleEntry<>( - PipelineStepStatusEnum.failure, System.currentTimeMillis())); - pipeStatus = PipelineStatusEnum.failure; + PipelineStepStatusEnum.FAILURE, System.currentTimeMillis())); + pipeStatus = PipelineStatusEnum.FAILURE; error = PipeError.of(cause.getMessage(), cause); logInfo( "Building pipeline step[" @@ -148,7 +157,7 @@ protected void skipStep(int step) { stepsStatus.put( step, new AbstractMap.SimpleEntry<>( - PipelineStepStatusEnum.skipped, System.currentTimeMillis())); + PipelineStepStatusEnum.SKIPPED, System.currentTimeMillis())); logInfo( "Building pipeline step[" + step @@ -162,26 +171,26 @@ protected void skipStep(int step) { /** Launch the building pipeline. */ @Override public BuildResult launch() { - pipeStatus = PipelineStatusEnum.running; + pipeStatus = PipelineStatusEnum.RUNNING; try { notifyStart(); logInfo("Building pipeline is launching, params=" + offerBuildParam()); BuildResult result = EXEC_POOL.submit(this::buildProcess).get(20, TimeUnit.MINUTES); - pipeStatus = PipelineStatusEnum.success; + pipeStatus = PipelineStatusEnum.SUCCESS; logInfo("Building pipeline has finished successfully."); notifyFinish(result); return result; } catch (InterruptedException e) { Thread.currentThread().interrupt(); - pipeStatus = PipelineStatusEnum.failure; + pipeStatus = PipelineStatusEnum.FAILURE; error = PipeError.of(e.getMessage(), e); logError("Building pipeline has failed.", e); BuildResult result = new ErrorResult(); notifyFinish(result); return result; } catch (ExecutionException e) { - pipeStatus = PipelineStatusEnum.failure; + pipeStatus = PipelineStatusEnum.FAILURE; Throwable cause = e.getCause(); if (cause == null) { cause = e; @@ -192,7 +201,7 @@ public BuildResult launch() { notifyFinish(result); return result; } catch (TimeoutException e) { - pipeStatus = PipelineStatusEnum.failure; + pipeStatus = PipelineStatusEnum.FAILURE; error = PipeError.of(e.getMessage(), e); logError("Building pipeline has failed.", e); BuildResult result = new ErrorResult(); @@ -241,15 +250,59 @@ protected void logError(String msg, Throwable throwable) { super.logError("[streampark-packer] " + msg + " | " + logSuffix(), throwable); } - protected RuntimeException pipelineException() { + protected IllegalStateException pipelineException() { Throwable ex = getError().exception(); - if (ex instanceof RuntimeException) { - return (RuntimeException) ex; + if (ex instanceof IllegalStateException) { + return (IllegalStateException) ex; } if (ex != null) { - return new RuntimeException(ex); + return new IllegalStateException(ex.getMessage(), ex); } - return new RuntimeException(getError().summary()); + return new IllegalStateException(getError().summary()); + } + + protected void runYarnSqlBuildSteps( + String localWorkspace, + String yarnProvidedPath, + boolean sqlMode, + DependencyInfo dependencyInfo) { + execStep( + 1, + () -> { + if (sqlMode) { + LfsOperator.mkCleanDirs(localWorkspace); + HdfsOperator.mkCleanDirs(yarnProvidedPath); + } + logInfo("Recreate building workspace: " + yarnProvidedPath); + return null; + }) + .orElseThrow(this::pipelineException); + + List mavenJars = + execStep( + 2, + () -> { + if (!sqlMode) { + return Collections.emptyList(); + } + List mavenArts = MavenTool.resolveArtifacts(dependencyInfo.mavenArts()); + List paths = + mavenArts.stream().map(File::getAbsolutePath).collect(Collectors.toList()); + paths.addAll(dependencyInfo.extJarLibs()); + return paths; + }) + .orElseThrow(this::pipelineException); + + execStep( + 3, + () -> { + for (String jar : mavenJars) { + YarnJarUploader.uploadJarToHdfsOrLfs(FsOperator.lfs(), jar, localWorkspace); + YarnJarUploader.uploadJarToHdfsOrLfs(FsOperator.hdfs(), jar, yarnProvidedPath); + } + return null; + }) + .orElseThrow(this::pipelineException); } /** intercept snapshot */ diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineExpose.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineExpose.java index 555c2c59a5..e717a05ab4 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineExpose.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildPipelineExpose.java @@ -40,8 +40,7 @@ public interface BuildPipelineExpose { /** launch the pipeline instance */ BuildResult launch(); - @SuppressWarnings("unchecked") default T as(Class clz) { - return (T) this; + return clz.cast(this); } } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java index f6668dc067..cf72af2d3a 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/BuildResult.java @@ -25,8 +25,7 @@ public interface BuildResult extends Serializable { /** is pass aka is successfully */ boolean pass(); - @SuppressWarnings("unchecked") default T as(Class clz) { - return (T) this; + return clz.cast(this); } } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sDockerBuildSupport.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sDockerBuildSupport.java new file mode 100644 index 0000000000..7c44e99ab5 --- /dev/null +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/K8sDockerBuildSupport.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.packer.pipeline; + +import org.apache.streampark.common.util.ThreadUtils; +import org.apache.streampark.flink.packer.docker.DockerConf; +import org.apache.streampark.flink.packer.docker.DockerUtils; + +import org.apache.commons.lang3.StringUtils; + +import com.github.dockerjava.api.command.PushImageCmd; +import com.github.dockerjava.core.command.HackBuildImageCmd; +import com.github.dockerjava.core.command.HackPullImageCmd; +import com.github.dockerjava.core.command.HackPushImageCmd; +import com.google.common.collect.Sets; + +import java.io.File; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** Shared Docker pull/build/push steps for Kubernetes application build pipelines. */ +public final class K8sDockerBuildSupport { + + static final ExecutorService DOCKER_EXECUTOR = + new ThreadPoolExecutor( + Runtime.getRuntime().availableProcessors() * 5, + Runtime.getRuntime().availableProcessors() * 10, + 60L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(2048), + ThreadUtils.threadFactory("streampark-docker-progress-watcher-executor"), + new ThreadPoolExecutor.DiscardOldestPolicy()); + + private K8sDockerBuildSupport() { + } + + public static String compileTag(String tag, String registerAddress, String imageNamespace) { + String tagName = tag.contains("/") ? tag : imageNamespace + "/" + tag; + if (StringUtils.isNotBlank(registerAddress) && !tagName.startsWith(registerAddress)) { + tagName = registerAddress + "/" + tagName; + } + return tagName.toLowerCase(); + } + + public static void pullImage( + DockerConf dockerConf, + String baseImageTag, + DockerResolveProgress dockerProcess, + DockerProgressWatcher watcher, + Consumer logInfo, + boolean checkLocalImageFirst) { + DockerUtils.usingDockerClient( + dockerClient -> { + if (checkLocalImageFirst) { + boolean imgExists = + dockerClient.listImagesCmd().exec().stream() + .anyMatch( + image -> image.getRepoTags() != null + && java.util.Arrays.stream(image.getRepoTags()) + .anyMatch(tag -> tag.contains(baseImageTag))); + if (imgExists) { + logInfo.accept( + "found local docker image " + + baseImageTag + + ", no need to pull from remote."); + return null; + } + } + HackPullImageCmd pullImageCmd = + resolvePullCommand(dockerClient, dockerConf, baseImageTag); + awaitPull(pullImageCmd, dockerProcess, watcher); + logInfo.accept( + "Already pulled docker image from remote register, imageTag=" + baseImageTag); + return null; + }, + err -> { + throw new IllegalStateException( + "Pull docker image failed, imageTag=" + baseImageTag, err); + }); + } + + public static void buildImage( + String buildWorkspace, + File dockerfile, + String pushImageTag, + DockerResolveProgress dockerProcess, + DockerProgressWatcher watcher, + Consumer logInfo) { + DockerUtils.usingDockerClient( + dockerClient -> { + HackBuildImageCmd buildImageCmd = + (HackBuildImageCmd) dockerClient + .buildImageCmd() + .withBaseDirectory(new File(buildWorkspace)) + .withDockerfile(dockerfile) + .withTags(Sets.newHashSet(pushImageTag)); + String imageId = + buildImageCmd + .start( + DockerUtils.watchDockerBuildStep( + buildStep -> { + dockerProcess.getBuild().update(buildStep); + DOCKER_EXECUTOR.submit( + () -> watcher.onDockerBuildProgressChange( + dockerProcess.getBuild().snapshot())); + })) + .awaitImageId(); + logInfo.accept( + "Built docker image, imageId=" + imageId + ", imageTag=" + pushImageTag); + return null; + }, + err -> { + throw new IllegalStateException( + "Build docker image failed. tag=" + pushImageTag, err); + }); + } + + public static void pushImage( + DockerConf dockerConf, + String pushImageTag, + DockerResolveProgress dockerProcess, + DockerProgressWatcher watcher, + Consumer logInfo) { + DockerUtils.usingDockerClient( + dockerClient -> { + PushImageCmd pushCmd = + dockerClient.pushImageCmd(pushImageTag).withAuthConfig(dockerConf.toAuthConf()); + awaitPush((HackPushImageCmd) pushCmd, dockerProcess, watcher); + logInfo.accept("Already pushed docker image, imageTag=" + pushImageTag); + return null; + }, + err -> { + throw new IllegalStateException( + "Push docker image failed. tag=" + pushImageTag, err); + }); + } + + private static HackPullImageCmd resolvePullCommand( + com.github.dockerjava.api.DockerClient dockerClient, + DockerConf dockerConf, + String baseImageTag) { + if (dockerConf.registerAddress() != null + && !baseImageTag.startsWith(dockerConf.registerAddress())) { + return (HackPullImageCmd) dockerClient.pullImageCmd(baseImageTag); + } + return (HackPullImageCmd) dockerClient.pullImageCmd(baseImageTag) + .withAuthConfig(dockerConf.toAuthConf()); + } + + private static void awaitPull( + HackPullImageCmd pullImageCmd, + DockerResolveProgress dockerProcess, + DockerProgressWatcher watcher) { + try { + pullImageCmd + .start( + DockerUtils.watchDockerPullProcess( + pullRsp -> { + dockerProcess.getPull().update(pullRsp); + DOCKER_EXECUTOR.submit( + () -> watcher.onDockerPullProgressChange( + dockerProcess.getPull().snapshot())); + })) + .awaitCompletion(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + private static void awaitPush( + HackPushImageCmd pushCmd, + DockerResolveProgress dockerProcess, + DockerProgressWatcher watcher) { + try { + pushCmd + .start( + DockerUtils.watchDockerPushProcess( + pushRsp -> { + dockerProcess.getPush().update(pushRsp); + DOCKER_EXECUTOR.submit( + () -> watcher.onDockerPushProgressChange( + dockerProcess.getPush().snapshot())); + })) + .awaitCompletion(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } +} diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStatusEnum.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStatusEnum.java index 2702aa8fba..24f6421787 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStatusEnum.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStatusEnum.java @@ -33,10 +33,18 @@ * └───────────────────────────────────┘ * */ -@SuppressWarnings("AlibabaEnumConstantsMustHaveComment") public enum PipelineStatusEnum { - unknown(0), pending(1), running(2), success(3), failure(4); + /** Unknown pipeline status. */ + UNKNOWN(0), + /** Pipeline is pending execution. */ + PENDING(1), + /** Pipeline is running. */ + RUNNING(2), + /** Pipeline finished successfully. */ + SUCCESS(3), + /** Pipeline finished with failure. */ + FAILURE(4); private final Integer code; @@ -51,7 +59,7 @@ public static PipelineStatusEnum of(Integer code) { return status; } } - return unknown; + return UNKNOWN; } @JsonValue @@ -60,6 +68,6 @@ public Integer getCode() { } public boolean isUnknown() { - return this == unknown; + return this == UNKNOWN; } } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStepStatusEnum.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStepStatusEnum.java index 6447553d53..704a21b422 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStepStatusEnum.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/PipelineStepStatusEnum.java @@ -36,10 +36,20 @@ * └──────────────────────────────────┘ * */ -@SuppressWarnings("AlibabaEnumConstantsMustHaveComment") public enum PipelineStepStatusEnum { - unknown(0), waiting(1), running(2), success(3), failure(4), skipped(5); + /** Unknown step status. */ + UNKNOWN(0), + /** Step is waiting to run. */ + WAITING(1), + /** Step is running. */ + RUNNING(2), + /** Step finished successfully. */ + SUCCESS(3), + /** Step finished with failure. */ + FAILURE(4), + /** Step was skipped. */ + SKIPPED(5); private final Integer code; @@ -54,7 +64,7 @@ public static PipelineStepStatusEnum of(Integer code) { return status; } } - return unknown; + return UNKNOWN; } @JsonValue @@ -63,6 +73,6 @@ public Integer getCode() { } public boolean isUnknown() { - return this == unknown; + return this == UNKNOWN; } } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java new file mode 100644 index 0000000000..cb524119bf --- /dev/null +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.packer.pipeline; + +import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.fs.FsOperator; +import org.apache.streampark.common.util.AutoCloseUtils; + +import org.apache.commons.codec.digest.DigestUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +/** Uploads application jars to local or HDFS Yarn-provided paths. */ +public final class YarnJarUploader { + + private YarnJarUploader() { + } + + public static void uploadJarToHdfsOrLfs(FsOperator fsOperator, String origin, String target) throws IOException { + File originFile = new File(origin); + if (!fsOperator.exists(target)) { + fsOperator.mkdirs(target); + } + if (originFile.isFile()) { + uploadFile(fsOperator, originFile, target); + } else if (fsOperator == FsOperator.hdfs()) { + fsOperator.upload(originFile.getAbsolutePath(), target); + } + } + + private static void uploadFile(FsOperator fsOperator, File originFile, String target) throws IOException { + if (fsOperator == FsOperator.lfs()) { + fsOperator.copy(originFile.getAbsolutePath(), target); + return; + } + String uploadFile = Workspace.remote().APP_UPLOADS() + "/" + originFile.getName(); + if (fsOperator.exists(uploadFile)) { + AutoCloseUtils.using( + new FileInputStream(originFile), + inputStream -> { + try { + if (!DigestUtils.md5Hex(inputStream).equals(fsOperator.fileMd5(uploadFile))) { + fsOperator.upload(originFile.getAbsolutePath(), uploadFile); + } + } catch (IOException e) { + throw new IllegalStateException(e); + } + return null; + }); + } else { + fsOperator.upload(originFile.getAbsolutePath(), uploadFile); + } + fsOperator.copy(uploadFile, target); + } +} diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/AbstractK8sApplicationBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/AbstractK8sApplicationBuildPipeline.java new file mode 100644 index 0000000000..8ff6547921 --- /dev/null +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/AbstractK8sApplicationBuildPipeline.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.packer.pipeline.impl; + +import org.apache.streampark.flink.packer.docker.DockerConf; +import org.apache.streampark.flink.packer.pipeline.BuildPipeline; +import org.apache.streampark.flink.packer.pipeline.DockerBuildProgress; +import org.apache.streampark.flink.packer.pipeline.DockerProgressWatcher; +import org.apache.streampark.flink.packer.pipeline.DockerPullProgress; +import org.apache.streampark.flink.packer.pipeline.DockerPushProgress; +import org.apache.streampark.flink.packer.pipeline.DockerResolveProgress; +import org.apache.streampark.flink.packer.pipeline.K8sDockerBuildSupport; +import org.apache.streampark.flink.packer.pipeline.SilentDockerProgressWatcher; + +import java.io.File; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.Callable; + +/** Shared docker build steps for Kubernetes application build pipelines. */ +abstract class AbstractK8sApplicationBuildPipeline extends BuildPipeline { + + protected DockerProgressWatcher dockerProcessWatcher = new SilentDockerProgressWatcher(); + + protected final DockerResolveProgress dockerProcess = + new DockerResolveProgress( + DockerPullProgress.empty(), + DockerBuildProgress.empty(), + DockerPushProgress.empty()); + + public void registerDockerProgressWatcher(DockerProgressWatcher watcher) { + this.dockerProcessWatcher = watcher; + } + + protected void runDockerBuildSteps( + String buildWorkspace, + File dockerfile, + DockerConf dockerConf, + String baseImageTag, + String pushImageTag, + boolean checkLocalImageFirst) { + execStep( + 5, + () -> { + K8sDockerBuildSupport.pullImage( + dockerConf, + baseImageTag, + dockerProcess, + dockerProcessWatcher, + this::logInfo, + checkLocalImageFirst); + return null; + }) + .orElseThrow(this::pipelineException); + + execStep( + 6, + () -> { + K8sDockerBuildSupport.buildImage( + buildWorkspace, + dockerfile, + pushImageTag, + dockerProcess, + dockerProcessWatcher, + this::logInfo); + return null; + }) + .orElseThrow(this::pipelineException); + + execStep( + 7, + () -> { + K8sDockerBuildSupport.pushImage( + dockerConf, pushImageTag, dockerProcess, dockerProcessWatcher, this::logInfo); + return null; + }) + .orElseThrow(this::pipelineException); + } + + protected Map preparePodTemplateStep( + String buildWorkspace, + int stepId, + boolean empty, + Callable> exporter, + String engineLabel) { + if (empty) { + skipStep(stepId); + return Collections.emptyMap(); + } + return execStep( + stepId, + () -> { + Map podTemplateFiles = exporter.call(); + logInfo( + "Export " + + engineLabel + + " podTemplates: " + + String.join(",", podTemplateFiles.values())); + return podTemplateFiles; + }) + .orElseThrow(this::pipelineException); + } +} diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.java index 502446699a..cf6b0a0597 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkK8sApplicationBuildPipeline.java @@ -18,67 +18,29 @@ package org.apache.streampark.flink.packer.pipeline.impl; import org.apache.streampark.common.fs.LfsOperator; -import org.apache.streampark.common.util.ThreadUtils; import org.apache.streampark.flink.kubernetes.PodTemplateTool; import org.apache.streampark.flink.kubernetes.ingress.IngressController; -import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates; import org.apache.streampark.flink.packer.docker.DockerConf; -import org.apache.streampark.flink.packer.docker.DockerUtils; import org.apache.streampark.flink.packer.docker.FlinkDockerfileTemplate; import org.apache.streampark.flink.packer.docker.FlinkDockerfileTemplateTrait; import org.apache.streampark.flink.packer.docker.FlinkHadoopDockerfileTemplate; import org.apache.streampark.flink.packer.maven.MavenTool; -import org.apache.streampark.flink.packer.pipeline.BuildPipeline; -import org.apache.streampark.flink.packer.pipeline.DockerBuildProgress; import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse; -import org.apache.streampark.flink.packer.pipeline.DockerProgressWatcher; -import org.apache.streampark.flink.packer.pipeline.DockerPullProgress; -import org.apache.streampark.flink.packer.pipeline.DockerPushProgress; -import org.apache.streampark.flink.packer.pipeline.DockerResolveProgress; import org.apache.streampark.flink.packer.pipeline.FlinkK8sApplicationBuildRequest; +import org.apache.streampark.flink.packer.pipeline.K8sDockerBuildSupport; import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum; -import org.apache.streampark.flink.packer.pipeline.SilentDockerProgressWatcher; import org.apache.commons.lang3.StringUtils; -import com.github.dockerjava.api.command.PushImageCmd; -import com.github.dockerjava.core.command.HackBuildImageCmd; -import com.github.dockerjava.core.command.HackPullImageCmd; -import com.github.dockerjava.core.command.HackPushImageCmd; -import com.google.common.collect.Sets; - import java.io.File; -import java.util.Collections; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** Building pipeline for flink kubernetes-native application mode */ -public class FlinkK8sApplicationBuildPipeline extends BuildPipeline { - - private static final ExecutorService DOCKER_EXECUTOR = - new ThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 5, - Runtime.getRuntime().availableProcessors() * 10, - 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(2048), - ThreadUtils.threadFactory("streampark-docker-progress-watcher-executor"), - new ThreadPoolExecutor.DiscardOldestPolicy()); +public class FlinkK8sApplicationBuildPipeline extends AbstractK8sApplicationBuildPipeline { private final FlinkK8sApplicationBuildRequest request; - private DockerProgressWatcher dockerProcessWatcher = new SilentDockerProgressWatcher(); - - private final DockerResolveProgress dockerProcess = - new DockerResolveProgress( - DockerPullProgress.empty(), - DockerBuildProgress.empty(), - DockerPushProgress.empty()); - public FlinkK8sApplicationBuildPipeline(FlinkK8sApplicationBuildRequest request) { this.request = request; } @@ -93,10 +55,6 @@ public FlinkK8sApplicationBuildRequest offerBuildParam() { return request; } - public void registerDockerProgressWatcher(DockerProgressWatcher watcher) { - this.dockerProcessWatcher = watcher; - } - @Override public DockerImageBuildResponse buildProcess() { String buildWorkspace = @@ -113,32 +71,16 @@ public DockerImageBuildResponse buildProcess() { logInfo("Recreate building workspace: " + workspace); return workspace; }) - .orElseThrow(() -> { - throw pipelineException(); - }); + .orElseThrow(this::pipelineException); - Map podTemplatePaths; - K8sPodTemplates podTemplate = request.flinkPodTemplate(); - if (podTemplate.isEmpty()) { - skipStep(2); - podTemplatePaths = Collections.emptyMap(); - } else { - podTemplatePaths = - execStep( - 2, - () -> { - Map podTemplateFiles = - PodTemplateTool.preparePodTemplateFiles(buildWorkspace, podTemplate) - .tmplFiles(); - logInfo( - "Export flink podTemplates: " - + String.join(",", podTemplateFiles.values())); - return podTemplateFiles; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - } + Map podTemplatePaths = + preparePodTemplateStep( + buildWorkspace, + 2, + request.flinkPodTemplate().isEmpty(), + () -> PodTemplateTool.preparePodTemplateFiles(buildWorkspace, request.flinkPodTemplate()) + .tmplFiles(), + "flink"); final File shadedJar = execStep( @@ -153,14 +95,9 @@ public DockerImageBuildResponse buildProcess() { logInfo("Output shaded flink job jar: " + jar.getAbsolutePath()); return jar; }) - .orElseThrow( - () -> { - throw pipelineException(); - }); + .orElseThrow(this::pipelineException); final Set extJarLibs = request.dependencyInfo().extJarLibs(); - File dockerfile; - FlinkDockerfileTemplateTrait dockerFileTemplate; Object[] dockerResult = execStep( 4, @@ -189,159 +126,22 @@ public DockerImageBuildResponse buildProcess() { + template.offerDockerfileContent()); return new Object[]{dockerFile, template}; }) - .orElseThrow(() -> { - throw pipelineException(); - }); - dockerfile = (File) dockerResult[0]; - dockerFileTemplate = (FlinkDockerfileTemplateTrait) dockerResult[1]; + .orElseThrow(this::pipelineException); + File dockerfile = (File) dockerResult[0]; + FlinkDockerfileTemplateTrait dockerFileTemplate = (FlinkDockerfileTemplateTrait) dockerResult[1]; DockerConf dockerConf = request.dockerConfig(); String baseImageTag = request.flinkBaseImage().trim(); if (request.k8sNamespace().isEmpty() || request.clusterId().isEmpty()) { throw new IllegalArgumentException("k8sNamespace or clusterId cannot be empty"); } - String expectedImageTag = - "streampark-flinkjob-" + request.k8sNamespace() + "-" + request.clusterId(); String pushImageTag = - compileTag( - expectedImageTag, dockerConf.registerAddress(), dockerConf.imageNamespace()); - - execStep( - 5, - () -> { - DockerUtils.usingDockerClient( - dockerClient -> { - boolean imgExists = - dockerClient.listImagesCmd().exec().stream() - .anyMatch( - image -> image.getRepoTags() != null - && java.util.Arrays.stream(image.getRepoTags()) - .anyMatch(tag -> tag.contains(baseImageTag))); - if (imgExists) { - logInfo( - "found local docker image " - + baseImageTag - + ", no need to pull from remote."); - } else { - HackPullImageCmd pullImageCmd; - if (dockerConf.registerAddress() != null - && !baseImageTag.startsWith(dockerConf.registerAddress())) { - pullImageCmd = - (HackPullImageCmd) dockerClient.pullImageCmd(baseImageTag); - } else { - pullImageCmd = - (HackPullImageCmd) dockerClient - .pullImageCmd(baseImageTag) - .withAuthConfig(dockerConf.toAuthConf()); - } - try { - pullImageCmd - .start( - DockerUtils.watchDockerPullProcess( - pullRsp -> { - dockerProcess.getPull().update(pullRsp); - DOCKER_EXECUTOR.submit( - () -> dockerProcessWatcher - .onDockerPullProgressChange( - dockerProcess.getPull().snapshot())); - })) - .awaitCompletion(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - logInfo( - "Already pulled docker image from remote register, imageTag=" - + baseImageTag); - } - return null; - }, - err -> { - throw new RuntimeException( - "Pull docker image failed, imageTag=" + baseImageTag, err); - }); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - - execStep( - 6, - () -> { - DockerUtils.usingDockerClient( - dockerClient -> { - HackBuildImageCmd buildImageCmd = - (HackBuildImageCmd) dockerClient - .buildImageCmd() - .withBaseDirectory(new File(buildWorkspace)) - .withDockerfile(dockerfile) - .withTags(Sets.newHashSet(pushImageTag)); - String imageId = - buildImageCmd - .start( - DockerUtils.watchDockerBuildStep( - buildStep -> { - dockerProcess.getBuild().update(buildStep); - DOCKER_EXECUTOR.submit( - () -> dockerProcessWatcher - .onDockerBuildProgressChange( - dockerProcess.getBuild().snapshot())); - })) - .awaitImageId(); - logInfo( - "Built docker image, imageId=" - + imageId - + ", imageTag=" - + pushImageTag); - return null; - }, - err -> { - throw new RuntimeException( - "Build docker image failed. tag=" + pushImageTag, err); - }); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); + K8sDockerBuildSupport.compileTag( + "streampark-flinkjob-" + request.k8sNamespace() + "-" + request.clusterId(), + dockerConf.registerAddress(), + dockerConf.imageNamespace()); - execStep( - 7, - () -> { - DockerUtils.usingDockerClient( - dockerClient -> { - PushImageCmd pushCmd = - dockerClient - .pushImageCmd(pushImageTag) - .withAuthConfig(dockerConf.toAuthConf()); - try { - ((HackPushImageCmd) pushCmd) - .start( - DockerUtils.watchDockerPushProcess( - pushRsp -> { - dockerProcess.getPush().update(pushRsp); - DOCKER_EXECUTOR.submit( - () -> dockerProcessWatcher.onDockerPushProgressChange( - dockerProcess.getPush().snapshot())); - })) - .awaitCompletion(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - logInfo("Already pushed docker image, imageTag=" + pushImageTag); - return null; - }, - err -> { - throw new RuntimeException( - "Push docker image failed. tag=" + pushImageTag, err); - }); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); + runDockerBuildSteps(buildWorkspace, dockerfile, dockerConf, baseImageTag, pushImageTag, true); if (StringUtils.isBlank(request.ingressTemplate())) { skipStep(8); @@ -355,9 +155,7 @@ public DockerImageBuildResponse buildProcess() { logInfo("Export flink ingress: " + ingressOutputPath); return ingressOutputPath; }) - .orElseThrow(() -> { - throw pipelineException(); - }); + .orElseThrow(this::pipelineException); } return new DockerImageBuildResponse( @@ -367,14 +165,6 @@ public DockerImageBuildResponse buildProcess() { dockerFileTemplate.innerMainJarPath()); } - private String compileTag(String tag, String registerAddress, String imageNamespace) { - String tagName = tag.contains("/") ? tag : imageNamespace + "/" + tag; - if (StringUtils.isNotBlank(registerAddress) && !tagName.startsWith(registerAddress)) { - tagName = registerAddress + "/" + tagName; - } - return tagName.toLowerCase(); - } - public static FlinkK8sApplicationBuildPipeline of(FlinkK8sApplicationBuildRequest request) { return new FlinkK8sApplicationBuildPipeline(request); } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.java index 0bf759194c..249ca08fbf 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/FlinkYarnApplicationBuildPipeline.java @@ -17,27 +17,12 @@ package org.apache.streampark.flink.packer.pipeline.impl; -import org.apache.streampark.common.conf.Workspace; import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.common.fs.FsOperator; -import org.apache.streampark.common.fs.HdfsOperator; -import org.apache.streampark.common.fs.LfsOperator; -import org.apache.streampark.common.util.AutoCloseUtils; -import org.apache.streampark.flink.packer.maven.MavenTool; import org.apache.streampark.flink.packer.pipeline.BuildPipeline; import org.apache.streampark.flink.packer.pipeline.FlinkYarnApplicationBuildRequest; import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum; import org.apache.streampark.flink.packer.pipeline.SimpleBuildResponse; -import org.apache.commons.codec.digest.DigestUtils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - /** Building pipeline for flink yarn application mode */ public class FlinkYarnApplicationBuildPipeline extends BuildPipeline { @@ -59,92 +44,17 @@ public FlinkYarnApplicationBuildRequest offerBuildParam() { @Override public SimpleBuildResponse buildProcess() { - execStep( - 1, - () -> { - if (request.flinkJobType() == FlinkJobType.FLINK_SQL - || request.flinkJobType() == FlinkJobType.PYFLINK) { - LfsOperator.mkCleanDirs(request.localWorkspace()); - HdfsOperator.mkCleanDirs(request.yarnProvidedPath()); - } - logInfo("Recreate building workspace: " + request.yarnProvidedPath()); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - - List mavenJars = - execStep( - 2, - () -> { - if (request.flinkJobType() == FlinkJobType.FLINK_SQL - || request.flinkJobType() == FlinkJobType.PYFLINK) { - List mavenArts = - MavenTool.resolveArtifacts(request.dependencyInfo().mavenArts()); - List paths = - mavenArts.stream() - .map(File::getAbsolutePath) - .collect(Collectors.toList()); - paths.addAll(request.dependencyInfo().extJarLibs()); - return paths; - } - return Collections.emptyList(); - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - - execStep( - 3, - () -> { - for (String jar : mavenJars) { - uploadJarToHdfsOrLfs(FsOperator.lfs(), jar, request.localWorkspace()); - uploadJarToHdfsOrLfs(FsOperator.hdfs(), jar, request.yarnProvidedPath()); - } - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - + boolean sqlMode = + request.flinkJobType() == FlinkJobType.FLINK_SQL + || request.flinkJobType() == FlinkJobType.PYFLINK; + runYarnSqlBuildSteps( + request.localWorkspace(), + request.yarnProvidedPath(), + sqlMode, + request.dependencyInfo()); return new SimpleBuildResponse(); } - private void uploadJarToHdfsOrLfs(FsOperator fsOperator, String origin, String target) throws IOException { - File originFile = new File(origin); - if (!fsOperator.exists(target)) { - fsOperator.mkdirs(target); - } - if (originFile.isFile()) { - if (fsOperator == FsOperator.lfs()) { - fsOperator.copy(originFile.getAbsolutePath(), target); - } else { - String uploadFile = Workspace.remote().APP_UPLOADS() + "/" + originFile.getName(); - if (fsOperator.exists(uploadFile)) { - AutoCloseUtils.using( - new FileInputStream(originFile), - inputStream -> { - try { - if (!DigestUtils.md5Hex(inputStream) - .equals(fsOperator.fileMd5(uploadFile))) { - fsOperator.upload(originFile.getAbsolutePath(), uploadFile); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return null; - }); - } else { - fsOperator.upload(originFile.getAbsolutePath(), uploadFile); - } - fsOperator.copy(uploadFile, target); - } - } else if (fsOperator == FsOperator.hdfs()) { - fsOperator.upload(originFile.getAbsolutePath(), target); - } - } - public static FlinkYarnApplicationBuildPipeline of(FlinkYarnApplicationBuildRequest request) { return new FlinkYarnApplicationBuildPipeline(request); } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.java index 2bdd431f89..9d857259b6 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkK8sApplicationBuildPipeline.java @@ -18,67 +18,27 @@ package org.apache.streampark.flink.packer.pipeline.impl; import org.apache.streampark.common.fs.LfsOperator; -import org.apache.streampark.common.util.ThreadUtils; import org.apache.streampark.flink.kubernetes.PodTemplateTool; import org.apache.streampark.flink.packer.docker.DockerConf; -import org.apache.streampark.flink.packer.docker.DockerUtils; import org.apache.streampark.flink.packer.docker.SparkDockerfileTemplate; import org.apache.streampark.flink.packer.docker.SparkDockerfileTemplateTrait; import org.apache.streampark.flink.packer.docker.SparkHadoopDockerfileTemplate; -import org.apache.streampark.flink.packer.pipeline.BuildPipeline; -import org.apache.streampark.flink.packer.pipeline.DockerBuildProgress; import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse; -import org.apache.streampark.flink.packer.pipeline.DockerProgressWatcher; -import org.apache.streampark.flink.packer.pipeline.DockerPullProgress; -import org.apache.streampark.flink.packer.pipeline.DockerPushProgress; -import org.apache.streampark.flink.packer.pipeline.DockerResolveProgress; +import org.apache.streampark.flink.packer.pipeline.K8sDockerBuildSupport; import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum; -import org.apache.streampark.flink.packer.pipeline.SilentDockerProgressWatcher; import org.apache.streampark.flink.packer.pipeline.SparkK8sApplicationBuildRequest; -import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates; - -import org.apache.commons.lang3.StringUtils; - -import com.github.dockerjava.api.command.PushImageCmd; -import com.github.dockerjava.core.command.HackBuildImageCmd; -import com.github.dockerjava.core.command.HackPullImageCmd; -import com.github.dockerjava.core.command.HackPushImageCmd; -import com.google.common.collect.Sets; import java.io.File; import java.nio.file.Paths; -import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** Building pipeline for Spark kubernetes-native application mode */ -public class SparkK8sApplicationBuildPipeline extends BuildPipeline { - - private static final ExecutorService DOCKER_EXECUTOR = - new ThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 5, - Runtime.getRuntime().availableProcessors() * 10, - 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(2048), - ThreadUtils.threadFactory("streampark-docker-progress-watcher-executor"), - new ThreadPoolExecutor.DiscardOldestPolicy()); +public class SparkK8sApplicationBuildPipeline extends AbstractK8sApplicationBuildPipeline { private final SparkK8sApplicationBuildRequest request; - private DockerProgressWatcher dockerProcessWatcher = new SilentDockerProgressWatcher(); - - private final DockerResolveProgress dockerProcess = - new DockerResolveProgress( - DockerPullProgress.empty(), - DockerBuildProgress.empty(), - DockerPushProgress.empty()); - public SparkK8sApplicationBuildPipeline(SparkK8sApplicationBuildRequest request) { this.request = request; } @@ -93,10 +53,6 @@ public SparkK8sApplicationBuildRequest offerBuildParam() { return request; } - public void registerDockerProgressWatcher(DockerProgressWatcher watcher) { - this.dockerProcessWatcher = watcher; - } - @Override public DockerImageBuildResponse buildProcess() { String buildWorkspace = @@ -108,52 +64,30 @@ public DockerImageBuildResponse buildProcess() { logInfo("Recreate building workspace: " + workspace); return workspace; }) - .orElseThrow(() -> { - throw pipelineException(); - }); + .orElseThrow(this::pipelineException); - Map podTemplatePaths; - SparkK8sPodTemplates podTemplate = request.sparkPodTemplate(); - if (podTemplate.isEmpty()) { - skipStep(2); - podTemplatePaths = Collections.emptyMap(); - } else { - podTemplatePaths = - execStep( - 2, - () -> { - Map podTemplateFiles = - PodTemplateTool.preparePodTemplateFiles(buildWorkspace, podTemplate) - .tmplFiles(); - logInfo( - "Export spark podTemplates: " - + String.join(",", podTemplateFiles.values())); - return podTemplateFiles; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - } + Map podTemplatePaths = + preparePodTemplateStep( + buildWorkspace, + 2, + request.sparkPodTemplate().isEmpty(), + () -> PodTemplateTool.preparePodTemplateFiles(buildWorkspace, request.sparkPodTemplate()) + .tmplFiles(), + "spark"); final String mainJarPath = execStep( 3, () -> { - String mainJarName = - Paths.get(request.mainJar()).getFileName().toString(); + String mainJarName = Paths.get(request.mainJar()).getFileName().toString(); String path = buildWorkspace + "/" + mainJarName; LfsOperator.copy(request.mainJar(), path); logInfo("Prepared spark job jar: " + path); return path; }) - .orElseThrow( - () -> { - throw pipelineException(); - }); + .orElseThrow(this::pipelineException); final Set extJarLibs = new HashSet<>(); - File dockerfile; - SparkDockerfileTemplateTrait dockerFileTemplate; Object[] dockerResult = execStep( 4, @@ -182,145 +116,22 @@ public DockerImageBuildResponse buildProcess() { + template.offerDockerfileContent()); return new Object[]{dockerFile, template}; }) - .orElseThrow(() -> { - throw pipelineException(); - }); - dockerfile = (File) dockerResult[0]; - dockerFileTemplate = (SparkDockerfileTemplateTrait) dockerResult[1]; + .orElseThrow(this::pipelineException); + File dockerfile = (File) dockerResult[0]; + SparkDockerfileTemplateTrait dockerFileTemplate = (SparkDockerfileTemplateTrait) dockerResult[1]; DockerConf dockerConf = request.dockerConfig(); String baseImageTag = request.sparkBaseImage().trim(); if (request.k8sNamespace().isEmpty() || request.appName().isEmpty()) { throw new IllegalArgumentException("k8sNamespace or appName cannot be empty"); } - String expectedImageTag = - "streampark-sparkjob-" + request.k8sNamespace() + "-" + request.appName(); String pushImageTag = - compileTag( - expectedImageTag, dockerConf.registerAddress(), dockerConf.imageNamespace()); - - execStep( - 5, - () -> { - DockerUtils.usingDockerClient( - dockerClient -> { - HackPullImageCmd pullImageCmd; - if (dockerConf.registerAddress() != null - && !baseImageTag.startsWith(dockerConf.registerAddress())) { - pullImageCmd = - (HackPullImageCmd) dockerClient.pullImageCmd(baseImageTag); - } else { - pullImageCmd = - (HackPullImageCmd) dockerClient - .pullImageCmd(baseImageTag) - .withAuthConfig(dockerConf.toAuthConf()); - } - try { - pullImageCmd - .start( - DockerUtils.watchDockerPullProcess( - pullRsp -> { - dockerProcess.getPull().update(pullRsp); - DOCKER_EXECUTOR.submit( - () -> dockerProcessWatcher.onDockerPullProgressChange( - dockerProcess.getPull().snapshot())); - })) - .awaitCompletion(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - logInfo( - "Already pulled docker image from remote register, imageTag=" - + baseImageTag); - return null; - }, - err -> { - throw new RuntimeException( - "Pull docker image failed, imageTag=" + baseImageTag, err); - }); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - - execStep( - 6, - () -> { - DockerUtils.usingDockerClient( - dockerClient -> { - HackBuildImageCmd buildImageCmd = - (HackBuildImageCmd) dockerClient - .buildImageCmd() - .withBaseDirectory(new File(buildWorkspace)) - .withDockerfile(dockerfile) - .withTags(Sets.newHashSet(pushImageTag)); - String imageId = - buildImageCmd - .start( - DockerUtils.watchDockerBuildStep( - buildStep -> { - dockerProcess.getBuild().update(buildStep); - DOCKER_EXECUTOR.submit( - () -> dockerProcessWatcher - .onDockerBuildProgressChange( - dockerProcess.getBuild().snapshot())); - })) - .awaitImageId(); - logInfo( - "Built docker image, imageId=" - + imageId - + ", imageTag=" - + pushImageTag); - return null; - }, - err -> { - throw new RuntimeException( - "Build docker image failed. tag=" + pushImageTag, err); - }); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); + K8sDockerBuildSupport.compileTag( + "streampark-sparkjob-" + request.k8sNamespace() + "-" + request.appName(), + dockerConf.registerAddress(), + dockerConf.imageNamespace()); - execStep( - 7, - () -> { - DockerUtils.usingDockerClient( - dockerClient -> { - PushImageCmd pushCmd = - dockerClient - .pushImageCmd(pushImageTag) - .withAuthConfig(dockerConf.toAuthConf()); - try { - ((HackPushImageCmd) pushCmd) - .start( - DockerUtils.watchDockerPushProcess( - pushRsp -> { - dockerProcess.getPush().update(pushRsp); - DOCKER_EXECUTOR.submit( - () -> dockerProcessWatcher.onDockerPushProgressChange( - dockerProcess.getPush().snapshot())); - })) - .awaitCompletion(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - logInfo("Already pushed docker image, imageTag=" + pushImageTag); - return null; - }, - err -> { - throw new RuntimeException( - "Push docker image failed. tag=" + pushImageTag, err); - }); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); + runDockerBuildSteps(buildWorkspace, dockerfile, dockerConf, baseImageTag, pushImageTag, false); return new DockerImageBuildResponse( buildWorkspace, @@ -329,14 +140,6 @@ public DockerImageBuildResponse buildProcess() { dockerFileTemplate.innerMainJarPath()); } - private String compileTag(String tag, String registerAddress, String imageNamespace) { - String tagName = tag.contains("/") ? tag : imageNamespace + "/" + tag; - if (StringUtils.isNotBlank(registerAddress) && !tagName.startsWith(registerAddress)) { - tagName = registerAddress + "/" + tagName; - } - return tagName.toLowerCase(); - } - public static SparkK8sApplicationBuildPipeline of(SparkK8sApplicationBuildRequest request) { return new SparkK8sApplicationBuildPipeline(request); } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.java index 52cc4f5908..2046ab18ea 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/impl/SparkYarnBuildPipeline.java @@ -17,27 +17,12 @@ package org.apache.streampark.flink.packer.pipeline.impl; -import org.apache.streampark.common.conf.Workspace; import org.apache.streampark.common.enums.SparkJobType; -import org.apache.streampark.common.fs.FsOperator; -import org.apache.streampark.common.fs.HdfsOperator; -import org.apache.streampark.common.fs.LfsOperator; -import org.apache.streampark.common.util.AutoCloseUtils; -import org.apache.streampark.flink.packer.maven.MavenTool; import org.apache.streampark.flink.packer.pipeline.BuildPipeline; import org.apache.streampark.flink.packer.pipeline.PipelineTypeEnum; import org.apache.streampark.flink.packer.pipeline.SimpleBuildResponse; import org.apache.streampark.flink.packer.pipeline.SparkYarnBuildRequest; -import org.apache.commons.codec.digest.DigestUtils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - /** Building pipeline for spark yarn application mode */ public class SparkYarnBuildPipeline extends BuildPipeline { @@ -59,90 +44,14 @@ public SparkYarnBuildRequest offerBuildParam() { @Override public SimpleBuildResponse buildProcess() { - execStep( - 1, - () -> { - if (request.jobType() == SparkJobType.SPARK_SQL) { - LfsOperator.mkCleanDirs(request.localWorkspace()); - HdfsOperator.mkCleanDirs(request.yarnProvidedPath()); - } - logInfo("Recreate building workspace: " + request.yarnProvidedPath()); - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - - List mavenJars = - execStep( - 2, - () -> { - if (request.jobType() == SparkJobType.SPARK_SQL) { - List mavenArts = - MavenTool.resolveArtifacts(request.dependencyInfo().mavenArts()); - List paths = - mavenArts.stream() - .map(File::getAbsolutePath) - .collect(Collectors.toList()); - paths.addAll(request.dependencyInfo().extJarLibs()); - return paths; - } - return Collections.emptyList(); - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - - execStep( - 3, - () -> { - for (String jar : mavenJars) { - uploadJarToHdfsOrLfs(FsOperator.lfs(), jar, request.localWorkspace()); - uploadJarToHdfsOrLfs(FsOperator.hdfs(), jar, request.yarnProvidedPath()); - } - return null; - }) - .orElseThrow(() -> { - throw pipelineException(); - }); - + runYarnSqlBuildSteps( + request.localWorkspace(), + request.yarnProvidedPath(), + request.jobType() == SparkJobType.SPARK_SQL, + request.dependencyInfo()); return new SimpleBuildResponse(); } - private void uploadJarToHdfsOrLfs(FsOperator fsOperator, String origin, String target) throws IOException { - File originFile = new File(origin); - if (!fsOperator.exists(target)) { - fsOperator.mkdirs(target); - } - if (originFile.isFile()) { - if (fsOperator == FsOperator.lfs()) { - fsOperator.copy(originFile.getAbsolutePath(), target); - } else { - String uploadFile = Workspace.remote().APP_UPLOADS() + "/" + originFile.getName(); - if (fsOperator.exists(uploadFile)) { - AutoCloseUtils.using( - new FileInputStream(originFile), - inputStream -> { - try { - if (!DigestUtils.md5Hex(inputStream) - .equals(fsOperator.fileMd5(uploadFile))) { - fsOperator.upload(originFile.getAbsolutePath(), uploadFile); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return null; - }); - } else { - fsOperator.upload(originFile.getAbsolutePath(), uploadFile); - } - fsOperator.copy(uploadFile, target); - } - } else if (fsOperator == FsOperator.hdfs()) { - fsOperator.upload(originFile.getAbsolutePath(), target); - } - } - public static SparkYarnBuildPipeline of(SparkYarnBuildRequest request) { return new SparkYarnBuildPipeline(request); } diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java index 3e4d85ba8c..743c073ca8 100644 --- a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java +++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java @@ -66,7 +66,7 @@ static void prepareDocker() { .awaitCompletion(2, TimeUnit.MINUTES); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); - throw new RuntimeException( + throw new IllegalStateException( "Interrupted while pulling docker test image: hello-world:latest", interrupted); } @@ -76,7 +76,7 @@ static void prepareDocker() { return null; }, err -> { - throw new RuntimeException("Failed to prepare docker test image: " + TEST_IMAGE, err); + throw new IllegalStateException("Failed to prepare docker test image: " + TEST_IMAGE, err); }); } diff --git a/streampark-flink/streampark-flink-proxy/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java b/streampark-flink/streampark-flink-proxy/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java index 5337e0d117..53f5ec0ad9 100644 --- a/streampark-flink/streampark-flink-proxy/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java +++ b/streampark-flink/streampark-flink-proxy/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java @@ -30,6 +30,7 @@ import java.io.File; import java.io.IOException; import java.io.ObjectOutputStream; +import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -108,8 +109,8 @@ public static T proxyVerifySql(FlinkVersion flinkVersion, Function func.apply(shimsClassLoader)); } - @SuppressWarnings("unchecked") - public static T getObject(ClassLoader loader, Object obj) throws IOException, ClassNotFoundException { + public static T getObject(ClassLoader loader, Object obj, + Class type) throws IOException, ClassNotFoundException { try ( ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(arrayOutputStream)) { @@ -119,7 +120,7 @@ public static T getObject(ClassLoader loader, Object obj) throws IOException new ByteArrayInputStream(arrayOutputStream.toByteArray()); ClassLoaderObjectInputStream in = new ClassLoaderObjectInputStream(loader, byteArrayInputStream)) { - return (T) in.readObject(); + return type.cast(in.readObject()); } } } @@ -140,11 +141,7 @@ private static ClassLoader getVerifySqlLibClassLoader(FlinkVersion flinkVersion) flinkVersion, file -> { if (file.getName().startsWith("streampark-flink-shims")) { - try { - shimsUrls.add(file.toURI().toURL()); - } catch (Exception e) { - throw new RuntimeException(e); - } + shimsUrls.add(toUrl(file)); } }); @@ -223,11 +220,7 @@ private static ClassLoader getFlinkShimsClassLoader(FlinkVersion flinkVersion) { flinkVersion, file -> { if (file != null) { - try { - shimsUrls.add(file.toURI().toURL()); - } catch (Exception e) { - throw new RuntimeException(e); - } + shimsUrls.add(toUrl(file)); } }); @@ -254,13 +247,17 @@ private static List getFlinkHomeLib( List urls = new ArrayList<>(); for (File f : files) { if (filterFun.test(f)) { - try { - urls.add(f.toURI().toURL()); - } catch (Exception e) { - throw new RuntimeException(e); - } + urls.add(toUrl(f)); } } return urls; } + + private static URL toUrl(File file) { + try { + return file.toURI().toURL(); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Invalid file URL: " + file, e); + } + } } diff --git a/streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java b/streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java index dbb3d60366..c16d1ee869 100644 --- a/streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java +++ b/streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java @@ -28,7 +28,8 @@ class FlinkShimsProxyTest { void getObjectShouldDeserializeAcrossClassLoaders() throws Exception { String payload = "streampark-proxy-test"; ClassLoader loader = Thread.currentThread().getContextClassLoader(); - Serializable roundTripped = FlinkShimsProxy.getObject(loader, payload); + Serializable roundTripped = + FlinkShimsProxy.getObject(loader, payload, String.class); Assertions.assertEquals(payload, roundTripped); } @@ -36,7 +37,7 @@ void getObjectShouldDeserializeAcrossClassLoaders() throws Exception { void getObjectShouldPreserveCustomSerializableType() throws Exception { TestBean original = new TestBean("app-1", 42); ClassLoader loader = Thread.currentThread().getContextClassLoader(); - TestBean roundTripped = FlinkShimsProxy.getObject(loader, original); + TestBean roundTripped = FlinkShimsProxy.getObject(loader, original, TestBean.class); Assertions.assertEquals(original, roundTripped); } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkClientTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkClientTrait.java index 2f8da33f78..a920be68ee 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkClientTrait.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkClientTrait.java @@ -37,7 +37,9 @@ public CompletableFuture triggerSavepoint(JobID jobID, String savepointD public CompletableFuture triggerSavepoint( JobID jobID, String savepointDir, boolean nativeFormat) { - return clusterClient.triggerSavepoint(jobID, savepointDir, SavepointFormatType.DEFAULT); + SavepointFormatType formatType = + nativeFormat ? SavepointFormatType.NATIVE : SavepointFormatType.DEFAULT; + return clusterClient.triggerSavepoint(jobID, savepointDir, formatType); } public CompletableFuture cancelWithSavepoint(JobID jobID, String savepointDir) { @@ -46,7 +48,9 @@ public CompletableFuture cancelWithSavepoint(JobID jobID, String savepoi public CompletableFuture cancelWithSavepoint( JobID jobID, String savepointDir, boolean nativeFormat) { - return clusterClient.cancelWithSavepoint(jobID, savepointDir, SavepointFormatType.DEFAULT); + SavepointFormatType formatType = + nativeFormat ? SavepointFormatType.NATIVE : SavepointFormatType.DEFAULT; + return clusterClient.cancelWithSavepoint(jobID, savepointDir, formatType); } public CompletableFuture stopWithSavepoint( @@ -60,7 +64,9 @@ public CompletableFuture stopWithSavepoint( boolean advanceToEndOfEventTime, String savepointDir, boolean nativeFormat) { + SavepointFormatType formatType = + nativeFormat ? SavepointFormatType.NATIVE : SavepointFormatType.DEFAULT; return clusterClient.stopWithSavepoint( - jobID, advanceToEndOfEventTime, savepointDir, SavepointFormatType.DEFAULT); + jobID, advanceToEndOfEventTime, savepointDir, formatType); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java index 6880f3f3c3..0fabead7dd 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java @@ -29,8 +29,8 @@ import org.apache.flink.table.api.TableResult; import java.lang.reflect.Field; -import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; @@ -66,108 +66,8 @@ static void executeSql( List commands = SqlCommandParser.parseSQL(flinkSql); for (SqlCommandCall call : commands) { - String args = call.operands().length == 0 ? null : call.operands()[0]; - String command = call.command().getCommandName(); - switch (call.command()) { - case SHOW_CATALOGS: - callback( - callbackFunc, - command + ": " + String.join("\n", context.listCatalogs())); - break; - case SHOW_CURRENT_CATALOG: - callback(callbackFunc, command + ": " + context.getCurrentCatalog()); - break; - case SHOW_DATABASES: - callback( - callbackFunc, - command + ": " + String.join("\n", context.listDatabases())); - break; - case SHOW_CURRENT_DATABASE: - callback(callbackFunc, command + ": " + context.getCurrentDatabase()); - break; - case SHOW_TABLES: - StringBuilder tables = new StringBuilder(); - for (String table : context.listTables()) { - if (!table.startsWith("UnnamedTable")) { - if (tables.length() > 0) { - tables.append('\n'); - } - tables.append(table); - } - } - callback(callbackFunc, command + ": " + tables); - break; - case SHOW_FUNCTIONS: - callback( - callbackFunc, - command + ": " + String.join("\n", context.listUserDefinedFunctions())); - break; - case SHOW_MODULES: - callback( - callbackFunc, - command + ": " + String.join("\n", context.listModules())); - break; - case DESC: - case DESCRIBE: - var schema = context.scan(args).getSchema(); - StringBuilder builder = new StringBuilder("Column\tType\n"); - for (int i = 0; i <= schema.getFieldCount(); i++) { - builder - .append(schema.getFieldName(i).get()) - .append('\t') - .append(schema.getFieldDataType(i).get()) - .append('\n'); - } - callback(callbackFunc, builder.toString()); - break; - case EXPLAIN: - TableResult tableResult = context.executeSql(call.originSql()); - String explain = tableResult.collect().next().getField(0).toString(); - callback(callbackFunc, explain); - break; - case SET: - String operand = call.operands()[1]; - LOG.info(command + ": " + args + " --> " + operand); - context.getConfig().getConfiguration().setString(args, operand); - break; - case RESET: - case RESET_ALL: - resetConfiguration(context, call.command(), args); - LOG.info(command + ": " + args); - break; - case BEGIN_STATEMENT_SET: - case END_STATEMENT_SET: - LOG.warn("SQL Client Syntax: " + call.command().getCommandName()); - break; - case INSERT: - statementSet.addInsertSql(call.originSql()); - hasInsert = true; - break; - case SELECT: - LOG.error("StreamPark dose not support 'SELECT' statement now!"); - throw new RuntimeException("StreamPark dose not support 'select' statement now!"); - case DELETE: - case UPDATE: - AssertUtils.required( - !"STREAMING".equals(runMode), - "Currently, " - + command.toUpperCase() - + " statement only supports in batch mode, " - + "and it requires the target table connector implements the SupportsRowLevelDelete, " - + "For more details please refer to: https://nightlies.apache.org/flink/flink-docs-release-1.18/docs/dev/table/sql/" - + command.toLowerCase()); - break; - default: - LOCK.lock(); - try { - context.executeSql(call.originSql()); - LOG.info(command + ":" + args); - } finally { - if (LOCK.isHeldByCurrentThread()) { - LOCK.unlock(); - } - } - break; + if (handleCommand(call, context, statementSet, callbackFunc, runMode)) { + hasInsert = true; } } @@ -178,13 +78,138 @@ static void executeSql( } } else { LOG.error("No 'INSERT' statement to trigger the execution of the Flink job."); - throw new RuntimeException("No 'INSERT' statement to trigger the execution of the Flink job."); + throw new IllegalStateException( + "No 'INSERT' statement to trigger the execution of the Flink job."); } LOG.info( "\n\n\n==============flinkSql==============\n\n " + flinkSql + "\n\n============================\n\n\n"); } + /** + * @return {@code true} when an INSERT statement was added to the statement set + */ + private static boolean handleCommand( + SqlCommandCall call, + TableEnvironment context, + StatementSet statementSet, + Consumer callbackFunc, + String runMode) { + String args = call.operands().length == 0 ? null : call.operands()[0]; + String command = call.command().getCommandName(); + switch (call.command()) { + case SHOW_CATALOGS: + callback(callbackFunc, command + ": " + String.join("\n", context.listCatalogs())); + return false; + case SHOW_CURRENT_CATALOG: + callback(callbackFunc, command + ": " + context.getCurrentCatalog()); + return false; + case SHOW_DATABASES: + callback(callbackFunc, command + ": " + String.join("\n", context.listDatabases())); + return false; + case SHOW_CURRENT_DATABASE: + callback(callbackFunc, command + ": " + context.getCurrentDatabase()); + return false; + case SHOW_TABLES: + callback(callbackFunc, command + ": " + listVisibleTables(context)); + return false; + case SHOW_FUNCTIONS: + callback( + callbackFunc, + command + ": " + String.join("\n", context.listUserDefinedFunctions())); + return false; + case SHOW_MODULES: + callback(callbackFunc, command + ": " + String.join("\n", context.listModules())); + return false; + case DESC: + case DESCRIBE: + callback(callbackFunc, describeTable(context, args)); + return false; + case EXPLAIN: + TableResult tableResult = context.executeSql(call.originSql()); + String explain = tableResult.collect().next().getField(0).toString(); + callback(callbackFunc, explain); + return false; + case SET: + String operand = call.operands()[1]; + LOG.info(command + ": " + args + " --> " + operand); + context.getConfig().getConfiguration().setString(args, operand); + return false; + case RESET: + case RESET_ALL: + resetConfiguration(context, call.command(), args); + LOG.info(command + ": " + args); + return false; + case BEGIN_STATEMENT_SET: + case END_STATEMENT_SET: + LOG.warn("SQL Client Syntax: " + call.command().getCommandName()); + return false; + case INSERT: + statementSet.addInsertSql(call.originSql()); + return true; + case SELECT: + LOG.error("StreamPark dose not support 'SELECT' statement now!"); + throw new UnsupportedOperationException( + "StreamPark dose not support 'select' statement now!"); + case DELETE: + case UPDATE: + AssertUtils.required( + !"STREAMING".equals(runMode), + "Currently, " + + command.toUpperCase() + + " statement only supports in batch mode, " + + "and it requires the target table connector implements the SupportsRowLevelDelete, " + + "For more details please refer to: https://nightlies.apache.org/flink/flink-docs-release-1.18/docs/dev/table/sql/" + + command.toLowerCase()); + return false; + default: + executeDefaultSql(context, call, command, args); + return false; + } + } + + private static String listVisibleTables(TableEnvironment context) { + StringBuilder tables = new StringBuilder(); + for (String table : context.listTables()) { + if (!table.startsWith("UnnamedTable")) { + if (tables.length() > 0) { + tables.append('\n'); + } + tables.append(table); + } + } + return tables.toString(); + } + + private static String describeTable(TableEnvironment context, String args) { + var schema = context.scan(args).getSchema(); + StringBuilder builder = new StringBuilder("Column\tType\n"); + for (int i = 0; i <= schema.getFieldCount(); i++) { + builder + .append(schema.getFieldName(i).get()) + .append('\t') + .append(schema.getFieldDataType(i).get()) + .append('\n'); + } + return builder.toString(); + } + + private static void executeDefaultSql( + TableEnvironment context, + SqlCommandCall call, + String command, + String args) { + LOCK.lock(); + try { + context.executeSql(call.originSql()); + LOG.info(command + ":" + args); + } finally { + if (LOCK.isHeldByCurrentThread()) { + LOCK.unlock(); + } + } + } + private static void callback(Consumer callbackFunc, String message) { if (callbackFunc == null) { LOG.info(message); @@ -193,13 +218,15 @@ private static void callback(Consumer callbackFunc, String message) { } } - @SuppressWarnings("unchecked") private static void resetConfiguration(TableEnvironment context, SqlCommand command, String args) { try { Field confDataField = Configuration.class.getDeclaredField("confData"); confDataField.setAccessible(true); - HashMap confData = - (HashMap) confDataField.get(context.getConfig().getConfiguration()); + Object confDataObject = confDataField.get(context.getConfig().getConfiguration()); + if (!(confDataObject instanceof Map)) { + throw new IllegalStateException("Unexpected Flink configuration internal structure"); + } + Map confData = (Map) confDataObject; synchronized (confData) { if (command == SqlCommand.RESET) { confData.remove(args); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidationResult.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidationResult.java index 8c6748fea9..0008b5df01 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidationResult.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidationResult.java @@ -31,26 +31,30 @@ public class FlinkSqlValidationResult { private String sql; private String exception; + /** Default success result; mirrors the Scala case-class no-arg constructor. */ public FlinkSqlValidationResult() { - } - - public FlinkSqlValidationResult( - boolean success, - FlinkSqlValidationFailedType failedType, - int lineStart, - int lineEnd, - int errorLine, - int errorColumn, - String sql, - String exception) { - this.success = success; - this.failedType = failedType; - this.lineStart = lineStart; - this.lineEnd = lineEnd; - this.errorLine = errorLine; - this.errorColumn = errorColumn; - this.sql = sql; - this.exception = exception; + // Intentionally empty: fields default to a successful validation result. + } + + /** Builds a failed validation result without exposing an oversized public constructor. */ + public static FlinkSqlValidationResult failure( + FlinkSqlValidationFailedType failedType, + int lineStart, + int lineEnd, + int errorLine, + int errorColumn, + String sql, + String exception) { + FlinkSqlValidationResult result = new FlinkSqlValidationResult(); + result.success = false; + result.failedType = failedType; + result.lineStart = lineStart; + result.lineEnd = lineEnd; + result.errorLine = errorLine; + result.errorColumn = errorColumn; + result.sql = sql; + result.exception = exception; + return result; } /** Scala field-style access. */ diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidator.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidator.java index 889b491a57..ed61674330 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidator.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlValidator.java @@ -46,7 +46,7 @@ public final class FlinkSqlValidator { "org.apache.flink.table.planner.parse.CalciteParser"; private static final Pattern SYNTAX_ERROR_PATTERN = - Pattern.compile(".*at\\sline\\s(\\d+),\\scolumn\\s(\\d+).*"); + Pattern.compile("at\\sline\\s(\\d+),\\scolumn\\s(\\d+)"); private static final Map SQL_PARSER_CONFIG_MAP = createSqlParserConfigMap(); @@ -64,84 +64,79 @@ public static FlinkSqlValidationResult verifySql(String sql) { return earlyResult; } - String sqlDialect = SqlDialect.DEFAULT.name().toLowerCase(); - boolean hasInsert = false; + ValidationState state = new ValidationState(); for (SqlCommandCall call : sqlCommands) { - String args = call.operands()[0]; - SqlCommand command = call.command(); - switch (command) { - case SET: - case RESET: - if (command == SqlCommand.SET - && args.equals(TableConfigOptions.TABLE_SQL_DIALECT.key())) { - sqlDialect = call.operands()[call.operands().length - 1]; - } - break; - case BEGIN_STATEMENT_SET: - case END_STATEMENT_SET: - LOG.warn("SQL Client Syntax: " + call.command().getCommandName()); - break; - default: - if (command == SqlCommand.INSERT) { - hasInsert = true; - } - try { - Class calciteClass; - try { - calciteClass = Class.forName(FLINK112_CALCITE_PARSER_CLASS); - } catch (ClassNotFoundException e) { - calciteClass = Class.forName(FLINK113_PLUS_CALCITE_PARSER_CLASS); - } - String dialectUpper = sqlDialect.toUpperCase(); - if ("HIVE".equals(dialectUpper)) { - // skip calcite validation for HIVE dialect - } else if ("DEFAULT".equals(dialectUpper)) { - Constructor constructor = - calciteClass.getConstructor(SqlParser.Config.class); - Object parser = - constructor.newInstance( - SQL_PARSER_CONFIG_MAP.get(sqlDialect.toUpperCase())); - Method method = parser.getClass().getDeclaredMethod("parse", String.class); - method.setAccessible(true); - method.invoke(parser, call.originSql()); - } else { - throw new UnsupportedOperationException( - "unsupported dialect: " + sqlDialect); - } - } catch (ReflectiveOperationException e) { - return syntaxErrorResult(call, e); - } catch (RuntimeException e) { - return syntaxErrorResult(call, e); - } - break; + FlinkSqlValidationResult failure = processCommand(call, state); + if (failure != null) { + return failure; } } + return state.toResult(sqlCommands); + } - if (hasInsert) { - return new FlinkSqlValidationResult(); + private static FlinkSqlValidationResult processCommand(SqlCommandCall call, ValidationState state) { + SqlCommand command = call.command(); + switch (command) { + case SET: + case RESET: + state.updateDialect(command, call); + return null; + case BEGIN_STATEMENT_SET: + case END_STATEMENT_SET: + LOG.warn("SQL Client Syntax: " + command.getCommandName()); + return null; + default: + if (command == SqlCommand.INSERT) { + state.markInsert(); + } + try { + validateSqlCommand(call, state.sqlDialect); + } catch (IllegalStateException | UnsupportedOperationException e) { + return syntaxErrorResult(call, e); + } + return null; + } + } + + private static void validateSqlCommand(SqlCommandCall call, String sqlDialect) { + if ("HIVE".equalsIgnoreCase(sqlDialect)) { + return; + } + if (!"DEFAULT".equalsIgnoreCase(sqlDialect)) { + throw new UnsupportedOperationException("unsupported dialect: " + sqlDialect); + } + try { + Class calciteClass = resolveCalciteParserClass(); + Constructor constructor = calciteClass.getConstructor(SqlParser.Config.class); + Object parser = + constructor.newInstance(SQL_PARSER_CONFIG_MAP.get(sqlDialect.toUpperCase())); + Method method = parser.getClass().getDeclaredMethod("parse", String.class); + method.setAccessible(true); + method.invoke(parser, call.originSql()); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Failed to parse SQL with Calcite", e); + } + } + + private static Class resolveCalciteParserClass() throws ClassNotFoundException { + try { + return Class.forName(FLINK112_CALCITE_PARSER_CLASS); + } catch (ClassNotFoundException e) { + return Class.forName(FLINK113_PLUS_CALCITE_PARSER_CLASS); } - return new FlinkSqlValidationResult( - false, - FlinkSqlValidationFailedType.SYNTAX_ERROR, - sqlCommands.get(0).lineStart(), - sqlCommands.get(sqlCommands.size() - 1).lineEnd(), - 0, - 0, - null, - "No 'INSERT' statement to trigger the execution of the Flink job."); } private static FlinkSqlValidationResult syntaxErrorResult(SqlCommandCall call, Throwable e) { String exception = ExceptionUtils.stringifyException(e); - String causedBy = exception.substring(exception.indexOf("Caused by:")); + int causedByIndex = exception.indexOf("Caused by:"); + String causedBy = causedByIndex >= 0 ? exception.substring(causedByIndex) : exception; String cleanUpError = exception.replaceAll("[\r\n]", ""); Matcher matcher = SYNTAX_ERROR_PATTERN.matcher(cleanUpError); - if (matcher.matches()) { + if (matcher.find()) { int line = Integer.parseInt(matcher.group(1)); int column = Integer.parseInt(matcher.group(2)); int errorLine = call.lineStart() + line - 1; - return new FlinkSqlValidationResult( - false, + return FlinkSqlValidationResult.failure( FlinkSqlValidationFailedType.SYNTAX_ERROR, call.lineStart(), call.lineEnd(), @@ -150,8 +145,7 @@ private static FlinkSqlValidationResult syntaxErrorResult(SqlCommandCall call, T call.originSql(), causedBy.replaceAll("at\\sline\\s" + line, "at line " + errorLine)); } - return new FlinkSqlValidationResult( - false, + return FlinkSqlValidationResult.failure( FlinkSqlValidationFailedType.SYNTAX_ERROR, call.lineStart(), call.lineEnd(), @@ -169,16 +163,8 @@ private static Map createSqlParserConfigMap() { } private static SqlParser.Config getConfig(SqlDialect sqlDialect) { - org.apache.calcite.sql.validate.SqlConformance conformance; - if (sqlDialect == SqlDialect.HIVE) { - try { - conformance = FlinkSqlConformance.DEFAULT; - } catch (NoSuchFieldError e) { - conformance = FlinkSqlConformance.DEFAULT; - } - } else if (sqlDialect == SqlDialect.DEFAULT) { - conformance = FlinkSqlConformance.DEFAULT; - } else { + org.apache.calcite.sql.validate.SqlConformance conformance = FlinkSqlConformance.DEFAULT; + if (sqlDialect != SqlDialect.DEFAULT && sqlDialect != SqlDialect.HIVE) { throw new UnsupportedOperationException("Unsupported sqlDialect: " + sqlDialect); } return SqlParser.config() @@ -188,6 +174,37 @@ private static SqlParser.Config getConfig(SqlDialect sqlDialect) { .withIdentifierMaxLength(256); } + private static final class ValidationState { + + private String sqlDialect = SqlDialect.DEFAULT.name().toLowerCase(); + private boolean hasInsert; + + private void updateDialect(SqlCommand command, SqlCommandCall call) { + if (command == SqlCommand.SET + && call.operands()[0].equals(TableConfigOptions.TABLE_SQL_DIALECT.key())) { + sqlDialect = call.operands()[call.operands().length - 1]; + } + } + + private void markInsert() { + hasInsert = true; + } + + private FlinkSqlValidationResult toResult(List sqlCommands) { + if (hasInsert) { + return new FlinkSqlValidationResult(); + } + return FlinkSqlValidationResult.failure( + FlinkSqlValidationFailedType.SYNTAX_ERROR, + sqlCommands.get(0).lineStart(), + sqlCommands.get(sqlCommands.size() - 1).lineEnd(), + 0, + 0, + null, + "No 'INSERT' statement to trigger the execution of the Flink job."); + } + } + private static final class Log extends org.apache.streampark.common.util.LoggerSupport { void warn(String msg) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java index 1688fbac22..d9006d5151 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java @@ -45,12 +45,19 @@ import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.streaming.api.scala.DataStream; import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment; +import org.apache.flink.table.api.CompiledPlan; import org.apache.flink.table.api.ExplainDetail; +import org.apache.flink.table.api.ExplainFormat; +import org.apache.flink.table.api.PlanReference; +import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableResult; import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment; import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.catalog.CatalogDescriptor; +import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.functions.AggregateFunction; import org.apache.flink.table.functions.ScalarFunction; @@ -58,7 +65,10 @@ import org.apache.flink.table.functions.TableFunction; import org.apache.flink.table.functions.UserDefinedFunction; import org.apache.flink.table.module.Module; +import org.apache.flink.table.module.ModuleEntry; +import org.apache.flink.table.resource.ResourceUri; import org.apache.flink.table.types.AbstractDataType; +import org.apache.flink.types.Row; import org.apache.flink.util.SplittableIterator; import com.esotericsoftware.kryo.Serializer; @@ -73,7 +83,10 @@ import scala.runtime.BoxedUnit; /** - * Integration api of stream and table + * Integration api of stream and table. + * + *

Methods prefixed with {@code $} implement the Flink Scala {@code StreamTableEnvironment} API + * and retain Scala-compatible names required by that interface. */ public abstract class FlinkStreamTableTrait implements StreamTableEnvironment { @@ -114,7 +127,6 @@ public JobExecutionResult start(String name) { return execute(appName); } - @Deprecated public JobExecutionResult execute(String jobName) { Utils.printLogo("FlinkStreamTable " + jobName + " Starting..."); if (isConvertedToDataStream) { @@ -366,7 +378,6 @@ public org.apache.flink.streaming.api.environment.StreamExecutionEnvironment get return streamEnv.isForceUnalignedCheckpoints(); } - @Deprecated public StreamExecutionEnvironment $enableCheckpointing( long interval, CheckpointingMode mode, @@ -374,17 +385,14 @@ public org.apache.flink.streaming.api.environment.StreamExecutionEnvironment get return streamEnv.enableCheckpointing(interval, mode, force); } - @Deprecated public StreamExecutionEnvironment $enableCheckpointing() { return streamEnv.enableCheckpointing(); } - @Deprecated public DataStream $generateSequence(long from, long to) { return streamEnv.generateSequence(from, to); } - @Deprecated public DataStream $readFileStream( String streamPath, long intervalMillis, @@ -392,7 +400,6 @@ public org.apache.flink.streaming.api.environment.StreamExecutionEnvironment get return streamEnv.readFileStream(streamPath, intervalMillis, watchType); } - @Deprecated public DataStream $readFile( FileInputFormat inputFormat, String filePath, @@ -413,6 +420,26 @@ public Table fromDataStream(DataStream dataStream, Seq fields return tableEnv.fromDataStream(dataStream, fields); } + @Override + public Table fromDataStream(DataStream dataStream, Schema schema) { + return tableEnv.fromDataStream(dataStream, schema); + } + + @Override + public Table fromChangelogStream(DataStream dataStream) { + return tableEnv.fromChangelogStream(dataStream); + } + + @Override + public Table fromChangelogStream(DataStream dataStream, Schema schema) { + return tableEnv.fromChangelogStream(dataStream, schema); + } + + @Override + public Table fromChangelogStream(DataStream dataStream, Schema schema, ChangelogMode changelogMode) { + return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); + } + @Override public void createTemporaryView(String path, DataStream dataStream) { tableEnv.createTemporaryView(path, dataStream); @@ -423,6 +450,47 @@ public void createTemporaryView(String path, DataStream dataStream, Seq void createTemporaryView(String path, DataStream dataStream, Schema schema) { + tableEnv.createTemporaryView(path, dataStream, schema); + } + + @Override + public DataStream toDataStream(Table table) { + isConvertedToDataStream = true; + return tableEnv.toDataStream(table); + } + + @Override + public DataStream toDataStream(Table table, Class targetClass) { + isConvertedToDataStream = true; + return tableEnv.toDataStream(table, targetClass); + } + + @Override + public DataStream toDataStream(Table table, AbstractDataType targetDataType) { + isConvertedToDataStream = true; + return tableEnv.toDataStream(table, targetDataType); + } + + @Override + public DataStream toChangelogStream(Table table) { + isConvertedToDataStream = true; + return tableEnv.toChangelogStream(table); + } + + @Override + public DataStream toChangelogStream(Table table, Schema targetSchema) { + isConvertedToDataStream = true; + return tableEnv.toChangelogStream(table, targetSchema); + } + + @Override + public DataStream toChangelogStream(Table table, Schema targetSchema, ChangelogMode changelogMode) { + isConvertedToDataStream = true; + return tableEnv.toChangelogStream(table, targetSchema, changelogMode); + } + @Override public DataStream toAppendStream(Table table, TypeInformation typeInfo) { isConvertedToDataStream = true; @@ -630,65 +698,127 @@ public TableConfig getConfig() { return tableEnv.getConfig(); } - @Deprecated @Override public void registerFunction(String name, TableFunction tf, TypeInformation typeInfo) { tableEnv.registerFunction(name, tf, typeInfo); } - @Deprecated @Override - public void registerFunction( - String name, - AggregateFunction f, - TypeInformation typeInfo1, - TypeInformation typeInfo2) { + public void registerFunction( + String name, + AggregateFunction f, + TypeInformation typeInfo1, + TypeInformation typeInfo2) { tableEnv.registerFunction(name, f, typeInfo1, typeInfo2); } - @Deprecated @Override - public void registerFunction( - String name, - TableAggregateFunction f, - TypeInformation typeInfo1, - TypeInformation typeInfo2) { + public void registerFunction( + String name, + TableAggregateFunction f, + TypeInformation typeInfo1, + TypeInformation typeInfo2) { tableEnv.registerFunction(name, f, typeInfo1, typeInfo2); } - @Deprecated @Override public void registerDataStream(String name, DataStream dataStream) { tableEnv.registerDataStream(name, dataStream); } - @Deprecated @Override public void registerDataStream(String name, DataStream dataStream, Seq fields) { tableEnv.registerDataStream(name, dataStream, fields); } - @Deprecated @Override public void registerFunction(String name, ScalarFunction function) { tableEnv.registerFunction(name, function); } - @Deprecated @Override public void registerTable(String name, Table table) { tableEnv.registerTable(name, table); } - @Deprecated @Override public Table scan(String... tablePath) { return tableEnv.scan(tablePath); } - @Deprecated @Override public String[] getCompletionHints(String statement, int position) { return tableEnv.getCompletionHints(statement, position); } + + @Override + public void useModules(String... modules) { + tableEnv.useModules(modules); + } + + @Override + public ModuleEntry[] listFullModules() { + return tableEnv.listFullModules(); + } + + @Override + public void createTemporaryTable(String path, TableDescriptor descriptor) { + tableEnv.createTemporaryTable(path, descriptor); + } + + @Override + public void createTable(String path, TableDescriptor descriptor) { + tableEnv.createTable(path, descriptor); + } + + @Override + public Table from(TableDescriptor descriptor) { + return tableEnv.from(descriptor); + } + + @Override + public String[] listTables(String catalogName, String databaseName) { + return tableEnv.listTables(catalogName, databaseName); + } + + @Override + public CompiledPlan loadPlan(PlanReference planReference) { + return tableEnv.loadPlan(planReference); + } + + @Override + public CompiledPlan compilePlanSql(String stmt) { + return tableEnv.compilePlanSql(stmt); + } + + @Override + public void createFunction(String path, String className, List resourceUris) { + tableEnv.createFunction(path, className, resourceUris); + } + + @Override + public void createFunction( + String path, String className, List resourceUris, boolean ignoreIfExists) { + tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); + } + + @Override + public void createTemporaryFunction(String path, String className, List resourceUris) { + tableEnv.createTemporaryFunction(path, className, resourceUris); + } + + @Override + public void createTemporarySystemFunction(String name, String className, List resourceUris) { + tableEnv.createTemporarySystemFunction(name, className, resourceUris); + } + + @Override + public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { + return tableEnv.explainSql(statement, format, extraDetails); + } + + @Override + public void createCatalog(String catalog, CatalogDescriptor catalogDescriptor) { + tableEnv.createCatalog(catalog, catalogDescriptor); + } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java index 92b163b1a7..94f8722325 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java @@ -115,55 +115,58 @@ Map parseConfig(String config) { String format = config.contains(".") ? config.substring(config.lastIndexOf('.') + 1).toLowerCase() : ""; - java.util.function.Function> readConfig = - text -> { - switch (format) { - case "yml": - case "yaml": - return PropertiesUtils.fromYamlText(text); - case "conf": - return PropertiesUtils.fromHoconText(text); - case "properties": - return PropertiesUtils.fromPropertiesText(text); - default: - throw new IllegalArgumentException( - "[StreamPark] Usage: application config file error,must be [yaml|conf|properties]"); - } - }; + Map map = readConfigContent(config, format); + Map filtered = new HashMap<>(); + map.forEach((key, value) -> { + if (value != null && !value.isEmpty()) { + filtered.put(key, value); + } + }); + return filtered; + } - Map map; + private Map readConfigContent(String config, String format) { if (config.startsWith("yaml://")) { - map = PropertiesUtils.fromYamlText(DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("conf://")) { - map = PropertiesUtils.fromHoconText(DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("prop://")) { - map = PropertiesUtils.fromPropertiesText(DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("hdfs://")) { + return readConfigText(format, DeflaterUtils.unzipString(config.substring(7))); + } + if (config.startsWith("conf://")) { + return readConfigText(format, DeflaterUtils.unzipString(config.substring(7))); + } + if (config.startsWith("prop://")) { + return readConfigText(format, DeflaterUtils.unzipString(config.substring(7))); + } + if (config.startsWith("hdfs://")) { try { - map = readConfig.apply(HdfsUtils.read(config)); + return readConfigText(format, HdfsUtils.read(config)); } catch (java.io.IOException e) { throw new IllegalStateException("Failed to read HDFS config: " + config, e); } - } else { - File configFile = new File(config); - if (!configFile.exists()) { - throw new IllegalArgumentException( - "[StreamPark] Usage: application config file: " + configFile + " is not found!!!"); - } - try { - map = readConfig.apply(FileUtils.readFile(configFile)); - } catch (java.io.IOException e) { - throw new IllegalStateException("Failed to read config file: " + configFile, e); - } } + File configFile = new File(config); + if (!configFile.exists()) { + throw new IllegalArgumentException( + "[StreamPark] Usage: application config file: " + configFile + " is not found!!!"); + } + try { + return readConfigText(format, FileUtils.readFile(configFile)); + } catch (java.io.IOException e) { + throw new IllegalStateException("Failed to read config file: " + configFile, e); + } + } - Map filtered = new HashMap<>(); - map.forEach((key, value) -> { - if (value != null && !value.isEmpty()) { - filtered.put(key, value); - } - }); - return filtered; + private Map readConfigText(String format, String text) { + switch (format) { + case "yml": + case "yaml": + return PropertiesUtils.fromYamlText(text); + case "conf": + return PropertiesUtils.fromHoconText(text); + case "properties": + return PropertiesUtils.fromPropertiesText(text); + default: + throw new IllegalArgumentException( + "[StreamPark] Usage: application config file error,must be [yaml|conf|properties]"); + } } Map extractConfigByPrefix(Map configMap, String prefix) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java index 60db5517f6..c8d47444b8 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java @@ -21,19 +21,27 @@ import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.table.api.CompiledPlan; import org.apache.flink.table.api.ExplainDetail; +import org.apache.flink.table.api.ExplainFormat; +import org.apache.flink.table.api.PlanReference; import org.apache.flink.table.api.StatementSet; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableResult; import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.catalog.CatalogDescriptor; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.functions.UserDefinedFunction; import org.apache.flink.table.module.Module; +import org.apache.flink.table.module.ModuleEntry; +import org.apache.flink.table.resource.ResourceUri; import org.apache.flink.table.types.AbstractDataType; +import java.util.List; import java.util.Optional; public abstract class FlinkTableTrait implements TableEnvironment { @@ -52,10 +60,10 @@ protected TableEnvironment delegate() { public JobExecutionResult start() { String appName = FlinkEnvironmentUtils.getAppName(parameter, null, true); - return execute(appName); + return printStartupLogo(appName); } - public JobExecutionResult execute(String jobName) { + JobExecutionResult printStartupLogo(String jobName) { Utils.printLogo("FlinkTable " + jobName + " Starting..."); return null; } @@ -267,27 +275,94 @@ public StatementSet createStatementSet() { return delegate().createStatementSet(); } - @Deprecated + @Override + public void useModules(String... modules) { + delegate().useModules(modules); + } + + @Override + public ModuleEntry[] listFullModules() { + return delegate().listFullModules(); + } + + @Override + public void createTemporaryTable(String path, TableDescriptor descriptor) { + delegate().createTemporaryTable(path, descriptor); + } + + @Override + public void createTable(String path, TableDescriptor descriptor) { + delegate().createTable(path, descriptor); + } + + @Override + public Table from(TableDescriptor descriptor) { + return delegate().from(descriptor); + } + @Override public void registerFunction(String name, ScalarFunction function) { delegate().registerFunction(name, function); } - @Deprecated @Override public void registerTable(String name, Table table) { delegate().registerTable(name, table); } - @Deprecated @Override public Table scan(String... tablePath) { return delegate().scan(tablePath); } - @Deprecated @Override public String[] getCompletionHints(String statement, int position) { return delegate().getCompletionHints(statement, position); } + + @Override + public String[] listTables(String catalogName, String databaseName) { + return delegate().listTables(catalogName, databaseName); + } + + @Override + public CompiledPlan loadPlan(PlanReference planReference) { + return delegate().loadPlan(planReference); + } + + @Override + public CompiledPlan compilePlanSql(String stmt) { + return delegate().compilePlanSql(stmt); + } + + @Override + public void createFunction(String path, String className, List resourceUris) { + delegate().createFunction(path, className, resourceUris); + } + + @Override + public void createFunction( + String path, String className, List resourceUris, boolean ignoreIfExists) { + delegate().createFunction(path, className, resourceUris, ignoreIfExists); + } + + @Override + public void createTemporaryFunction(String path, String className, List resourceUris) { + delegate().createTemporaryFunction(path, className, resourceUris); + } + + @Override + public void createTemporarySystemFunction(String name, String className, List resourceUris) { + delegate().createTemporarySystemFunction(name, className, resourceUris); + } + + @Override + public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { + return delegate().explainSql(statement, format, extraDetails); + } + + @Override + public void createCatalog(String catalog, CatalogDescriptor catalogDescriptor) { + delegate().createCatalog(catalog, catalogDescriptor); + } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandConverters.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandConverters.java index ebd8bb9635..b787ba9b69 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandConverters.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandConverters.java @@ -26,6 +26,9 @@ private SqlCommandConverters() { } static Optional noOperands(String[] groups) { + if (groups == null) { + return Optional.empty(); + } return Optional.of(new String[0]); } @@ -44,6 +47,9 @@ static Optional setOperands(String[] groups) { } static Optional resetAll(String[] groups) { + if (groups == null) { + return Optional.empty(); + } return Optional.of(new String[]{"ALL"}); } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java index 68cb5225f4..a58bbf8e6e 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.Consumer; @@ -51,7 +52,7 @@ public static List parseSQL( .withSuccess(false) .withFailedType(FlinkSqlValidationFailedType.VERIFY_FAILED) .withException(sqlEmptyError)); - return null; + return Collections.emptyList(); } throw new IllegalArgumentException(sqlEmptyError); } @@ -70,7 +71,7 @@ public static List parseSQL( .withLineEnd(segment.end()) .withException("unsupported sql") .withSql(segment.sql())); - return null; + return Collections.emptyList(); } else { throw new UnsupportedOperationException("unsupported sql: " + segment.sql()); } @@ -83,7 +84,7 @@ public static List parseSQL( .withSuccess(false) .withFailedType(FlinkSqlValidationFailedType.VERIFY_FAILED) .withException("flink sql syntax error, no executable sql")); - return null; + return Collections.emptyList(); } throw new UnsupportedOperationException("flink sql syntax error, no executable sql"); } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java index 0aec230e69..af2c8c1267 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java @@ -46,163 +46,40 @@ private SqlSplitter() { } static List splitSql(String sql) { - List queries = new ArrayList<>(); - int lastIndex = StringUtils.isNotBlank(sql) ? sql.length() - 1 : 0; - StringBuilder query = new StringBuilder(); - - boolean multiLineComment = false; - boolean singleLineComment = false; - boolean singleQuoteString = false; - boolean doubleQuoteString = false; - int lineNum = 0; - Map lineNumMap = new HashMap<>(); - - Map lineDescriptor = buildLineDescriptor(sql); - - for (int idx = 0; idx < sql.length(); idx++) { - if (sql.charAt(idx) == '\n') { - lineNum++; - } - - char ch = sql.charAt(idx); - - if (singleLineComment && ch == '\n') { - singleLineComment = false; - query.append(ch); - if (idx == lastIndex && query.toString().trim().length() > 0) { - queries.add(query.toString()); - } - continue; - } - - if (multiLineComment && idx - 1 >= 0 && sql.charAt(idx - 1) == '/' - && idx - 2 >= 0 && sql.charAt(idx - 2) == '*') { - multiLineComment = false; - } - - if (ch == '\'' && !singleLineComment && !multiLineComment) { - if (singleQuoteString) { - singleQuoteString = false; - } else if (!doubleQuoteString) { - singleQuoteString = true; - } - } - - if (ch == '"' && !singleLineComment && !multiLineComment) { - if (doubleQuoteString && idx > 0) { - doubleQuoteString = false; - } else if (!singleQuoteString) { - doubleQuoteString = true; - } - } - - if (!singleQuoteString && !doubleQuoteString && !multiLineComment && !singleLineComment - && idx < lastIndex) { - if (isSingleLineComment(sql.charAt(idx), sql.charAt(idx + 1))) { - singleLineComment = true; - } else if (sql.charAt(idx) == '/' - && sql.length() > idx + 2 - && sql.charAt(idx + 1) == '*' - && sql.charAt(idx + 2) != '+') { - multiLineComment = true; - } - } - - if (ch == ';' && !singleQuoteString && !doubleQuoteString && !multiLineComment - && !singleLineComment) { - markLineNumber(lineNum, lineNumMap, lineDescriptor); - if (query.toString().trim().length() > 0) { - queries.add(query.toString()); - query = new StringBuilder(); - } - } else if (idx == lastIndex) { - markLineNumber(lineNum, lineNumMap, lineDescriptor); - if (!singleLineComment && !multiLineComment) { - query.append(ch); - } - if (query.toString().trim().length() > 0) { - queries.add(query.toString()); - } - } else if (!singleLineComment && !multiLineComment) { - query.append(ch); - } else if (ch == '\n') { - query.append(ch); - } - } + QueryExtractor extractor = new QueryExtractor(sql); + Map refinedQueries = refineQueries(extractor.queries()); + return buildSegments(refinedQueries, extractor.lineNumMap()); + } + private static Map refineQueries(List queries) { Map refinedQueries = new HashMap<>(); for (int i = 0; i < queries.size(); i++) { String currStatement = queries.get(i); if (isSingleLineComment(currStatement) || isMultipleLineComment(currStatement)) { - if (!refinedQueries.isEmpty()) { - int lastKey = refinedQueries.size() - 1; - refinedQueries.put( - lastKey, - refinedQueries.get(lastKey) + extractLineBreaks(currStatement)); - } + mergeCommentIntoPrevious(refinedQueries, currStatement); } else { - String linesPlaceholder = ""; - if (i > 0) { - linesPlaceholder = extractLineBreaks(refinedQueries.get(i - 1)); - } - refinedQueries.put(refinedQueries.size(), linesPlaceholder + currStatement); + appendStatement(refinedQueries, i, currStatement); } } - - List segments = new ArrayList<>(); - for (Map.Entry entry : refinedQueries.entrySet()) { - int[] line = lineNumMap.get(entry.getKey()); - segments.add(new SqlSegment(line[0], line[1], entry.getValue())); - } - segments.sort(Comparator.comparingInt(SqlSegment::start)); - return segments; - } - - private static Map buildLineDescriptor(String sql) { - Map descriptor = new HashMap<>(); - Scanner scanner = new Scanner(sql); - int lineNumber = 0; - boolean startComment = false; - boolean hasComment = false; - - while (scanner.hasNextLine()) { - lineNumber++; - String line = scanner.nextLine().trim(); - boolean nonEmpty = - StringUtils.isNotBlank(line) && !line.startsWith(ConfigKeys.PARAM_PREFIX()); - if (line.startsWith("/*")) { - startComment = true; - hasComment = true; - } - descriptor.put(lineNumber, nonEmpty && !hasComment); - if (startComment && line.endsWith("*/")) { - startComment = false; - hasComment = false; - } - } - scanner.close(); - return descriptor; + return refinedQueries; } - private static int findStartLine(int num, Map lineDescriptor) { - if (num >= lineDescriptor.size() || Boolean.TRUE.equals(lineDescriptor.get(num))) { - return num; + private static void mergeCommentIntoPrevious(Map refinedQueries, String comment) { + if (!refinedQueries.isEmpty()) { + int lastKey = refinedQueries.size() - 1; + refinedQueries.put(lastKey, refinedQueries.get(lastKey) + extractLineBreaks(comment)); } - return findStartLine(num + 1, lineDescriptor); } - private static void markLineNumber( - int lineNum, - Map lineNumMap, - Map lineDescriptor) { - int line = lineNum + 1; - if (lineNumMap.isEmpty()) { - lineNumMap.put(0, new int[]{findStartLine(1, lineDescriptor), line}); - } else { - int index = lineNumMap.size(); - int start = lineNumMap.get(lineNumMap.size() - 1)[1] + 1; - lineNumMap.put(index, new int[]{findStartLine(start, lineDescriptor), line}); + private static void appendStatement( + Map refinedQueries, + int index, + String statement) { + String linesPlaceholder = ""; + if (index > 0) { + linesPlaceholder = extractLineBreaks(refinedQueries.get(index - 1)); } + refinedQueries.put(refinedQueries.size(), linesPlaceholder + statement); } private static String extractLineBreaks(String text) { @@ -215,6 +92,18 @@ private static String extractLineBreaks(String text) { return builder.toString(); } + private static List buildSegments( + Map refinedQueries, + Map lineNumMap) { + List segments = new ArrayList<>(); + for (Map.Entry entry : refinedQueries.entrySet()) { + int[] line = lineNumMap.get(entry.getKey()); + segments.add(new SqlSegment(line[0], line[1], entry.getValue())); + } + segments.sort(Comparator.comparingInt(SqlSegment::start)); + return segments; + } + private static boolean isSingleLineComment(String text) { return text.trim().startsWith(ConfigKeys.PARAM_PREFIX()); } @@ -236,4 +125,186 @@ private static boolean isSingleLineComment(char curChar, char nextChar) { } return false; } + + private static final class QueryExtractor { + + private final String sql; + private final int lastIndex; + private final Map lineNumMap = new HashMap<>(); + private final Map lineDescriptor; + private final List queries = new ArrayList<>(); + + private final StringBuilder query = new StringBuilder(); + private boolean multiLineComment; + private boolean singleLineComment; + private boolean singleQuoteString; + private boolean doubleQuoteString; + private int lineNum; + + private QueryExtractor(String sql) { + this.sql = sql; + this.lastIndex = StringUtils.isNotBlank(sql) ? sql.length() - 1 : 0; + this.lineDescriptor = buildLineDescriptor(sql); + for (int idx = 0; idx < sql.length(); idx++) { + processCharacter(idx); + } + } + + private List queries() { + return queries; + } + + private Map lineNumMap() { + return lineNumMap; + } + + private static Map buildLineDescriptor(String sql) { + Map descriptor = new HashMap<>(); + Scanner scanner = new Scanner(sql); + int lineNumber = 0; + boolean startComment = false; + boolean hasComment = false; + + while (scanner.hasNextLine()) { + lineNumber++; + String line = scanner.nextLine().trim(); + boolean nonEmpty = + StringUtils.isNotBlank(line) && !line.startsWith(ConfigKeys.PARAM_PREFIX()); + if (line.startsWith("/*")) { + startComment = true; + hasComment = true; + } + descriptor.put(lineNumber, nonEmpty && !hasComment); + if (startComment && line.endsWith("*/")) { + startComment = false; + hasComment = false; + } + } + scanner.close(); + return descriptor; + } + + private static int findStartLine(int num, Map lineDescriptor) { + if (num >= lineDescriptor.size() || Boolean.TRUE.equals(lineDescriptor.get(num))) { + return num; + } + return findStartLine(num + 1, lineDescriptor); + } + + private static void markLineNumber( + int lineNum, + Map lineNumMap, + Map lineDescriptor) { + int line = lineNum + 1; + if (lineNumMap.isEmpty()) { + lineNumMap.put(0, new int[]{findStartLine(1, lineDescriptor), line}); + } else { + int index = lineNumMap.size(); + int start = lineNumMap.get(lineNumMap.size() - 1)[1] + 1; + lineNumMap.put(index, new int[]{findStartLine(start, lineDescriptor), line}); + } + } + + private static boolean hasNonBlankQuery(StringBuilder query) { + return !query.toString().trim().isEmpty(); + } + + private void processCharacter(int idx) { + if (sql.charAt(idx) == '\n') { + lineNum++; + } + char ch = sql.charAt(idx); + if (handleSingleLineCommentEnd(idx, ch)) { + return; + } + updateMultiLineCommentEnd(idx); + updateQuoteState(idx, ch); + updateCommentStart(idx); + handleStatementBoundary(idx, ch); + } + + private boolean handleSingleLineCommentEnd(int idx, char ch) { + if (!singleLineComment || ch != '\n') { + return false; + } + singleLineComment = false; + query.append(ch); + if (idx == lastIndex && hasNonBlankQuery(query)) { + queries.add(query.toString()); + } + return true; + } + + private void updateMultiLineCommentEnd(int idx) { + if (multiLineComment && idx - 1 >= 0 && sql.charAt(idx - 1) == '/' + && idx - 2 >= 0 && sql.charAt(idx - 2) == '*') { + multiLineComment = false; + } + } + + private void updateQuoteState(int idx, char ch) { + if (ch == '\'' && !singleLineComment && !multiLineComment) { + if (singleQuoteString) { + singleQuoteString = false; + } else if (!doubleQuoteString) { + singleQuoteString = true; + } + } + if (ch == '"' && !singleLineComment && !multiLineComment) { + if (doubleQuoteString && idx > 0) { + doubleQuoteString = false; + } else if (!singleQuoteString) { + doubleQuoteString = true; + } + } + } + + private void updateCommentStart(int idx) { + if (singleQuoteString || doubleQuoteString || multiLineComment || singleLineComment + || idx >= lastIndex) { + return; + } + if (isSingleLineComment(sql.charAt(idx), sql.charAt(idx + 1))) { + singleLineComment = true; + } else if (sql.charAt(idx) == '/' + && sql.length() > idx + 2 + && sql.charAt(idx + 1) == '*' + && sql.charAt(idx + 2) != '+') { + multiLineComment = true; + } + } + + private void handleStatementBoundary(int idx, char ch) { + if (ch == ';' && !singleQuoteString && !doubleQuoteString && !multiLineComment + && !singleLineComment) { + markLineNumber(lineNum, lineNumMap, lineDescriptor); + if (hasNonBlankQuery(query)) { + queries.add(query.toString()); + query.setLength(0); + } + return; + } + if (idx == lastIndex) { + handleLastCharacter(ch); + return; + } + appendNonCommentCharacter(ch); + } + + private void handleLastCharacter(char ch) { + markLineNumber(lineNum, lineNumMap, lineDescriptor); + if (!singleLineComment && !multiLineComment) { + query.append(ch); + } + if (hasNonBlankQuery(query)) { + queries.add(query.toString()); + } + } + + private void appendNonCommentCharacter(char ch) { + if (!singleLineComment && !multiLineComment || ch == '\n') { + query.append(ch); + } + } + } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkRunOption.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkRunOption.java index 3a40107063..5f838536fa 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkRunOption.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkRunOption.java @@ -26,6 +26,8 @@ /** Applies to all optional parameters under flink run */ public final class FlinkRunOption { + private static final String SAVEPOINT_PATH = "savepointPath"; + public static final Option HELP_OPTION = new Option("h", "help", false, null); public static final Option JAR_OPTION = new Option("j", "jarfile", true, null); public static final Option CLASS_OPTION = new Option("c", "class", true, null); @@ -49,7 +51,7 @@ public final class FlinkRunOption { public static final Option CANCEL_WITH_SAVEPOINT_OPTION = new Option("s", "withSavepoint", true, null); public static final Option STOP_WITH_SAVEPOINT_PATH = - new Option("p", "savepointPath", true, null); + new Option("p", SAVEPOINT_PATH, true, null); public static final Option STOP_AND_DRAIN = new Option("d", "drain", false, null); public static final Option PY_OPTION = new Option("py", "python", true, null); public static final Option PYFILES_OPTION = new Option("pyfs", "pyFiles", true, null); @@ -95,7 +97,7 @@ public final class FlinkRunOption { SCHEDULED_OPTION.setRequired(false); SAVEPOINT_PATH_OPTION.setRequired(false); - SAVEPOINT_PATH_OPTION.setArgName("savepointPath"); + SAVEPOINT_PATH_OPTION.setArgName(SAVEPOINT_PATH); SAVEPOINT_ALLOW_NON_RESTORED_OPTION.setRequired(false); @@ -107,7 +109,7 @@ public final class FlinkRunOption { CANCEL_WITH_SAVEPOINT_OPTION.setOptionalArg(true); STOP_WITH_SAVEPOINT_PATH.setRequired(false); - STOP_WITH_SAVEPOINT_PATH.setArgName("savepointPath"); + STOP_WITH_SAVEPOINT_PATH.setArgName(SAVEPOINT_PATH); STOP_WITH_SAVEPOINT_PATH.setOptionalArg(true); STOP_AND_DRAIN.setRequired(false); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java index 74946a6bb7..6b03a2886c 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java @@ -25,9 +25,9 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import java.io.PrintStream; import java.net.URLClassLoader; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -46,37 +46,39 @@ private ParameterCli() { } public static void main(String[] args) { - System.out.print(read(args)); + emit(read(args), System.out); + } + + static void emit(String output, PrintStream out) { + out.print(output); } public static String read(String[] args) { - switch (args[0]) { - case "--vmopt": - ClassLoader loader = ClassLoader.getSystemClassLoader(); - if (loader instanceof URLClassLoader) { - return ""; - } - return "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED " - + "--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED"; + if ("--vmopt".equals(args[0])) { + ClassLoader loader = ClassLoader.getSystemClassLoader(); + if (loader instanceof URLClassLoader) { + return ""; + } + return "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED " + + "--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED"; + } + String action = args[0]; + String conf = args[1]; + Map map = loadConfig(conf); + String[] programArgs = new String[args.length - 2]; + System.arraycopy(args, 2, programArgs, 0, programArgs.length); + switch (action) { + case "--option": + return buildOption(map, programArgs); + case "--property": + return buildProperty(map); + case "--name": + return map.getOrDefault( + PROPERTY_PREFIX + ConfigKeys.KEY_FLINK_APP_NAME(), "").trim(); + case "--detached": + return buildDetachedMode(map, programArgs); default: - String action = args[0]; - String conf = args[1]; - Map map = loadConfig(conf); - String[] programArgs = new String[args.length - 2]; - System.arraycopy(args, 2, programArgs, 0, programArgs.length); - switch (action) { - case "--option": - return buildOption(map, programArgs); - case "--property": - return buildProperty(map); - case "--name": - return map.getOrDefault( - PROPERTY_PREFIX + ConfigKeys.KEY_FLINK_APP_NAME(), "").trim(); - case "--detached": - return buildDetachedMode(map, programArgs); - default: - return null; - } + return null; } } @@ -96,7 +98,8 @@ private static Map loadConfig(String conf) { "[StreamPark] Usage:flink.conf file error,must be (yml|conf|properties)"); } } catch (Exception e) { - return Collections.emptyMap(); + throw new IllegalArgumentException( + "[StreamPark] Failed to load flink config file: " + conf, e); } } @@ -112,7 +115,7 @@ private static String buildOption(Map map, String[] programArgs) } } } catch (ParseException exception) { - exception.printStackTrace(); + // Ignore unrecognized CLI tokens; valid options are still collected below. } String mainClass = map.get(OPTION_MAIN); if (mainClass != null) { @@ -151,12 +154,18 @@ private static String buildDetachedMode(Map map, String[] progra || line.hasOption(FlinkRunOption.DETACHED_OPTION.getLongOpt()); return detached ? "Detached" : "Attach"; } catch (ParseException e) { - throw new RuntimeException(e); + throw new IllegalArgumentException("Failed to parse Flink detached mode options", e); } } public static String[] getOption(Map map, String[] args) { Map optionMap = new LinkedHashMap<>(); + mergeConfigOptions(map, optionMap); + mergeProgramOptions(args, optionMap); + return flattenOptions(optionMap); + } + + private static void mergeConfigOptions(Map map, Map optionMap) { map.entrySet().stream() .filter(x -> x.getKey().startsWith(OPTION_PREFIX)) .filter(x -> x.getValue() != null && !x.getValue().isEmpty()) @@ -175,22 +184,27 @@ public static String[] getOption(Map map, String[] args) { optionMap.put(optKey, value); } }); + } - if (args.length > 0) { - try { - CommandLine line = PARSER.parse(FLINK_OPTIONS, args, false); - for (org.apache.commons.cli.Option x : line.getOptions()) { - if (x.hasArg()) { - optionMap.put("-" + x.getLongOpt().trim(), x.getValue()); - } else { - optionMap.put("-" + x.getLongOpt().trim(), true); - } + private static void mergeProgramOptions(String[] args, Map optionMap) { + if (args.length == 0) { + return; + } + try { + CommandLine line = PARSER.parse(FLINK_OPTIONS, args, false); + for (org.apache.commons.cli.Option x : line.getOptions()) { + if (x.hasArg()) { + optionMap.put("-" + x.getLongOpt().trim(), x.getValue()); + } else { + optionMap.put("-" + x.getLongOpt().trim(), true); } - } catch (ParseException e) { - e.printStackTrace(); } + } catch (ParseException e) { + // Ignore unrecognized CLI tokens merged from program arguments. } + } + private static String[] flattenOptions(Map optionMap) { List array = new ArrayList<>(); optionMap.forEach( (key, value) -> { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/conf/ParameterCliTest.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/conf/ParameterCliTest.java new file mode 100644 index 0000000000..dfc508cf6e --- /dev/null +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/conf/ParameterCliTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.core.conf; + +import org.apache.streampark.common.conf.ConfigKeys; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class ParameterCliTest { + + @TempDir + Path tempDir; + + @Test + void readVmOptForNonUrlClassLoader() { + String result = ParameterCli.read(new String[]{"--vmopt"}); + assertThat(result).contains("--add-opens"); + } + + @Test + void readAppNameFromPropertiesFile() throws Exception { + Path conf = + tempDir.resolve( + "app.properties"); + Files.writeString( + conf, + ConfigKeys.KEY_FLINK_PROPERTY_PREFIX() + + ConfigKeys.KEY_FLINK_APP_NAME() + + "=demo_app\n"); + + String name = ParameterCli.read(new String[]{"--name", conf.toString()}); + assertThat(name).isEqualTo("demo_app"); + } + + @Test + void readPropertyFromYamlFile() throws Exception { + Path conf = tempDir.resolve("app.yml"); + Files.writeString( + conf, + ConfigKeys.KEY_FLINK_PROPERTY_PREFIX() + + ConfigKeys.KEY_FLINK_APP_NAME() + + ": demo_yaml\n"); + + String property = ParameterCli.read(new String[]{"--property", conf.toString()}); + assertThat(property).contains("-D" + ConfigKeys.KEY_FLINK_APP_NAME() + "=demo_yaml"); + } + + @Test + void emitWritesToProvidedStream() { + java.io.ByteArrayOutputStream buffer = new java.io.ByteArrayOutputStream(); + ParameterCli.emit("hello", new java.io.PrintStream(buffer)); + assertThat(buffer.toString()).isEqualTo("hello"); + } +} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java index 7b3a85dcc6..b718de2391 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java @@ -27,11 +27,10 @@ import java.util.ArrayList; import java.util.List; -import java.util.function.Consumer; class FlinkSqlExecuteTest { - private void execute(String sql, Consumer callback) { + private void execute(String sql) { List args = new ArrayList<>(); args.add(ConfigKeys.KEY_FLINK_SQL(ConfigKeys.PARAM_PREFIX())); args.add(DeflaterUtils.zipString(sql)); @@ -73,7 +72,6 @@ void execute() { + "insert into sink_kafka1\n" + "select id, name, age\n" + "from source_kafka1;\n" - + "\n", - System.out::println); + + "\n"); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index fe8d21c999..29a6093463 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -20,25 +20,23 @@ import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment; +import org.apache.flink.table.api.StatementSet; +import org.apache.flink.table.api.Table; import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment; +import org.apache.flink.table.descriptors.ConnectorDescriptor; +import org.apache.flink.table.descriptors.StreamTableDescriptor; +import org.apache.flink.table.sources.TableSource; import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -50,66 +48,56 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Deprecated @Override - public org.apache.flink.table.descriptors.StreamTableDescriptor connect( - org.apache.flink.table.descriptors.ConnectorDescriptor connectorDescriptor) { - return tableEnv.connect(connectorDescriptor); + public StreamTableDescriptor connect(ConnectorDescriptor connectorDescriptor) { + return tableEnv().connect(connectorDescriptor); } public StreamGraph $getStreamGraph(String jobName) { - return streamEnv.getStreamGraph(jobName); + return streamEnv().getStreamGraph(jobName); } public StreamGraph $getStreamGraph(String jobName, boolean clearTransformations) { - return streamEnv.getStreamGraph(jobName, clearTransformations); + return streamEnv().getStreamGraph(jobName, clearTransformations); } @Override - public org.apache.flink.table.api.StatementSet createStatementSet() { - return tableEnv.createStatementSet(); + public StatementSet createStatementSet() { + return tableEnv().createStatementSet(); } - @Deprecated @Override - public org.apache.flink.table.api.Table fromTableSource(org.apache.flink.table.sources.TableSource source) { - return tableEnv.fromTableSource(source); + public Table fromTableSource(TableSource source) { + return tableEnv().fromTableSource(source); } - @Deprecated @Override - public void insertInto(org.apache.flink.table.api.Table table, String sinkPath, String... sinkPathContinued) { - tableEnv.insertInto(table, sinkPath, sinkPathContinued); + public void insertInto(Table table, String sinkPath, String... sinkPathContinued) { + tableEnv().insertInto(table, sinkPath, sinkPathContinued); } - @Deprecated @Override - public void insertInto(String targetPath, org.apache.flink.table.api.Table table) { - tableEnv.insertInto(targetPath, table); + public void insertInto(String targetPath, Table table) { + tableEnv().insertInto(targetPath, table); } - @Deprecated @Override - public String explain(org.apache.flink.table.api.Table table) { - return tableEnv.explain(table); + public String explain(Table table) { + return tableEnv().explain(table); } - @Deprecated @Override - public String explain(org.apache.flink.table.api.Table table, boolean extended) { - return tableEnv.explain(table, extended); + public String explain(Table table, boolean extended) { + return tableEnv().explain(table, extended); } - @Deprecated @Override public String explain(boolean extended) { - return tableEnv.explain(extended); + return tableEnv().explain(extended); } - @Deprecated @Override public void sqlUpdate(String stmt) { - tableEnv.sqlUpdate(stmt); + tableEnv().sqlUpdate(stmt); } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableContext.java index d777fe3428..8aebb0f929 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,8 +17,6 @@ package org.apache.streampark.flink.core; -import org.apache.streampark.common.util.Utils; - import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.table.api.Table; @@ -43,7 +41,6 @@ public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Deprecated @Override public ConnectTableDescriptor connect(ConnectorDescriptor connectorDescriptor) { return delegate().connect(connectorDescriptor); @@ -51,50 +48,41 @@ public ConnectTableDescriptor connect(ConnectorDescriptor connectorDescriptor) { @Override public JobExecutionResult execute(String jobName) { - Utils.printLogo("FlinkTable " + jobName + " Starting..."); - return null; + return printStartupLogo(jobName); } - @Deprecated @Override public Table fromTableSource(TableSource source) { return delegate().fromTableSource(source); } - @Deprecated @Override public void insertInto(Table table, String sinkPath, String... sinkPathContinued) { delegate().insertInto(table, sinkPath, sinkPathContinued); } - @Deprecated @Override public void insertInto(String targetPath, Table table) { delegate().insertInto(targetPath, table); } - @Deprecated @Override public String explain(Table table) { return delegate().explain(table); } - @Deprecated @Override public String explain(Table table, boolean extended) { return delegate().explain(table, extended); } - @Deprecated @Override public String explain(boolean extended) { return delegate().explain(extended); } - @Deprecated @Override public void sqlUpdate(String stmt) { delegate().sqlUpdate(stmt); } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableExt.java index 452f234624..61f029ef22 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.12/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { @@ -73,15 +71,13 @@ public DataStream> retractStream( } public DataStream toAppendStream(StreamTableContext context, TypeInformation typeInfo) { - context.isConvertedToDataStream = true; - return super.toAppendStream(typeInfo); + return appendStream(context, typeInfo); } public DataStream> toRetractStream( StreamTableContext context, TypeInformation typeInfo) { - context.isConvertedToDataStream = true; - return super.toRetractStream(typeInfo); + return retractStream(context, typeInfo); } public TableConversions(Table table) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index f3cbd88166..29a6093463 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -20,25 +20,23 @@ import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment; +import org.apache.flink.table.api.StatementSet; +import org.apache.flink.table.api.Table; import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment; +import org.apache.flink.table.descriptors.ConnectorDescriptor; +import org.apache.flink.table.descriptors.StreamTableDescriptor; +import org.apache.flink.table.sources.TableSource; import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -51,151 +49,55 @@ public StreamTableContext(StreamTableEnvConfig args) { } @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); + public StreamTableDescriptor connect(ConnectorDescriptor connectorDescriptor) { + return tableEnv().connect(connectorDescriptor); } - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); + public StreamGraph $getStreamGraph(String jobName) { + return streamEnv().getStreamGraph(jobName); } - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); + public StreamGraph $getStreamGraph(String jobName, boolean clearTransformations) { + return streamEnv().getStreamGraph(jobName, clearTransformations); } @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); + public StatementSet createStatementSet() { + return tableEnv().createStatementSet(); } @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); + public Table fromTableSource(TableSource source) { + return tableEnv().fromTableSource(source); } @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); + public void insertInto(Table table, String sinkPath, String... sinkPathContinued) { + tableEnv().insertInto(table, sinkPath, sinkPathContinued); } @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); + public void insertInto(String targetPath, Table table) { + tableEnv().insertInto(targetPath, table); } @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); + public String explain(Table table) { + return tableEnv().explain(table); } @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); + public String explain(Table table, boolean extended) { + return tableEnv().explain(table, extended); } - @Deprecated @Override - public org.apache.flink.table.descriptors.StreamTableDescriptor connect( - org.apache.flink.table.descriptors.ConnectorDescriptor connectorDescriptor) { - return tableEnv.connect(connectorDescriptor); - } - - public StreamGraph $getStreamGraph(String jobName) { - return streamEnv.getStreamGraph(jobName); - } - - public StreamGraph $getStreamGraph(String jobName, boolean clearTransformations) { - return streamEnv.getStreamGraph(jobName, clearTransformations); - } - - @Override - public org.apache.flink.table.api.StatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Deprecated - public org.apache.flink.table.api.Table fromTableSource(org.apache.flink.table.sources.TableSource source) { - return tableEnv.fromTableSource(source); - } - - @Deprecated - public void insertInto(org.apache.flink.table.api.Table table, String sinkPath, String... sinkPathContinued) { - tableEnv.insertInto(table, sinkPath, sinkPathContinued); - } - - @Deprecated - public void insertInto(String targetPath, org.apache.flink.table.api.Table table) { - tableEnv.insertInto(targetPath, table); - } - - @Deprecated - public String explain(org.apache.flink.table.api.Table table) { - return tableEnv.explain(table); - } - - @Deprecated - public String explain(org.apache.flink.table.api.Table table, boolean extended) { - return tableEnv.explain(table, extended); - } - - @Deprecated public String explain(boolean extended) { - return tableEnv.explain(extended); + return tableEnv().explain(extended); } - @Deprecated + @Override public void sqlUpdate(String stmt) { - tableEnv.sqlUpdate(stmt); + tableEnv().sqlUpdate(stmt); } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableContext.java index 4d125cade4..8aebb0f929 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,15 +17,12 @@ package org.apache.streampark.flink.core; -import org.apache.streampark.common.util.Utils; - import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.descriptors.ConnectTableDescriptor; import org.apache.flink.table.descriptors.ConnectorDescriptor; -import org.apache.flink.table.module.ModuleEntry; import org.apache.flink.table.sources.TableSource; import scala.Tuple2; @@ -44,17 +41,6 @@ public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - @Deprecated @Override public ConnectTableDescriptor connect(ConnectorDescriptor connectorDescriptor) { return delegate().connect(connectorDescriptor); @@ -62,50 +48,41 @@ public ConnectTableDescriptor connect(ConnectorDescriptor connectorDescriptor) { @Override public JobExecutionResult execute(String jobName) { - Utils.printLogo("FlinkTable " + jobName + " Starting..."); - return null; + return printStartupLogo(jobName); } - @Deprecated @Override public Table fromTableSource(TableSource source) { return delegate().fromTableSource(source); } - @Deprecated @Override public void insertInto(Table table, String sinkPath, String... sinkPathContinued) { delegate().insertInto(table, sinkPath, sinkPathContinued); } - @Deprecated @Override public void insertInto(String targetPath, Table table) { delegate().insertInto(targetPath, table); } - @Deprecated @Override public String explain(Table table) { return delegate().explain(table); } - @Deprecated @Override public String explain(Table table, boolean extended) { return delegate().explain(table, extended); } - @Deprecated @Override public String explain(boolean extended) { return delegate().explain(extended); } - @Deprecated @Override public void sqlUpdate(String stmt) { delegate().sqlUpdate(stmt); } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableExt.java index 452f234624..61f029ef22 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.13/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { @@ -73,15 +71,13 @@ public DataStream> retractStream( } public DataStream toAppendStream(StreamTableContext context, TypeInformation typeInfo) { - context.isConvertedToDataStream = true; - return super.toAppendStream(typeInfo); + return appendStream(context, typeInfo); } public DataStream> toRetractStream( StreamTableContext context, TypeInformation typeInfo) { - context.isConvertedToDataStream = true; - return super.toRetractStream(typeInfo); + return retractStream(context, typeInfo); } public TableConversions(Table table) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 64628c5785..c56e29d830 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -20,25 +20,21 @@ import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.scala.StreamStatementSet; import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment; +import org.apache.flink.table.sources.TableSource; import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -50,153 +46,47 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); + public StreamGraph $getStreamGraph(boolean clearTransformations) { + return streamEnv().getStreamGraph(clearTransformations); } @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); + public StreamStatementSet createStatementSet() { + return tableEnv().createStatementSet(); } @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); + public Table fromTableSource(TableSource source) { + return tableEnv().fromTableSource(source); } @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); + public void insertInto(Table table, String sinkPath, String... sinkPathContinued) { + tableEnv().insertInto(table, sinkPath, sinkPathContinued); } @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); + public void insertInto(String targetPath, Table table) { + tableEnv().insertInto(targetPath, table); } @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); + public String explain(Table table) { + return tableEnv().explain(table); } @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); + public String explain(Table table, boolean extended) { + return tableEnv().explain(table, extended); } @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - - public void createTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - public void createTemporaryTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - public org.apache.flink.table.api.Table from(org.apache.flink.table.api.TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - public StreamGraph $getStreamGraph(boolean clearTransformations) { - return streamEnv.getStreamGraph(clearTransformations); - } - - @Override - public org.apache.flink.table.api.bridge.scala.StreamStatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Deprecated - public org.apache.flink.table.api.Table fromTableSource(org.apache.flink.table.sources.TableSource source) { - return tableEnv.fromTableSource(source); - } - - @Deprecated - public void insertInto(org.apache.flink.table.api.Table table, String sinkPath, String... sinkPathContinued) { - tableEnv.insertInto(table, sinkPath, sinkPathContinued); - } - - @Deprecated - public void insertInto(String targetPath, org.apache.flink.table.api.Table table) { - tableEnv.insertInto(targetPath, table); - } - - @Deprecated - public String explain(org.apache.flink.table.api.Table table) { - return tableEnv.explain(table); - } - - @Deprecated - public String explain(org.apache.flink.table.api.Table table, boolean extended) { - return tableEnv.explain(table, extended); - } - - @Deprecated public String explain(boolean extended) { - return tableEnv.explain(extended); + return tableEnv().explain(extended); } - @Deprecated + @Override public void sqlUpdate(String stmt) { - tableEnv.sqlUpdate(stmt); + tableEnv().sqlUpdate(stmt); } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableContext.java index ba5c6fc581..73606f7381 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,14 +17,10 @@ package org.apache.streampark.flink.core; -import org.apache.streampark.common.util.Utils; - import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.module.ModuleEntry; import org.apache.flink.table.sources.TableSource; import scala.Tuple2; @@ -43,75 +39,43 @@ public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - public void createTable(String path, TableDescriptor descriptor) { - delegate().createTable(path, descriptor); - } - - public void createTemporaryTable(String path, TableDescriptor descriptor) { - delegate().createTemporaryTable(path, descriptor); - } - - public Table from(TableDescriptor descriptor) { - return delegate().from(descriptor); - } - - @Deprecated @Override public JobExecutionResult execute(String jobName) { - Utils.printLogo("FlinkTable " + jobName + " Starting..."); - return null; + return printStartupLogo(jobName); } - @Deprecated @Override public Table fromTableSource(TableSource source) { return delegate().fromTableSource(source); } - @Deprecated @Override public void insertInto(Table table, String sinkPath, String... sinkPathContinued) { delegate().insertInto(table, sinkPath, sinkPathContinued); } - @Deprecated @Override public void insertInto(String targetPath, Table table) { delegate().insertInto(targetPath, table); } - @Deprecated @Override public String explain(Table table) { return delegate().explain(table); } - @Deprecated @Override public String explain(Table table, boolean extended) { return delegate().explain(table, extended); } - @Deprecated @Override public String explain(boolean extended) { return delegate().explain(extended); } - @Deprecated @Override public void sqlUpdate(String stmt) { delegate().sqlUpdate(stmt); } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableExt.java index 3a8ec80235..53b5a889a7 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.14/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { @@ -73,15 +71,13 @@ public DataStream> retractStream( } public DataStream toAppendStream(StreamTableContext context, TypeInformation typeInfo) { - context.isConvertedToDataStream = true; - return super.toAppendStream(typeInfo); + return appendStream(context, typeInfo); } public DataStream> toRetractStream( StreamTableContext context, TypeInformation typeInfo) { - context.isConvertedToDataStream = true; - return super.toRetractStream(typeInfo); + return retractStream(context, typeInfo); } public TableConversions(Table table) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index d8e66eed31..5b222d68a3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -23,21 +23,14 @@ import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -49,135 +42,8 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); - } - - @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - @Override public org.apache.flink.table.api.bridge.scala.StreamStatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Override - public void createTemporaryTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public org.apache.flink.table.api.Table from(org.apache.flink.table.api.TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String s, String s1) { - return tableEnv.listTables(s, s1); + return tableEnv().createStatementSet(); } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan loadPlan(org.apache.flink.table.api.PlanReference planReference) { - return tableEnv.loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan compilePlanSql(String s) { - return tableEnv.compilePlanSql(s); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableContext.java index 8e54cb0c23..95199e4983 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,16 +17,9 @@ package org.apache.streampark.flink.core; -import org.apache.streampark.common.util.Utils; - import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.module.ModuleEntry; import scala.Tuple2; @@ -44,53 +37,7 @@ public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor tableDescriptor) { - delegate().createTemporaryTable(path, tableDescriptor); - } - - @Override - public void createTable(String path, TableDescriptor tableDescriptor) { - delegate().createTable(path, tableDescriptor); - } - - @Override - public Table from(TableDescriptor tableDescriptor) { - return delegate().from(tableDescriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - @Override public JobExecutionResult execute(String jobName) { - Utils.printLogo("FlinkTable " + jobName + " Starting..."); - return null; + return printStartupLogo(jobName); } - - /** @since 1.15 */ - @Override - public String[] listTables(String catalogName, String databaseName) { - return delegate().listTables(catalogName, databaseName); - } - - /** @since 1.15 */ - @Override - public CompiledPlan loadPlan(PlanReference planReference) { - return delegate().loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public CompiledPlan compilePlanSql(String stmt) { - return delegate().compilePlanSql(stmt); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableExt.java index 8a6f0cc38d..a8b69c6ee8 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.15/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index d8e66eed31..5b222d68a3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -23,21 +23,14 @@ import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -49,135 +42,8 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); - } - - @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - @Override public org.apache.flink.table.api.bridge.scala.StreamStatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Override - public void createTemporaryTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public org.apache.flink.table.api.Table from(org.apache.flink.table.api.TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String s, String s1) { - return tableEnv.listTables(s, s1); + return tableEnv().createStatementSet(); } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan loadPlan(org.apache.flink.table.api.PlanReference planReference) { - return tableEnv.loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan compilePlanSql(String s) { - return tableEnv.compilePlanSql(s); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableContext.java index e4e118dfb4..63a467da77 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -18,12 +18,7 @@ package org.apache.streampark.flink.core; import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.module.ModuleEntry; import scala.Tuple2; @@ -40,48 +35,4 @@ public TableContext(Tuple2 args) { public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - delegate().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - delegate().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor tableDescriptor) { - return delegate().from(tableDescriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String catalogName, String databaseName) { - return delegate().listTables(catalogName, databaseName); - } - - /** @since 1.15 */ - @Override - public CompiledPlan loadPlan(PlanReference planReference) { - return delegate().loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public CompiledPlan compilePlanSql(String stmt) { - return delegate().compilePlanSql(stmt); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableExt.java index 8a6f0cc38d..a8b69c6ee8 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.16/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 971aafce02..5b222d68a3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -23,21 +23,14 @@ import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -49,178 +42,8 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); - } - - @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - @Override public org.apache.flink.table.api.bridge.scala.StreamStatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Override - public void createTemporaryTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); + return tableEnv().createStatementSet(); } - - @Override - public void createTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public org.apache.flink.table.api.Table from(org.apache.flink.table.api.TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String s, String s1) { - return tableEnv.listTables(s, s1); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan loadPlan(org.apache.flink.table.api.PlanReference planReference) { - return tableEnv.loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan compilePlanSql(String s) { - return tableEnv.compilePlanSql(s); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, - String className, - java.util.List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, - java.util.List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql( - String statement, - org.apache.flink.table.api.ExplainFormat format, - org.apache.flink.table.api.ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java index d80a450890..63a467da77 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -18,15 +18,7 @@ package org.apache.streampark.flink.core; import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; import scala.Tuple2; @@ -43,82 +35,4 @@ public TableContext(Tuple2 args) { public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - delegate().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - delegate().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor tableDescriptor) { - return delegate().from(tableDescriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String catalogName, String databaseName) { - return delegate().listTables(catalogName, databaseName); - } - - /** @since 1.15 */ - @Override - public CompiledPlan loadPlan(PlanReference planReference) { - return delegate().loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public CompiledPlan compilePlanSql(String stmt) { - return delegate().compilePlanSql(stmt); - } - - /** @since 1.17 */ - @Override - public void createFunction(String path, String className, java.util.List resourceUris) { - delegate().createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, java.util.List resourceUris, - boolean ignoreIfExists) { - delegate().createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, java.util.List resourceUris) { - delegate().createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, java.util.List resourceUris) { - delegate().createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return delegate().explainSql(statement, format, extraDetails); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java index 8a6f0cc38d..a8b69c6ee8 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 8ea4bf4c7b..5b222d68a3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -23,21 +23,14 @@ import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -49,184 +42,8 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); - } - - @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - @Override public org.apache.flink.table.api.bridge.scala.StreamStatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Override - public void createTemporaryTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public org.apache.flink.table.api.Table from(org.apache.flink.table.api.TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String s, String s1) { - return tableEnv.listTables(s, s1); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan loadPlan(org.apache.flink.table.api.PlanReference planReference) { - return tableEnv.loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan compilePlanSql(String s) { - return tableEnv.compilePlanSql(s); + return tableEnv().createStatementSet(); } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, - String className, - java.util.List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, - java.util.List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql( - String statement, - org.apache.flink.table.api.ExplainFormat format, - org.apache.flink.table.api.ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); - } - - /** @since 1.18 */ - @Override - public void createCatalog(String catalog, org.apache.flink.table.catalog.CatalogDescriptor catalogDescriptor) { - tableEnv.createCatalog(catalog, catalogDescriptor); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java index ed7d2a0d31..63a467da77 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -18,16 +18,7 @@ package org.apache.streampark.flink.core; import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; import scala.Tuple2; @@ -44,88 +35,4 @@ public TableContext(Tuple2 args) { public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - delegate().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - delegate().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor tableDescriptor) { - return delegate().from(tableDescriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String catalogName, String databaseName) { - return delegate().listTables(catalogName, databaseName); - } - - /** @since 1.15 */ - @Override - public CompiledPlan loadPlan(PlanReference planReference) { - return delegate().loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public CompiledPlan compilePlanSql(String stmt) { - return delegate().compilePlanSql(stmt); - } - - /** @since 1.17 */ - @Override - public void createFunction(String path, String className, java.util.List resourceUris) { - delegate().createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, java.util.List resourceUris, - boolean ignoreIfExists) { - delegate().createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, java.util.List resourceUris) { - delegate().createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, java.util.List resourceUris) { - delegate().createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return delegate().explainSql(statement, format, extraDetails); - } - - /** @since 1.18 */ - @Override - public void createCatalog(String catalog, CatalogDescriptor catalogDescriptor) { - delegate().createCatalog(catalog, catalogDescriptor); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java index 8a6f0cc38d..a8b69c6ee8 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 8ea4bf4c7b..5b222d68a3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -23,21 +23,14 @@ import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -49,184 +42,8 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); - } - - @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - @Override public org.apache.flink.table.api.bridge.scala.StreamStatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Override - public void createTemporaryTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public org.apache.flink.table.api.Table from(org.apache.flink.table.api.TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String s, String s1) { - return tableEnv.listTables(s, s1); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan loadPlan(org.apache.flink.table.api.PlanReference planReference) { - return tableEnv.loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan compilePlanSql(String s) { - return tableEnv.compilePlanSql(s); + return tableEnv().createStatementSet(); } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, - String className, - java.util.List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, - java.util.List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql( - String statement, - org.apache.flink.table.api.ExplainFormat format, - org.apache.flink.table.api.ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); - } - - /** @since 1.18 */ - @Override - public void createCatalog(String catalog, org.apache.flink.table.catalog.CatalogDescriptor catalogDescriptor) { - tableEnv.createCatalog(catalog, catalogDescriptor); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java index ed7d2a0d31..63a467da77 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -18,16 +18,7 @@ package org.apache.streampark.flink.core; import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; import scala.Tuple2; @@ -44,88 +35,4 @@ public TableContext(Tuple2 args) { public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - delegate().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - delegate().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor tableDescriptor) { - return delegate().from(tableDescriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String catalogName, String databaseName) { - return delegate().listTables(catalogName, databaseName); - } - - /** @since 1.15 */ - @Override - public CompiledPlan loadPlan(PlanReference planReference) { - return delegate().loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public CompiledPlan compilePlanSql(String stmt) { - return delegate().compilePlanSql(stmt); - } - - /** @since 1.17 */ - @Override - public void createFunction(String path, String className, java.util.List resourceUris) { - delegate().createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, java.util.List resourceUris, - boolean ignoreIfExists) { - delegate().createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, java.util.List resourceUris) { - delegate().createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, java.util.List resourceUris) { - delegate().createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return delegate().explainSql(statement, format, extraDetails); - } - - /** @since 1.18 */ - @Override - public void createCatalog(String catalog, CatalogDescriptor catalogDescriptor) { - delegate().createCatalog(catalog, catalogDescriptor); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java index 8a6f0cc38d..a8b69c6ee8 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -24,6 +24,7 @@ /** * Table extension utilities for Flink Table API. */ + public final class TableExt { private TableExt() { @@ -49,9 +50,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 8ea4bf4c7b..5b222d68a3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -23,21 +23,14 @@ import scala.Tuple3; -/** - * Integration api of stream and table - */ +/** Integration api of stream and table */ public class StreamTableContext extends FlinkStreamTableTrait { - private final StreamExecutionEnvironment streamEnv; - private final StreamTableEnvironment tableEnv; - public StreamTableContext( ParameterTool parameter, StreamExecutionEnvironment streamEnv, StreamTableEnvironment tableEnv) { super(parameter, streamEnv, tableEnv); - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; } public StreamTableContext( @@ -49,184 +42,8 @@ public StreamTableContext(StreamTableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - @Override - public org.apache.flink.table.api.Table fromDataStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public org.apache.flink.table.api.Table fromChangelogStream( - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView( - String path, - org.apache.flink.streaming.api.scala.DataStream dataStream, - org.apache.flink.table.api.Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toDataStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.types.AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public org.apache.flink.streaming.api.scala.DataStream toChangelogStream( - org.apache.flink.table.api.Table table, - org.apache.flink.table.api.Schema targetSchema, - org.apache.flink.table.connector.ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public void useModules(String... strings) { - tableEnv.useModules(strings); - } - - @Override - public org.apache.flink.table.module.ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - @Override public org.apache.flink.table.api.bridge.scala.StreamStatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - @Override - public void createTemporaryTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, org.apache.flink.table.api.TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public org.apache.flink.table.api.Table from(org.apache.flink.table.api.TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String s, String s1) { - return tableEnv.listTables(s, s1); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan loadPlan(org.apache.flink.table.api.PlanReference planReference) { - return tableEnv.loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public org.apache.flink.table.api.CompiledPlan compilePlanSql(String s) { - return tableEnv.compilePlanSql(s); + return tableEnv().createStatementSet(); } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, - String className, - java.util.List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, - java.util.List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, - java.util.List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql( - String statement, - org.apache.flink.table.api.ExplainFormat format, - org.apache.flink.table.api.ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); - } - - /** @since 1.18 */ - @Override - public void createCatalog(String catalog, org.apache.flink.table.catalog.CatalogDescriptor catalogDescriptor) { - tableEnv.createCatalog(catalog, catalogDescriptor); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java index ed7d2a0d31..63a467da77 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -18,16 +18,7 @@ package org.apache.streampark.flink.core; import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; import scala.Tuple2; @@ -44,88 +35,4 @@ public TableContext(Tuple2 args) { public TableContext(TableEnvConfig args) { this(FlinkTableInitializer.initialize(args)); } - - @Override - public void useModules(String... strings) { - delegate().useModules(strings); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - delegate().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - delegate().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor tableDescriptor) { - return delegate().from(tableDescriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return delegate().listFullModules(); - } - - /** @since 1.15 */ - @Override - public String[] listTables(String catalogName, String databaseName) { - return delegate().listTables(catalogName, databaseName); - } - - /** @since 1.15 */ - @Override - public CompiledPlan loadPlan(PlanReference planReference) { - return delegate().loadPlan(planReference); - } - - /** @since 1.15 */ - @Override - public CompiledPlan compilePlanSql(String stmt) { - return delegate().compilePlanSql(stmt); - } - - /** @since 1.17 */ - @Override - public void createFunction(String path, String className, java.util.List resourceUris) { - delegate().createFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createFunction( - String path, String className, java.util.List resourceUris, - boolean ignoreIfExists) { - delegate().createFunction(path, className, resourceUris, ignoreIfExists); - } - - /** @since 1.17 */ - @Override - public void createTemporaryFunction( - String path, String className, java.util.List resourceUris) { - delegate().createTemporaryFunction(path, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public void createTemporarySystemFunction( - String name, String className, java.util.List resourceUris) { - delegate().createTemporarySystemFunction(name, className, resourceUris); - } - - /** @since 1.17 */ - @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return delegate().explainSql(statement, format, extraDetails); - } - - /** @since 1.18 */ - @Override - public void createCatalog(String catalog, CatalogDescriptor catalogDescriptor) { - delegate().createCatalog(catalog, catalogDescriptor); - } - } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java index 8a6f0cc38d..e1633ed67f 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java @@ -49,9 +49,6 @@ public Table alias(String field, String... fields) { return table.as(field, fields); } - public Table $minus$greater(String field, String... fields) { - return alias(field, fields); - } } public static final class TableConversions extends org.apache.flink.table.api.bridge.scala.TableConversions { diff --git a/streampark-flink/streampark-flink-sqlclient/pom.xml b/streampark-flink/streampark-flink-sqlclient/pom.xml index b09e6ad60f..793c5a3c02 100644 --- a/streampark-flink/streampark-flink-sqlclient/pom.xml +++ b/streampark-flink/streampark-flink-sqlclient/pom.xml @@ -81,11 +81,6 @@ - - net.alchim31.maven - scala-maven-plugin - - org.apache.maven.plugins maven-shade-plugin diff --git a/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java b/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java index 7556feec13..0a3240cbfb 100644 --- a/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java +++ b/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java @@ -54,10 +54,8 @@ public static void main(String[] args) { if (StringUtils.isBlank(sql)) { throw new IllegalArgumentException("Usage: flink sql cannot be null"); } - String flinkSql; - try { - flinkSql = DeflaterUtils.unzipString(sql); - } catch (Exception e) { + String flinkSql = DeflaterUtils.unzipString(sql); + if (StringUtils.isBlank(flinkSql)) { throw new IllegalArgumentException("Usage: flink sql is invalid or null, please check"); } diff --git a/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java b/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java index 9ceef81434..b69ae36faf 100644 --- a/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java +++ b/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java @@ -17,6 +17,7 @@ package org.apache.streampark.flink.cli; +import org.apache.streampark.common.conf.ConfigKeys; import org.apache.streampark.flink.core.SqlCommand; import org.apache.streampark.flink.core.SqlCommandCall; @@ -79,4 +80,25 @@ void resolveExecutionModeFromDynamicProperty() { assertEquals("BATCH", mode); assertTrue(arguments.isEmpty()); } + + @Test + void resolveExecutionModeFromAppConfYaml() { + List arguments = new ArrayList<>(); + String yamlContent = ConfigKeys.KEY_FLINK_TABLE_MODE() + ": batch\n"; + String appConf = "yaml://" + org.apache.streampark.common.util.DeflaterUtils.zipString(yamlContent); + Map params = new HashMap<>(); + params.put(ConfigKeys.KEY_APP_CONF(), appConf); + ParameterTool parameterTool = ParameterTool.fromMap(params); + + String mode = + SqlClient.resolveExecutionMode( + parameterTool, + Collections.emptyList(), + arguments, + RuntimeExecutionMode.STREAMING.name()); + + assertEquals("BATCH", mode); + assertTrue( + arguments.contains("-D" + ExecutionOptions.RUNTIME_MODE.key() + "=BATCH")); + } } diff --git a/streampark-flink/streampark-flink-udf/pom.xml b/streampark-flink/streampark-flink-udf/pom.xml index 8ce2a770c3..eadcebfbee 100644 --- a/streampark-flink/streampark-flink-udf/pom.xml +++ b/streampark-flink/streampark-flink-udf/pom.xml @@ -53,15 +53,6 @@ - - - - net.alchim31.maven - scala-maven-plugin - - - - apache-release