华为fusionInsigtht集群es连接工具

  华为fusionInsight为用户提供海量数据的管理及分析功能，快速从结构化和非结构化的海量数据中挖掘您所需要的价值数据。开源组件结构复杂，安装、配置、管理过程费时费力，使用华为FusionInsight Manager将为您提供企业级的集群的统一管理平台,在工作中遇到使用华为集群的es由于过于安全，操作反而不便，为此记录下使用工具

1.使用账号密码登陆web界面下载认证凭据

2.1使用如下pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.example</groupId>
  <artifactId>huawei_es_tools</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <name>huawei_es_tools</name>

  <description>huawei_es_tools</description>
  <properties>
    <java.version>1.8</java.version>
  </properties>

  <dependencies>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-core</artifactId>
      <version>2.12.5</version>
    </dependency>

    <dependency>
      <groupId>com.huawei</groupId>
      <artifactId>elasticsearch-rest-client</artifactId>
      <version>6.7.1</version>
    </dependency>
    <dependency>
      <groupId>com.huawei</groupId>
      <artifactId>elasticsearch</artifactId>
      <version>6.7.1</version>
    </dependency>
    <dependency>
      <groupId>com.huawei</groupId>
      <artifactId>elasticsearch-rest-high-level-client</artifactId>
      <version>6.7.1</version>
    </dependency>


  </dependencies>

  <repositories>

    <repository>
      <id>huaweicloudsdk</id>
      <url>https://mirrors.huaweicloud.com/repository/maven/huaweicloudsdk/</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>true</enabled>
      </snapshots>
    </repository>

    <repository>
      <id>central</id>
      <name>Mavn Centreal</name>
      <url>https://repo1.maven.org/maven2/</url>
    </repository>

  </repositories>
  <build>
    <sourceDirectory>src/main/java</sourceDirectory>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-dependency-plugin</artifactId>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-jar-plugin</artifactId>
        <version>3.2.0</version>
        <configuration>
          <excludes>
            <exclude>log4j.properties</exclude>
            <exclude>log4j2.xml</exclude>
          </excludes>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <configuration>
          <source>1.8</source>
          <target>1.8</target>
        </configuration>
      </plugin>
    </plugins>
  </build>
</project>

3.在项目目录下建立conf文件夹并存放1步骤中的凭据文件

4.测试工具类

public class EsTest {

  /**
   * 配置文件路径位置
   */
  private static final int CONFIG_PATH_ARGUMENT_INDEX = 0;

  /**
   * 获取HwRestClient
   *
   * @param args 配置参数
   * @return HwRestClient
   */
  public static HwRestClient getHwRestClient(String[] args) {
    HwRestClient hwRestClient;
    if (args == null
            || args.length < 1
            || args[CONFIG_PATH_ARGUMENT_INDEX] == null
            || args[CONFIG_PATH_ARGUMENT_INDEX].isEmpty()) {
      hwRestClient = new HwRestClient();
    } else {
      String configPath = args[CONFIG_PATH_ARGUMENT_INDEX];
      File configFile = new File(configPath);
      if (configFile.exists()) {
        if (configFile.isDirectory()) {
          hwRestClient = new HwRestClient(configPath);
        } else {
          try {
            hwRestClient =
                    new HwRestClient(
                            configFile
                                    .getCanonicalPath()
                                    .substring(
                                            0,
                                            configFile.getCanonicalPath().lastIndexOf(File.separator) + 1));
          } catch (IOException e) {
            hwRestClient = new HwRestClient();
          }
        }
      } else {
        hwRestClient = new HwRestClient();
      }
    }
    return hwRestClient;
  }

  /**
   * 查询指定索引下数据
   *
   * @param highLevelClient
   * @param index
   */
  public static void search(RestHighLevelClient highLevelClient, String index) {
    try {
      //A search source builder allowing to 创建一个搜索源
      SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
      //SearchRequest按一个或多个索引查询,需要一个SearchSourceBuilder,搜索源提供了搜索选项
      SearchRequest searchRequest = new SearchRequest();
      //text类型不能用于索引或排序，必须转成keyword类型
      //String AggregationName = "application_aggregations";
      //脚本
      //String painlessScript = "((doc['S_IP.keyword'].value))";
      //TermsAggregationBuilder aggregation = AggregationBuilders
      //        .terms(AggregationName)
      //        .script(new Script(ScriptType.INLINE, "painless", painlessScript, new HashMap<>()))
      //        //应返回桶的数量--全量返回
      //        .size(Integer.MAX_VALUE)
      //        //最少1条
      //        .minDocCount(1)
      //        .shardMinDocCount(0)
      //        //返回文档计数错误
      //        .showTermDocCountError(false);
      //添加bool过滤器,进行条件查询
      BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
      //must --时间必须满足
      //存在关键字sourceOrDestinationTerm(S_IP)
      //boolQueryBuilder.must(QueryBuilders.existsQuery("S_IP"));
      //定义sourceBuilder,范围为0-9999,按时间排序,正序,再传入之前的查询条件,from 0 size 0 不查原始数据
      //sourceBuilder.sort("TIME.keyword", SortOrder.ASC).from(0).size(0).query(boolQueryBuilder).aggregation(aggregation);
      //定义查询的索引,定义搜索源,即sourceBuilder对象
      searchRequest.indices(index);
      searchRequest.source(sourceBuilder);
      //开始搜索,拿到结果
      SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
      System.out.println("Search response is==" + searchResponse.toString());
    } catch (IOException e) {
      System.out.println("Search is failed, exception occurred." + e);
    }
  }

  public static void addData(RestHighLevelClient highLevelClient, String index, String id, String dataStr) {
    try {
      IndexRequest indexRequest = new IndexRequest(index).id(id);
      indexRequest.source(dataStr, XContentType.JSON);
      indexRequest.type("_doc");
      IndexResponse indexResponse = highLevelClient.index(indexRequest, RequestOptions.DEFAULT);

      System.out.println("addData response is " + indexResponse.toString());
    } catch (IOException e) {
      System.out.println("addData is failed,exception occurred." + e);
    }
  }


  public static void main(String[] args) {
    RestHighLevelClient highLevelClient = null;
    HwRestClient hwRestClient = getHwRestClient(args);
    try {
      highLevelClient = new RestHighLevelClient(hwRestClient.getRestClientBuilder());
      addData(highLevelClient, "sql_log_2023-11-07", "1", "{\"title\":\"其余信息\",\"key\":\"other\"}");
      search(highLevelClient, "sql_log_log*");
    } finally {
      try {
        if (highLevelClient != null) {
          highLevelClient.close();
        }
      } catch (IOException e) {
        System.out.println("Failed to close RestHighLevelClient." + e);
      }
    }
  }
}