Springcloud集成hadoop HDFS，使用Feign-form实现文件上传、下载_springcloud和hadoop

构建基于SpringCloud分布式，使用hadoop HDFS实现文件的上传下载功能

最近做springcloud项目开发，需要使用文件服务器HDFS进行文件的上传下载，所以将实现的方法步骤做个记录，分享出来，如有疑问，欢迎随时交流：

项目结构如下

项目使用到Eureka作为注册中心，通过Feign客户端进行服务调用，废话少说，直接上代码
代码片.

实现层（file_service_impl）

// application.yml配置文件
server:
  port: 8020
  undertow:
    direct-buffers: true
### hadoop配置
hdfs:
    path: hdfs://Hadoop的ip:9000
    username: Lee
###服务别名
spring:
  application:
    name: file-server
  servlet:
    multipart:
       enabled: true
       max-file-size: 50MB
       max-request-size: 50MB    
###注册到eureka地址
eureka:
  client:
    service-url:
           defaultZone: http://127.0.0.1:8100/eureka
    #####是否自己注册
    register-with-eureka: true
    ###是否需要从eureka上获取注册信息
    fetch-registry: true
  instance:
    # ip-address:
    prefer-ip-address: true
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

// HDFSUtil工具类
package com.shyikun.impl;
 
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;

import javax.annotation.PostConstruct;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
 
@Component
public class HDFSUtil {
   
 
	@Value("${hdfs.path}") 
	private String path;
	 
	private static String hdfsPath;
	
	/**
	 * 获取HDFS配置信息
	 * @return
	 */
	private static Configuration getConfiguration() {
   
		Configuration configuration = new Configuration();
		configuration.set("fs.defaultFS", hdfsPath);
		configuration.set("hadoop.home.dir", "D:/Hadoop/winutils-master/hadoop-3.0.0");
		
		return configuration;
	}
 
	/**
	 * 判断路径是否存在
	 * 
	 * @param conf
	 * @param path
	 * @return
	 * @throws IOException
	 */
	public static boolean exits(String path) throws IOException {
   
		FileSystem fs = FileSystem.get(getConfiguration());
		return fs.exists(new Path(path));
	}
 
	/**
	    * 判断文件是否存在
	 * @param hdfsPath
	 * @return
	 */
	public static boolean checkFileExist(String hdfsPath) {
   
		FileSystem fs;
		boolean isExists = false;
		try {
   
			fs = FileSystem.get(URI.create(hdfsPath), getConfiguration());
			Path delPath = new Path(hdfsPath);
			isExists = fs.exists(delPath);
		} catch (IOException e) {
   
			e.printStackTrace();
		}
		return isExists; 
	}
	
	/**
	 * 删除目录或文件
	 * 
	 * @param conf
	 * @param remoteFilePath
	 * @param recursive
	 * @return
	 * @throws IOException
	 */
	public static boolean deleteFile(String remoteFilePath, boolean recursive)
			throws IOException {
   
		FileSystem fs = FileSystem.get(getConfiguration());
		boolean result = fs.delete(new Path(remoteFilePath), recursive);
		fs.close();
		return result;
	}
 
	/**
	 * 删除目录或文件(如果有子目录,则级联删除)
	 * 
	 * @param conf
	 * @param remoteFilePath
	 * @return
	 * @throws IOException
	 */
	public static boolean deleteFile(String remoteFilePath) throws IOException {
   
		return deleteFile(remoteFilePath, true);
	}
 
	/**
	 * 文件重命名
	 * 
	 * @param conf
	 * @param oldFileName
	 * @param newFileName
	 * @return
	 * @throws IOException
	 */
	public static boolean renameFile(String oldFileName, String newFileName)
			throws IOException {
   
		FileSystem fs = FileSystem.get(getConfiguration());
		Path oldPath = new Path(oldFileName);
		Path newPath = new Path(newFileName);
		boolean result = fs.rename(oldPath, newPath);
		fs.close();
		return result;
	}
 
	/**
	 * 创建目录
	 * 
	 * @param conf
	 * @param dirName
	 * @return
	 * @throws IOException
	 */
	public static boolean createDirectory(String dirName) throws IOException {
   
		FileSystem fs = FileSystem.get(getConfiguration());
		Path dir = new Path(dirName);
		boolean result = false;
		if (!fs.exists(dir)) {
   
			result = fs.mkdirs(dir);
		}
		fs.close();
		return result;
	}
 
	/**
	 * 列出指定路径下的所有文件(不包含目录)
	 * 
	 * @param conf
	 * @param basePath
	 * @param recursive
	 */
	public static RemoteIterator<LocatedFileStatus> listFiles(String basePath, boolean recursive) throws IOException {
   
		FileSystem fs = FileSystem.get(getConfiguration());
		RemoteIterator<LocatedFileStatus> fileStatusRemoteIterator = fs.listFiles(new Path(basePath), recursive);
 
		return fileStatusRemoteIterator;
	}
 
	/**
	 * 列出指定路径下的文件（非递归）
	 * 
	 * @param conf
	 * @param basePath
	 * @return
	 * @throws IOException
	 */
	public static RemoteIterator<LocatedFileStatus> listFiles(String basePath)
			throws IOException {
   
		FileSystem fs = FileSystem.get(getConfiguration());
		RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(
				new Path(basePath), false);
		fs.close();
		return remoteIterator;
	}
 
	/**
	 * 列出指定目录下的文件\子目录信息（非递归）
	 * 
	 * @param conf
	 * @param dirPath
	 * @return
	 * @throws IOException
	 */
	public static FileStatus[] listStatus(String dirPath) throws IOException {
   
		FileSystem fs = FileSystem.get(getConfiguration());
		FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
		fs.close();
		return fileStatuses;
	}
	
	/**
	 * 上传文件到hdfs
	 * 
	 * @param fs
	 * @param localFileName
	 * @param hdfsFileName
	 * @throws IOException
	 */
	public static void upload(FileSystem fs,String localFilePath,String hdfsFilePath) throws IOException {
   
        fs.copyFromLocalFile(new Path(localFilePath), new Path(hdfsFilePath));
    }
	
	/** 
     * File对象上传到hdfs 
     * @param conf 
     * @param uri 
     * @param remote 
     * @param local 
     * @throws IOException 
     */  
    public static void createFile(File localPath, String hdfsPath) throws IOException {
     
        InputStream in = null;  
        try {
     
        	FileSystem fs = FileSystem.get(getConfiguration());
            FSDataOutputStream out = fs.create(new Path(hdfsPath));  
            in = new BufferedInputStream(new FileInputStream(localPath));  
            IOUtils.copyBytes(in, out, 4096, false);  
            out.hsync();  
            out.close();  
            System.out.println("create file in hdfs:" + hdfsPath);  
        } finally {
     
            IOUtils.closeStream(in);  
        }  
    }
 
	/**
	 * 下载 hdfs上的文件
	 * 
	 * @param conf
	 * @param uri
	 * @param remote
	 * @param local
	 * @throws IOException
	 */
	public static void download(String remote, String local) throws IOException {
   
		FileSystem fs = FileSystem.get(URI.create(remote), getConfiguration());
		Path path = new Path(remote);
		fs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253