200 2800 2200 3600 2017 10 300 7000 WebiOS Android ---- ......

iv>2017 " " 2013 12 200 2014 9 "" 8
" " 2015 33 15 1

"" 58 759 AI 8

<v bug

2018

1

6

18

doT.js

33

WebView

45

SDK

64

76

88

Shield

96

103

iPhone X

118

KIF iOS UI

132

Android

145

Android Walle

157

165

Android

177

192

App Spider

204

<vii

MGW Docker HULK Leaf ID 0 1

DBProxy SQL SQLAdvisor MyFlash MySQL Sysbench

212 212 238 254 270 281 290 306 321 334 343 361 361 372 378 389 403

viii>2017

Flink Storm ---- BI AI ETA O2O DSP

413 413 435 443 464 474 482 491 491 500 517 523 542 556 566 581 589 610 627 640

<ix

SRE Mt-FalconOpen-Falcon

Android Binder Android Code Arbiter

654 654 669 681 696 696 705 721 736 736

OS

2>2017

CDN MVVM Vue KNBNative Bridge AWP NodeServer AngularJS/Vue.js NGUIAngularJS Node Express OPS H5 SEO SDK Node 30% Top90 9ms Node

<3 Yeoman Gulp Webpack NPM Sass CSS CSS Babel ES6 ES6 ESLint Sonar Sentry Perf Web Falcon Server PM25 Node NodeServer Node

4>2017

Node

H5 H5

Vue React MVVM
Vue2.x SSRPC
Weex VueVue-Native Hybrid
IE8 PC
Vue AWP
2.x

<5 React Licence Vue

100 IM PC

6>2017

2016 6 C " "

UI

Java FTL CSS JS Cortex FTL

<7 AJAX CI CDN

ES6 Module

Bundle

8>2017 Split JS SPA Tree Shaking ES6 bundle JS Bundle
Scope HoistingES6 Bundle

API ReactVue Angular Vue.js APIW3C Web Component

19kBmin+gzip API webpack Loader Vue Dev Toolsvue-clivue-test-utils

<9 Virtual DOM DOM DOM

webpack Vue.js webpack loader Vue.js Vue.js dora-ui 20 Vue.js 2.0
Readmemarkdown

webpack plugin JSON JSON

10>2017

<11

demo webpack require.context API components demo Demo var demoRequire = require.context('@component', true, /demo/.*.vue$/); // demo const demos = demoRequire.keys().map(demoKey => { var [componentName, demoName] = demoKey.split('/demo/'); componentName = componentName.substring(2); demoName = demoName.substring(0, demoName.lastIndexOf('.')); return { componentName: componentName, demoName: demoName,

12>2017 component: demoRequire(demoKey) } }); // key + value demo const demosByComponent = _.groupBy(demos, demo => { return demo.componentName; }); // const routesByComponent = Object.keys(demosByComponent).map(componentName => { return { path: '/' + componentName, component: require('./component.vue'), meta: { componentName: componentName, demoComponents: demosByComponent[componentName] } } }); // demo const routesByDemo = demos.map(demo => { return { path: '/' + demo.componentName + '/' + demo.demoName, component: demo.component, meta: { componentName: demo.componentName, } } }); Mock web- pack-dev-server Mock API

Mock

<13

3

14>2017

CDN

AJAX SLB SLB

                           CDN  30   10

CDN

<15 & CDN

HTML Application Cache API

16>2017 manifest manifest manifest manifest applicationCache updateready updateready

Vue +
AppCache Service Worker SW SW

<17

Web WebGLWebVRHTTP/2 Service WorkerWeb AssemblyWebRTC Web

2016

18>2017

""

2.1 2.1.1

<19

""

3.2.1 Model

QPS

20>2017

2.1.2 UI UI 3.2.1 Script

2.1.3 FreeMarker EJS HTML

<21 Mock ModelView JSON Mock

2.2

22>2017

"" 0.username 2.3 Java Node.js SDK 2.3.1Java JAR SDK HTML FreeMarker /** * * @param pageId id vurl, pageIdType * @param pageIdTypeLegoService.PageIdType.IDLegoService. PageIdType.NAME id vurl * @return OK(200," "), FAILED(500," "); * / public static final LegoStatus getPageWithoutData(HttpServletResponse servletResponse, String pageId, PageIdType pageIdType) /* * * @param model * */ public static final LegoStatus getPage(HttpServletResponse servletResponse, String pageId, PageIdType pageIdType, ModelMap model) 2.3.2Node.js 'use strict'; var lego = require('lego');

<23 /** * id , * @param data, json * @param rootId * @param callback, callback(err, body), err body html err * / lego.renderById(vid, data, rootId, callback) /*

- */ lego.renderByUrl(vurl, data, rootId, callback)

3.1 AMDCMDCommonJS
JavaScript Web App
DOM

24>2017

3.2

3.2.1 / UI HTML HTML CSS Hack CSS 8 /** * 8 name, pyname, desc, leaf, uilib, model, script, render */ 'use strict';

<25

/** * * * [ name RD ] */ exports.name = 'Sample';

/** * * [ pyname PM hmtl ] */ exports.pyname = ' ';

/** * */ exports.desc = '';

/** * * 1. id : exports.leaf = [12,23,34] * 2. leaf null : exports.leaf = null * 3. leaf : exports.leaf = [] */ exports.leaf = [];

/** * */ exports.uilib = 'kui';

/**

/
type: : * text:

textEx: {

name: ' 1',

type: 'text',

def : ' ',

desc: ' '

}
select:

selectEx: {

name: ' 1', // 9

type: 'select',

options:{

26>2017

value1: ' 1',

value2: ' 2',

},

def : 'defValue',

desc: ' '

}
textarea: ( text) * radio: ( select)
checkbox: ( select value1,value2 )

*/

exports.model = {

}

/** * html * @param mvId mvId * @param evtMgr bind(evt, handler) unbind(evt, handler) trigger(evt, data, context) * @param modelData encode decodeURI json */ exports.script = function (mvid, evtMgr, modelData) { modelData = JSON.parse(decodeURI(modelData)); }

/** * * @param {Object} node node {_children:[], _parent:'', _innerHtml:'', _modelData:{}, _mid:'', _mvid:'', _mname:''}, */ exports.render = function (node) { var _modelData = node._modelData; var _children = node._children; var content = ''; _children.map(function (child) { content += child._innerHtml; });

return ``; }

3.2.2

<27

Script mvidevtMgr modalDataevtMgr

// evtMgr.trigger('tata', {a: 1});

// evtMgr.bind('tata', (params) => {

// TODO Here. console.log(params) })

// {a: 1}

AJAX

28>2017 Render exports.render = function (node) { var _modelData = node._modelData; var _children = node._children; var content = ''; _children.map(function (child) { content += child._innerHtml; }); return ${data_from_datasource}; // } "3.2.1 " 8

HTML   freemarker.jar   freemarker.jar

<29 HTML
3.3 Node.js Express

30>2017

3.4

_children_parent

{ 1_0: { _children: [101_1], _mvid: '1_0', _parent: '#', _mid: '1', _version: '1.0.0' _mname: 'Page', _xxx: '' }, 101_1: { _children: [5_2], _mvid: '101_1', _parent: '1_0, _mid: '101', _version: '1.0.0' _mname: 'Body', _xxx: '' }, 5_2: { _xxx: '' } }

// children in current page // unique id in current page // parent in current page // id // version // name // other properties

4.1 55 77 132 6 108

<31
4.2 UI ---- UI
UI 4.3

32>2017

Web 2013 10 900 to Bto C 2010 MTKIBM2016 2012 Web 2015

doT.js

<33

MVC-->MVP-- >MVVM ReactVueAngular MVVM

doT.js Laura Doktorova doT.js

doT.js mustache Handlebars artTemplate( ) BaiduTemplate( ) jQuery-tmpl

6KB

4KB

18.9 KB 9.3 KB

512KB 62.3KB

5.2KB

9.45KB 6KB

18.6KB 5.98KB

×

doT.js Mac Pro Chrome 56.0.2924.87 100 10000

34>2017 doT.js

6KB 4KB 2. 3. 4. doT.js

doT.jsdoT.js

name:{{= it.name}}

age:{{= it.age}}

hello:{{= it.sayHello() }}

</script> <script> $("#app").html(doT.template($("#tpl").html())({ name:'stringParams1', stringParams1:'stringParams1_value',

<35

stringParams2:1, arr:[{id:0,text:'val1'},{id:1,text:'val2'}], sayHello:function () { return this[this.name] } })); </script>

doT.js                        HTML

JavaScript

=

if

else/else if

for

funcName()

{{= }} {{? }} {{??}}/{{?? }} {{~ }} {{= funcName() }}

{{=it.name }} {{? i > 3}} {{?? i ==2}} {{~ it.arr:item}}...{{~}} {{= it.sayHello() }}

doT.js 1.
2. View JavaScript 3. doT.js ... // str = ("var out='" + (c.strip ? str.replace(/(^|\r|\n)\t* +| +\t*(\r|\n|$)/g," ") .replace(/\r|\n|\t|/*[\s\S]*?*//g,""): str) .replace(/'|\/g, "\$&")

36>2017 .replace(c.interpolate || skip, function(m, code) { return cse.start + unescape(code,c.canReturnNull) + cse.end; }) .replace(c.encode || skip, function(m, code) { needhtmlencode = true; return cse.startencode + unescape(code,c.canReturnNull) + cse.end; }) // if else .replace(c.conditional || skip, function(m, elsecase, code) { return elsecase ? (code ? "';}else if(" + unescape(code,c.canReturnNull) + ") {out+='" : "';}else{out+='") : (code ? "';if(" + unescape(code,c.canReturnNull) + "){out+='" : "';}out+='"); }) // .replace(c.iterate || skip, function(m, iterate, vname, iname) { if (!iterate) return "';} } out+='"; sid+=1; indv=iname || "i"+sid; iterate=unescape(iterate); return "';var arr"+sid+"="+iterate+";if(arr"+sid+"){var "+vname+"," +indv+"=-1,l"+sid+"=arr"+sid+".length-1;while("+indv+"<l"+sid+"){" +vname+"=arr"+sid+"["+indv+"+=1];out+='"; }) // .replace(c.evaluate || skip, function(m, code) { return "';" + unescape(code,c.canReturnNull) + "out+='"; }) + "';return out;") ... try { return new Function(c.varname, str);//c.varname new Function } catch (e) { /* istanbul ignore else */ if (typeof console !== "undefined") console.log("Could not create a template function: " + str); throw e; } ...

HTML new Function()

<37

1 doT.js doT.js doT.js

templateSettings: {

evaluate: /{{([\s\S]+?(}?)+)}}/g, //

interpolate: /{{=([\s\S]+?)}}/g, //

encode:

/{{!([\s\S]+?)}}/g, // {{

use:

/{{#([\s\S]+?)}}/g,

useParams: /(^|[^\w$])def(?:.|[['"])([\w$.]+)(?:['"]])?\

s*:\s*([\w$.]+|"[^\"]+"|'[^\']+'|{[^\}]+})/g,

define:

/{{##\s*([\w.$]+)\s*(:|=)([\s\S]+?)#}}/g,//

defineParams:/^\s*([\w$]+):([\s\S]+)/, //

conditional: /{{?(?)?\s*([\s\S]?)\s}}/g, //

iterate:

/{{~\s*(?:}}|([\s\S]+?)\s*:\s*([\w$]+)\s*(?::\s*([\

w$]+))?\s*}})/g, //

varname: "it", //

strip: true,

append:

true,

selfcontained: false, doNotSkipEncoded: false //

}

doT.js tmpl doT. js
doT.js

38>2017 2new Function() Function JavaScript Function Object Array Date var funcName = new Function(p1,p2,...,pn,body); p1 pn body funcName

// function func1(a,b){ return a+b; } // var func2 = new Function('a,b','return a+b'); // var func3 = new Function('a','b','return a+b'); // console.log(func1(1,2)); console.log(func2(2,3)); console.log(func3(1,3)); // 3 // func1 5 // func2 4 // func3 Function eval eval XSS eval new Function() doT "new Function(c.varname, str)" varname str new Fcuntion Function

<39

doT.js

jQuery-tmpl function buildTmplFn( markup ) { return new Function("jQuery","$item", // Use the variable __ to hold a string array while building the compiled template. (See https://github.com/jquery/jquery-tmpl/issues#issue/10). "var $=jQuery,call,=[],$data=$item.data;" + // Introduce the data as local variables using with(){} "with($data){.push('" + // Convert the template into pure JavaScript jQuery.trim(markup) .replace( /([\'])/g, "\$1" ) .replace( /[\r\t\n]/g, " " ) .replace( /${([^\}])}/g, "{{= $1}}" ) .replace( /{{(/?)(\w+|.)(?:(((?:[^\}]|}(?!}))?) ?))?(?:\s+(.?)?)?((((?:[^\}]|}(?!}))?)))?\s*}}/g, function( all, slash, type, fnargs, target, parens, args ) { // , : https://github.com/BorisMoore/jquery-tmpl }) + "');}return __;" ); } jQuery-teml new Function() jQuery-teml doT.js with with var datas = {persons:[' ',' ',' ',' ',' ',' ',' '], gifts:[' ',' ',' ',' ',' ']}; function go(){ with(datas){ var personIndex = 0,giftIndex = 0,i=100000; while(i){ personIndex = Math.floor(Math.random()*persons.length); giftIndex = Math.floor(Math.random()*gifts.length)

40>2017 console.log(persons[personIndex] +' :'+ gifts[giftIndex]); i--; } } } with datas with JavaScript with datas
GitHub Boris Moore

<41 1. String Dom jQuerytmpl with 2.
with
Boris Moore

42>2017

function buildTmplFn( markup ) { if(!compledStr){ // Convert the template into pure JavaScript compledStr = jQuery.trim(markup) .replace( /([\'])/g, "\$1" ) .replace( /[\r\t\n]/g, " " ) .replace( /${([^\}])}/g, "{{= $1}}" ) .replace( /{{(/?)(\w+|.)(?:(((?:[^\}]|}(?!}))?)?))? (?:\s+(.?)?)?((((?:[^\}]|}(?!}))?)))?\s*}}/g, // } return new Function("jQuery","$item", // Use the variable __ to hold a string array while building the compiled template. (See https://github.com/jquery/jquery-tmpl/issues#issue/10). "var $=jQuery,call,=[],$data=$item.data;" + // Introduce the data as local variables using with(){} ".push('" + compledStr + "');return __;" ) } doT.js with doT.js
25 tmpl (function(){ var cache = {}; this.tmpl = function (str, data){ var fn = !/\W/.test(str) ? cache[str] = cache[str] || tmpl(document.getElementById(str).innerHTML) : new Function("obj", "var p=[],print=function(){p.push.apply(p,arguments);};" + "with(obj){p.push('" + str

<43 .replace(/[\r\t\n]/g, " ") .split("<%").join("\t") .replace(/((^|%>)[^\t])'/g, "$1\r") .replace(/\t=(.?)%>/g, "',$1,'") .split("\t").join("');") .split("%>").join("p.push('") .split("\r").join("\'") + "');}return p.join('');"); return data ? fn( data ) : fn; }; })(); baiduTemplate baiduTemplate baiduTemplate doT.js with tmpl with

https://github.com/chen2009277025/TemplateTest

44>2017

doT.js

doT.js
doT.js baiduTemplate baiduTemplate
""

2015 Hi-FE

WebView

<45

App WebView AndroidiOS Web WebView
WebView WebView

WebView native WebView loading WebView

1. 1. loading 4. WebView

46>2017 WebView
WebView App WebView WebView App WebView

WebView WebView
WebView WebView WebView
WebView WebView 1iOS Titans 10.0.7 2OPPO R829T Android 4.2.2 10 App ms

<47

*Android WebView WebView

WebView 70~700ms " WebView " WebView WebView WebView

48>2017

? native
WebView WebView WebView WebView WebView App WebView

 -  CN106250434A

WebView native native WebView WebView
70% H5 QQ Hybrid

<49

WebView 2. Native WebView

/ DNS connection

n% ms

DNS API DNS WebView native API DNS api.meituan.com WebView i.meituan.com

50>2017 App api.meituan.com DNS WebView
i.meituan.com IP 10% WebView 60ms DNS WebView App API WebView DNS 1.3ms

chunk chunk flush -->Web API--> API Web API Web API

1. API API HTTP header transfer-encoding:chunked head body Web API API API chunk

<51 API API chunk-encoding: chunked API API
API

CSS

http://i.meituan.com/firework/meituanxianshifengqiang Mac 4G

52>2017

1

<53 HTML 2 1 Dom Layout rem 2 HTML HTML DOM JSJavaScript 90ms...... header ..... <script> window.fk = function (callback) { require(['util/native/risk.js'], function (risk) { risk.getFk(callback); }); } </script> .... link script

54>2017 CSS JS CSS JS JS HTML CSS HTML CSS JS JS CSS JS HTML

14 JS

CSS HTML CSS CSS
CSS JS JS HTML
CSS JS JS

<55 JS JS " " + " "

BPM JS PC JS kb JS
CPU PC JS JS

JS /

window.test = function () { // test code } </script> <script> window.t2 = performance.now() test(); window.t3 = performance.now(); alert(" " + (t2 - t1)); alert(" " + (t3 - t2)); </script>

56>2017 test code t1~~t2 JS t2~~t3 test WebView iPhone6 iOS 10.2.1 OPPO R829T Android 4.2.2 ms/ ms WebView

React 50ms ~ 350ms

<57 App
JS + + KB Web

React JS App WebView nativeCPU
WebView trunk WebView WebView DNS

58>2017 WebView

WebView 1. 2. 3. 4. 5. 6. iOS Titans 10.0.7 OPPO R829T Android 4.2.2 10 WKWebView UIWebView Android WebView WebView UIWebView WebView Web- View

<59 WKWebView WebView WebView WebView native

WebView WebView
body CSS

WebView click 300ms
metaviewpoint Chrome Android fastclick /

PC native WebView Scroll Event WebView setTimeout setInterval GIF

60>2017 background-position: fixed WebView

iOS position: sticky Android touchmove position crash WebView API App
2M WebGL WebView WebView WebView WiFi

header

<61 iframe HTTPS DNS

CSPContent Security Policy CSP
inline
HTML+CSS CSP
CSP
HTTPS HTTPS HTTPS HTTPS HTTPS CSP HTTPS HTTPS

62>2017 App Socket HTTP App Socket WebView WiFi HTTPS Socket HTML HTML ...... WebView HTML HTML WebView CDN ...... MD5
WebView WebView schema URL URL URL

App WebView schema appxx://web?url={weburl} App schema appxx://web?url={some_ hack_weburl}

<63 some_hack_weburl some_hack_weburl

WebView WebView

  native                     WebView

WebView App
ReactNativeWeek
WebView App App

64>2017 SDK

SDK

<65 DNS JavaScript crossorigin

66>2017

PMBD RD PM RD shopId PMxxx RD

<67 HUNT SDK

68>2017 HUNT SDKWeb
SDK Web
SDK
SDK
EventData ErrorData DNS API

Node LocalStorage

<69

SDK

Web JsBridge
UAISP Web Web init getEnv init getEnv URL env freeze

70>2017

Localstorage Localstorage Localstorage Localstorage

<71

72>2017 DNS HTTPS
HTTPS HTTP Node HTML JavaScript

onloadwindow.onload 2. onload_timeout: onload 5

<73 3. asyncwindow.onload 5 4. hash_changeonhashchange 5

Window error Window error src href JavaScript crossorigin API XMLHttpRequest hook open URLsend status XMLHttpRequest.prototype.open = function open(method, url, bool) { monitor.originXHR.open.apply(this, [method, url, bool]); // get something... // this.ajaxUrl = url; } XMLHttpRequest.prototype.send = function send(_data) { const self = this; this.addEventListener('readystatechange', () => { if (self.readyState === 4) { if (self.status !== 200 && self.status !== 304 && this.ajaxUrl !== REPORT_URL) { // filter urls // report error info // ... // monitor.reporter.report(dataTypes.API_ERROR, error); } } }, false); monitor.originXHR.send.apply(this, [_data]); }; SDK

74>2017 URL domain Window error error Window click data DOMPath

SDK

SDK head
SDK
v1 SDK head webpack +loaders/plugins SDK head

HUNT Web

<75

API

SDK

76>2017

9 UI

 DNS 



GMV

<77

JS WebView Hybrid
NGINXNode CDN SSL

DNS DNS

(rank) "

78>2017 " "too young too simple, some times naive" "" bug

<79

"""" ...... """" Web
""""

80>2017

"" ReactAngularEmber SPA
" "React Angular
" "
SPA Cocoa View Controller " Cyra"

<81

SSR

82>2017

SSR JavaScript"JS" CSSHTML SSR JSP ASP SSR ASP.NET Web Form

CSS JS HTML JS HTML JS HTML JS SSR Node

<83 Node Node Node "" Node "" ""

84>2017 Node Node "" "" ""

""JS 2015 TypeScript"TS" Facebook Flow Flow TS TS 99% TS

<85

""case "Web " Web case case
Web "Freekite" case case case Mock Mock case
case Freekite case Freekite Freekite Case

86>2017

"" 95%
div
NGINX CDN SRE SRE
DNS DNS DNS HTTP
HTTPS HTTP HTTPS HTTP HTTP DNS HTTP HTTPS DNS HTTPS SSL HTTPS 99.96% 96%

<87 JS Error Performance CAT DNS

88>2017

"" Mixpanel
"" GrowingIO

<89

EventInfo eventInfo = new EventInfo(); eventInfo.nm = EventName.MGE; eventInfo.val_bid = "xxx";

// MGE //

90>2017 eventInfo.val_lab = new HashMap<>(); // eventInfo.val_lab.put(Constants.Business.xx,"xxx"); Statistics.getChannel("hotel").writeEvent(eventInfo); ""

Android Android UI TextViewLinearLayoutListViewViewPager ---- Android v7 AppCompatDelegate UI public class GAAppCompatDelegateV14 extends AppCompatDelegateImplV14 { @Override View callActivityOnCreateView(View parent, String name, Context context, AttributeSet attrs) { switch (name) { case "TextView": return new NovaTextView(context, attrs); } return super.callActivityOnCreateView(parent, name, context, attrs); } } Activity getDelegate AppCompatDelegate UI

<91 @Override public AppCompatDelegate getDelegate() { if (mDelegate == null) { mDelegate = GAAppCompatUtil.create(this, this); } return mDelegate; } UI UI UI Gradle UI UI apply plugin: 'com.meituan.judasplugin'

GAHelper.bindClick(view, bid, lab); iOS iOS Objective-C UI UIControl action

(void)nvja_setAnalyticsParams:(NVJAMGEParameter *)params mgeType: (SAKStatisticsEventMGEType)type { if (self.wmja_clickParams == nil && type == SAKStatisticsEventClick) { [self addTarget:self action:@selector(wmja_controlDidTapped:) forControlEvents:UIControlEventTouchUpInside]; } [super nvja_setAnalyticsParams:params mgeType:type]; }

92>2017 UITableView UITableViewDelegate

(void)forwardInvocation:(NSInvocation *)anInvocation { SEL selector = [anInvocation selector]; if (self.originalDelegate && [self.originalDelegate respondsToSelector: selector]) { [anInvocation invokeWithTarget:self.originalDelegate]; } SEL nvjaSelector = [self nvjaSelector:selector]; if ([super respondsToSelector:nvjaSelector]) { [anInvocation setSelector:nvjaSelector]; [anInvocation invokeWithTarget:self]; } }

NVJAMGEParameter *parameter = [[NVJAMGEParameter alloc] init]; parameter.bid = @"bid"; parameter.lab = @{@"poi_id":@"1"}; button.nvja_clickParams = parameter;

bid

<93

ID Index Index bid bid

94>2017

API
URI Scheme PV PV " " "" 70% 30%

<95

96>2017 Shield

Android iOS

Shield UI NativeAndroid&iOS UI UI GitHub https:// github.com/Meituan-Dianping/Shield

<97

Shield Agent UI
UI
MVP MVVM Shield MVP Shield MVP MVVM

98>2017 UI Shield Context UI Shield Agent public interface AgentInterface { void onCreate(Bundle savedInstanceState); void onStart(); void onResume(); void onPause(); void onStop(); void onDestroy(); Bundle saveInstanceState(); void onActivityResult(int requestCode, int resultCode, Intent data); String getIndex(); void setIndex(String index); String getHostName(); void setHostName(String hostName); SectionCellInterface getSectionCellInterface(); String getAgentCellName(); } Agent

1. SectionCellInterface

<99 SectionCellInterface Section UI Row public interface SectionCellInterface { int getSectionCount(); int getRowCount(int sectionPosition); int getViewType(int sectionPosition, int rowPosition); int getViewTypeCount(); View onCreateView(ViewGroup parent, int viewType); void updateView(View view, int sectionPosition, int rowPosition, ViewGroup parent); }

Shield Config ConfigShield
Shield AgentManagerCellManager AgentManager CellManagerSectionCellInterface

100>2017

RxJava Shield WhiteBoardWhiteBoard WhiteBoard

<101 Shield

App

Shield Native loading App

ReactNative JSBridge

102>2017

ReactNative

Activity+Fragment+Agent Tab Team Shield

<103

App
" 3 57% " "Amazon 1 16 " ""

104>2017

GFW DNS HTTP Socket HTTP HTTP Post URL header HTTPS

<105 App

HTTP

App API search.api.dianping.com ad.api.dianping.com tuangou.api.dianping.com waimai.api.dianping.com

106>2017 movie.api.dianping.com ... App HTTP DNS IP

SLB(Server Load Balancing) URL "" URL "http://ad.api.dianping.com/command?param1=123" "http://api.dianping.com/ad/command?param1=123" "ad.api.dianping.com" "api.dianping.com" "ad"

<107 URL "api.dianping.com" path SLB "" "http://api.dianping.com/ad/command?param1=123" SLB "http://ad.api.dianping.com/command?param1=123" SLB 1. DNS DNS 2. Keep-Alive Http 3. IP "api.dianping.com" DNS "api.dianping.com" IP IP IP HTTPS IP "api.dianping.com" IP "1.23.456.789"

108>2017 URL"http://api.dianping.com/ad/command?param1=123" "http://1.23.456.789/ad/command?param1=123" IP 1. DNS DNS 2. DNS 3. IP App HttpDNS http://www.tuicool.com/articles/7nAJBb HTTPS HTTPS IP HTTPS http://blog. csdn.net/github_34613936/article/details/51490032 IP HTTP 95% 97.5% 1500 1000

HTTP/2
HTTP/2 HTTP/2

<109

DNS DNS 2. 3.
HTTP/2
HTTP/2
TCP HTTP TCP HTTP
HTTP/2 1. DNS IP DNS HTTP DNS DNS DNS DNS 2. 3.

110>2017 IP
4. SDK HTTP 5. WNS https://www.qcloud.com/product/wns WNS 99.6% PS 99.9% WNS

<111

TCP

Failover

112>2017 TCP Failover TCP UDP HTTP Failover

SDKSDK WNS

<113 SDK

CIP CIP CIP China Internet Plus App CIP TCP CIP Connection Server API Server TCP CIP UDP HTTP HTTP CIP 99.7% 350
WNS WNS SDK CIP WNS
HTTP HTTP API Server CDN HTTP
Push Server TCP SDK

114>2017 81% 99%

Failover 2015 Android Failover Failover

<115 UDP Failover iOS Failover

SDK CIP|WNS|HTTP
App SDK API

116>2017 WNS WNS 5 WNS

App 1.4% 160 TCP HTTP/2
SDK ABTest iOS Android Accept-Encoding Content-Type HTTP "Set-Cookie" "Cookie" header

<117

   Cat Cat  http://github.com/dianping/cat    IP     HTTP/2  WNS

SDK

118>2017 iPhone X starzhang

iPhone X 9 13 iPhone X App Tabbar App

1.1 App

1.2

<119

1.3

1.4" Tab"

1.1 1.2 "" " Tab" 1.3 1.4 UIBarButtonItem UI Bug developer.apple.com HIG (Human Interface Guideline) WWDC App

120>2017 HIG

2.1 iPhone iPhone X iPhone X iPhone 8 145pt 20% 2.2iPhone X

<121

2.2 Like this 2.3CGRectMake(0,0,100,100) iPhone X

122>2017 2.4iPhone X Status Bar iPhone X StatusBar iPhone 20pt frame (tu) (xue) iPhone X [[UIApplication sharedApplication] statusBarFrame] 44pt 2.5iPhone X " App StatusBar iPhone X " iPhone X StatusBar

<123

Home iPhone X - Home Indicator 34pt

2.6iPhone X Home Indicator

" TabBar Home Indicator TabBar feed feed " TabBar barTintColor Setting indicator UI

SafeArea iOS 11 iOS 7 topLayoutGuide/bottomLayoutGuide safeLayoutGuide UI

2.7 TabBar Home Indicator

124>2017 barNavgationBarToolBarTabBarStatusBar 2.8iPhone SafeArea AutoLayout safeAreaLayoutGuide safeLayoutGuide SafeArea 2.9 Guide self.additionalSafeAreaInsets

<125 2.9 SafeArea self.additionalSafeAreaInsets = UIEdgeInsetsMake(64, 0, 0, 0)

Aspect Ratio iPhone X
App Session 201 xib iPhone X UseSafeAreaLayoutGuides iOS 9 App 3.1 xib

126>2017 SearchViewController 3.2iOS 11 UISearchViewController iOS 11 NavigationBar SearchViewController UITableViewSenctionHeader 3.3iOS11 Tableview UITableViewCell Cell ContentView inset SafeArea Tableview contentView inset To SafeArea contentview cell contentView layoutMargin SafeArea

<127 UITableViewHeaderFooterView.backgroundView backgroundColor 3.4iOS11

    "       "           20pt     [[UIApplication sharedApplication] statusBarFrame].size.height]

128>2017 iOS 11 iPhone X iOS 11

4.1 iOS 11 NavigationBar

4.2 iOS 11 NavigationBar

_UIButtonBarStackView PFBNavigationBarContainerView X " Tab" Tableview Tableview

4.3iOS11 " Tab" Tableview iOS 11 scrollview adjustedContentInset

4.4iOS 11 ScrollView contentoffset.y

<129 4.5 iPhone X Tableview frame safeArea adjustedContentInset safeAreaInset Tableview 4.6iPhone X safeAreaInset adjustedContentInset readOnly Tableview.contentInsetAdjustmentBehavior=UIScrollViewContentInsetAdjustmentNever tableview. contentOffset " Tab" AutoLayout UIBarButtonItem UIBarButtonItem customView Button Button frame AutoLayout UIBarButtonItem UIBarButtonSystemItemFixedSpaceiOS 11 fixedspace trick https://forums. developer.apple.com/thread/80075 CustomView CustomButton CustomView Button lefttop size

130>2017 4.7

// offset if (@available(iOS 11.0, *)) { self.contentViewController.tableView.contentInsetAdjustmentBehavior = UIScrollViewContentInsetAdjustmentNever; } // UIBarButtonItem if (@available(iOS 11.0, *)) { [messageButtonsContainerView mas_makeConstraints:^(MASConstraintMaker *make) { make.size.mas_equalTo(CGSizeMake(themeButton.width + settingButton.width + messageButton.width, 44)); }]; [themeButton mas_makeConstraints:^(MASConstraintMaker *make) { make.top.equalTo(messageButtonsContainerView); make.left.equalTo(messageButtonsContainerView); make.size.mas_offset(CGSizeMake(44, 44)); }]; [settingButton mas_makeConstraints:^(MASConstraintMaker *make) {

<131 make.top.equalTo(messageButtonsContainerView); make.left.equalTo(themeButton.mas_right); make.size.mas_offset(CGSizeMake(44, 44)); }]; [messageButton mas_makeConstraints:^(MASConstraintMaker *make) { make.top.equalTo(messageButtonsContainerView); make.left.equalTo(settingButton.mas_right).offset(-10); make.size.mas_offset(CGSizeMake(44, 44)); }]; } UIBarButtonItem *rightBarItem = [[UIBarButtonItem alloc] initWithCustomView:messageButtonsContainerView]; self.navigationItem.rightBarButtonItems = @[rightBarItem];

Xcode 9 GM (9A235)

StatusBar 2. UI(titleViewUIBarButtonItem) iPhone 6s iOS 11 AppStore iOS 10 SDK 3. safeAreaInset Scrollview Tabbar

132>2017 KIF iOS UI

UI iOS KIF UI

iOS UI UI Automation Apple UI JavaScript UI UI UI Automation UI Automation UI Automation UI Automation JavaScript UI Automation TuneupJsynm3k UI Automation UI Automation TCP UI Automation JavaScript iOSDriverAppium API API Apple API UI KIF Server App Server UI FrankCalabash Xcode 7 Apple UI ----UI Testing XCTest

<133 UI API UI AutomationAppium KIFFrankUI Testing iOS Objective--C XCTest UI API UI UIWebView App UI UI
KIF KIF XCTest iOS KIF KIF iOS UI KIF KIF Apple accessibility attributes Xcode XCTest

134>2017 XCTest command line build
KIF

KIF KIF KIF Xcode KIF XCTest XCTest XCTest Cocoa Touch Testing Bundle Target Target " Build Phases" Target Dependencies UI App Target Test Target Target Dependencies " Build Settings": "Bundle loader"$(BUILT_PRODUCTS_DIR)/MyApp.app/MyApp "Test Host"$(BUILT_PRODUCTS_DIR) "Wrapper Extensions"xctest KIF KIF
KIF AccessibilityLabel, AccessibilityIdentifier, AccessibilityTraits,Value... accessibility cell cell accessibility "Section XX Row XX" cell Accessibility Inspector

<135 KIF accessibility

(void)tapViewWithAccessibilityLabel:(NSString *)label - (UIView *)waitForViewWithAccessibilityLa- bel:(NSString *)label Target KIFTestCase
KIF Product->Test ( U)

1accessibility accessibility Apple UI UIKit

136>2017

Accessibility Inspector accessibility AccessibilityLabel VoiceOver AccessibilityIdentifier

2 UI ( KIFUITestActor.h )

tapThisView:

(void)tapViewWithAccessibilityLabel:

(NSString *)label;

waitForView:

(UIView *)waitForViewWithAccessibilityLabel:

(NSString *)label;

View

enterTextIntoView:

(void)enterText:(NSString *)text

intoViewWithAccessibilityLabel:(NSString *)label;

tapRowOnTableView:

(void)tapRowAtIndexPath:(NSIndexPath *)

indexPath inTableViewWithAccessibilityIdentifier:(NSString *)identifier

NS_AVAILABLE_IOS(5_0);

dismisses a system alert: - (void)acknowledgeSystemAlert;

KIFUITestActor KIFUITestActor AccessibilityIdentifier Label tapThisView waitForView

( KIFTestCase.h )

(void)beforeAll; test case test case
(void)beforeEach; test case test case - (void)afterEach; test case

<137

App test case test case

(void)afterAll; test case App

( KIFSystemTestActor.h
(void)simulateDeviceRotationToOrientation:

(UIDeviceOrientation)orientation;

(void)captureScreenshotWithDescription:

(NSString *)description;

3 :

a. b. c. App

ac beforeEachafterEach

a. b. c. App

ac beforeAllafterAll #import " TimerTests.h" #import " KIFUITestActor+AccessibilityLabelAddition.h" #import " KIFUITestActor+IdentifierAdditions.h"

138>2017 #import "KIFUITestActor+TimerAdditions.h" @implementation TimerTests - (void)beforeAll { [tester setDebugModel]; } - (void)afterAll { [tester resetDebugModel]; [tester clearHistory]; } - (void)beforeEach { [tester setDebugModel]; } - (void)afterEach { [tester clearParams]; } - (void)testNameedTask { [tester enterText:@"myTask" intoViewWithAccessibilityLabel:@"Task Name Input"]; [tester enterWorktime:10 Breaktime:4 Repetitions:5]; [tester tapViewWithAccessibilityLabel:@"Start Working"]; [tester waitForViewWithAccessibilityLabel:@"myTask"]; [tester waitForViewWithAccessibilityLabel:@"Start Working"]; } - (void)testnoNameTask { [tester enterWorktime:10 Breaktime:4 Repetitions:5]; [tester tapViewWithAccessibilityLabel:@"Start Working"]; [tester waitForViewWithAccessibilityLabel:@"myTask"]; [tester waitForViewWithAccessibilityLabel:@"Start Working"]; } - (void)testPresetTask { [tester tapViewWithAccessibilityLabel:@"Presets"]; [tester tapRowAtIndexPath:@"Classic" inTableViewWithAccessibilityIdentifier:@"Presets List"]; [tester tapViewWithAccessibilityLabel:@"Start Working"]; [tester waitForViewWithAccessibilityLabel:@"myTask"]; [tester waitForViewWithAccessibilityLabel:@"Start Working"]; } @end

selenium pageObject

<139 a. b. c. test suite test suite additions KIF KIF API Tools 4 retry App case case retry xctool xctool build-tests app app

140>2017 run-tests xctool
xctool build-tests -workspace myApp.xcworkspace -scheme myKIFTestScheme -sdk iphonesimulator -configuration Debug -destination platform='iOS Simulator',OS=8.3,name='iPhone 6 Plus' array=( TimerTests HistoryTests ) for data in ${array[@]} do xctool -reporter pretty -reporter junit:tmp/test-report-tmp. xml -workspace myApp.xcworkspace -scheme myKIFTestScheme run-tests -only myKIFTestTarget:${data} -sdk iphonesimulator -configuration Debug -destination platform='iOS Simulator',OS=8.3,name='iPhone 6 Plus' done KIF

UI

Martin Fowler

<141 UI UI
2. Jenkins KIF UI Jenkins
Jenkins Job Job salve maven ant
Job 3 " Build after other project are build" build Job UI " Build periodically" Job Daily Build

142>2017 "Poll SCM" Jenkins
"Poll SCM" pull request Job Job pull request daily build Job "Build periodically" daily build Job xcodebuild/xctool Job Job xcodebuild/xctool XML Jenkins JUnit Plugin XML Cobertura plugin XML Job
3. KIF Jenkins (1) UI

<143 Job Job Job app Job
(2) 'beijing' Failed to get text in field; instead, it was`beiji' KIF KIF KIFTypist.m (NSTimeInterval) keystrokeDelay
(3) KIF acknowledgeSystemAlert acknowledgeSystemAlert KIF KIF acknowledgeSystemAlert while

Automate UI Testing in iOShttps://developer.apple.com/library/tvos/documentation/ DeveloperTools/Conceptual/InstrumentsUserGuide/UIAutomation.html
Appium http://appium.io/slate/cn/v1.2.0/?ruby#appium 3. Frank http://www.testingwithfrank.com/ 4. KIF https://github.com/kif-framework/KIF 5. iOS UI Testing with KIFhttp://www.raywenderlich.com/61419/ios-ui-testing-with-kif 6. The current state of iOS automated functional testinghttp://watirmelon. com/2013/11/04/the-current-state-of-ios-automated-functional-testing/ 7. Page Objecthttp://martinfowler.com/bliki/PageObject.html 8. Test Pyramidhttp://martinfowler.com/bliki/TestPyramid.html

144>2017 9. Continuous Integrationhttp://www.martinfowler.com/articles/continuousIntegration. html 10. xcodebuildhttps://developer.apple.com/library/mac/documentation/Darwin/ Reference/ManPages/man1/xcodebuild.1.html 11. xctoolhttps://github.com/facebook/xctool 12. Jenkins https://wiki.jenkins-ci.org/display/JENKINS/Home 13. JUnit Pluginhttps://wiki.jenkins-ci.org/display/JENKINS/JUnit+Plugin 14. Cobertura pluginhttps://wiki.jenkins-ci.org/display/JENKINS/Cobertura+Plugin 15. Xcode 7 UI Testinghttps://developer.apple.com/videos/play/wwdc2015/406/

Android

<145

Android " " CPU/GPU
Android 6.0 App App API Android PNG XML/Java

Bitmap Bitmap

146>2017 3D GPU

CPU GPU CPUCentral Processing Unit GPUGraphics Processing Unit "" GPU CPU GPU Control CPU ALUArithmetic Logic Unit Cache DRAM RAM CPU ALU CPU 8086 16 1~3

<147 CPU CPU FPU CPU 100 CPU CPU GPU GPU ALUGPU ALU CPU GPU GPU GPU GPU

AB C ABC 32 CPU 32 0 1 Clock A B C

148>2017 8 CPU for 7 clock 8 A1~B4

<149 GPU 1GPU GPU 1

150>2017 Android Android View OpenGL ES Android 6.0 Java View DisplayList DisplayList Canvas drawXxx() Canvas(Java API) --> OpenGL(C/C++ Lib) --> --> GPU Android 4.1 DisplayList View ScaleAlphaTranslate GPU DisplayList RenderNode RenderNode DisplayList RenderNode View View View DisplayList

<151

152>2017 Android Android 6.0 View
ViewRootImpl.performTraversals PhoneWindow.DecroView. drawChild View Canvas Canvas Canvas.class DisplayListCanvas.class isHardwareAccelerated() falsetrue View View draw(canvas,parent,drawingTime) - draw(canvas) onDraw - dispachDraw - drawChild Draw Canvas.drawXxx() DisplayList View updateDisplayListIfDirty - dispatchGetDisplayList - recreateChildDisplayList DisplayList View DisplayList DisplayList View Android 6.0 DisplayList API "@hide" draw DisplayList
ThreadedRenderer.nSyncAndDrawFrame() GPU DisplayList

<153

154>2017 VS Android 6.0

1 View Draw Draw DisplayList GPU 2 TextView Layout TextView TextView View View View draw(canvas,parent,drawingTime) View TextView DisplayList Draw View DisplayList GPU 3 DisplayList DisplayList

<155 4 RenderNode invalidate View View Alpha onSetAlpha() true public class View { // ... public void setAlpha(@FloatRange(from=0.0, to=1.0) float alpha) { ensureTransformationInfo(); if (mTransformationInfo.mAlpha != alpha) { mTransformationInfo.mAlpha = alpha; if (onSetAlpha((int) (alpha * 255))) { // ... invalidate(true); } else { // ... mRenderNode.setAlpha(getFinalAlpha()); // ... } } } protected boolean onSetAlpha(int alpha) { return false; } // ... }

 View  clipChildren    true    View       View  clipChildren  false View  View

View invalidate layout View PFLAG_DIRTY View PFLAG_DIRTY_OPAQUE draw(canvas) View

156>2017 View View PFLAG_DIRTY clipChildren true ViewRoot Rect View View View View View clipChildren false ViewGroup.invalidateChildInParent() View

CPU GPU ALU
DisplayList CPU DisplayList GPU DisplayList CPU DisplayList DisplayList Shape Bitmap

GPU-- 2. ""GPU 3. Matlab GPU 4. CPU 5. CPU 6. 7. Android UI Display List 8. Android UI Display List 9. Android Choreographer 10. Android Project Butter

<157 Android Walle

Android 7.0Nougat APK Signature Scheme v2 Android -- APK Signature Scheme v2
APK Signature Scheme v2 Android 7.0Nougat APK Signature Scheme v2 APK Android Gradle 2.2.0 APK Signature Scheme v2 APK Signature Scheme v2 build.gradle v2SigningEnabled false
android { ... defaultConfig { ... } signingConfigs { release { storeFile file("myreleasekey.keystore") storePassword "password" keyAlias "MyReleaseKey" keyPassword "password" v2SigningEnabled false } } }

158>2017 Android 7.0 Nougat

ZIP Central Directory APK Signing Block ZIP APK APKZIP 1. Contents of ZIP entriesfrom offset 0 until the start of APK Signing Block 2. APK Signing Block 3. ZIP Central Directory 4. ZIP End of Central Directory 2APK Signing Block 1Contents of ZIP entries 3ZIP Central Directory 4 ZIP End of Central Directory 134 META-INF

<159 META-INF META-INF META-INF 134
Failure [INSTALL_PARSE_FAILED_NO_CERTIFICATES: Failed to collect certificates from base.apk: META-INF/CERT.SF indicates base.apk is signed using APK Signature Scheme v2, but no such signature was found. Signature stripped?] APK ZIP Comment APKTool Flavor BuildType
900 3
2----Block APK Signature Scheme v2 Block APK 134 2APK Signing Block 2APK Signing Block 2

160>2017 2 APK Signing Block 2 8 APK Sig Block 42+ ID-value ID-value 8 4 ID V2 ID0x7109871a ID-value ID-value ID-value

<161

APK Signing Block
APK Signing Block

Android Android ID-value

public static ByteBuffer findApkSignatureSchemeV2Block(

ByteBuffer apkSigningBlock,

Result result) throws SignatureNotFoundException {

checkByteOrderLittleEndian(apkSigningBlock);

// FORMAT:

// OFFSET

DATA TYPE DESCRIPTION

// * @+0 bytes uint64: size in bytes (excluding this field)

// * @+8 bytes pairs

// * @-24 bytes uint64: size in bytes (same as the one above)

// * @-16 bytes uint128: magic

ByteBuffer pairs = sliceFromTo(apkSigningBlock, 8, apkSigningBlock.

capacity() - 24);

int entryCount = 0; while (pairs.hasRemaining()) { entryCount++; if (pairs.remaining() < 8) { throw new SignatureNotFoundException( "Insufficient data to read size of APK Signing Block entry #" + entryCount); } long lenLong = pairs.getLong(); if ((lenLong < 4) || (lenLong > Integer.MAX_VALUE)) { throw new SignatureNotFoundException( "APK Signing Block entry #" + entryCount + " size out of range: " + lenLong); } int len = (int) lenLong; int nextEntryPos = pairs.position() + len; if (len > pairs.remaining()) { throw new SignatureNotFoundException( "APK Signing Block entry #" + entryCount + " size out of range: " + len

162>2017

", available: " + pairs.remaining()); } int id = pairs.getInt(); if (id == APK_SIGNATURE_SCHEME_V2_BLOCK_ID) { return getByteBuffer(pairs, len - 4); } result.addWarning(Issue.APK_SIG_BLOCK_UNKNOWN_ENTRY_ID, id); pairs.position(nextEntryPos); } throw new SignatureNotFoundException( "No APK Signature Scheme v2 block in APK Signing Block"); } if (id == APK_SIGNATURE_SCHEME_V2_ BLOCK_ID) {return getByteBuffer(pairs, len - 4);} Android ID APK_SIGNATURE_SCHEME_V2_BLOCK_ID = 0x7109871a ID-value APK Signature Scheme v2 Block ID-value APK Signature Scheme v2 ID ID-value
ID-value ZIP APK Signing Block APK ID-value public void writeApkSigningBlock(DataOutput dataOutput) { long length = 24; for (int index = 0; index < payloads.size(); ++index) { ApkSigningPayload payload = payloads.get(index); byte[] bytes = payload.getByteBuffer(); length += 12 + bytes.length; } ByteBuffer byteBuffer = ByteBuffer.allocate(Long.BYTES);

<163 byteBuffer.order(ByteOrder.LITTLE_ENDIAN); byteBuffer.putLong(length); dataOutput.write(byteBuffer.array()); for (int index = 0; index < payloads.size(); ++index) { ApkSigningPayload payload = payloads.get(index); byte[] bytes = payload.getByteBuffer(); byteBuffer = ByteBuffer.allocate(Integer.BYTES); byteBuffer.order(ByteOrder.LITTLE_ENDIAN); byteBuffer.putInt(payload.getId()); dataOutput.write(byteBuffer.array()); dataOutput.write(bytes); } ... }

APK ID-value ID value APK
APK ID-value
App ZIP EOCD(End of central directory)Central directory ZIP ID-value ZIP APK Signing Block Java Groovy

164>2017

ID-value Java 2. Gradle Android 3. ID-value Java 4. com.android.application AAR APK APK ID-value 30M APK 100
Walle Github https:// github.com/Meituan-Dianping/walle IssuePRStar APK Signature Scheme V2

ID-value ID-value

      https://source.android.com/security/apksigning/ v2.html

APK Signature Scheme v2 2. ApkSigner 3. apksig 4. ZIP Format

<165

C H5 App App PV ThunderJS

166>2017 Build Service

ThunderJS CMD ThunderJS Build Service C
ThunderJSThunderJS ThunderJS Build Service

ThunderJS

<167

ThunderJS JS LocalStorage Diff Patch LocalStorage

168>2017

1. 1. LocalStorage Diff Patch Diff Diff Web Web JSON Patch Diff

[ 'cmp/util.js', [0, 33], 'mn', [34, 10] ]

// // // //

0 33 mn 34 10 Patch Diff
Build Service Build Service CDN

<169 CDN CDN

O(N^2) Build Service Myers O(N^2) O(ND) Web D Myers O(N)

99% 95%

PV 2017 4 30 Build Service 3s 50%

170>2017 Build Service

Build Service Build Service Build Service Mixer Mixer
Mixer

Mixer Mixer 100% 50

Myers O(N^2)Mixer

<171 Node
Mixer CPU Mixer PM2 Mixer Mixer

ThunderJS

Mixer Mixer 10 30 3 10 Mixer

Mixer
ThunderJS createElement

172>2017 createElement XHR LocalStorage WiFi4G3G2Gunknown 2G unknown 10.35%

<173 50%+

Mixer CLOSE_WAIT CLOSE_WAIT Mixer Myers
Mixer 100% Mixer

Build Serivce Build Service Brain Brain

174>2017 Brain MySQL trace trace Brain Brain Brain
Brain Mixer 99.99% TP90 5800 ms 90 msBrain 10W+ Diff 50%

ThunderJS BuildService 3%

<175 Build Service
ThunderJS Git CommitId ThunderJS CommitId Hash Git Hash

176>2017 5 5 1.5% 10 1.1%

ThunderJS 90% 10% WiFi 49.37GB 33.41GB C

Android

<177

Bad-Case bug

onDestory receiver bug Android Android

178>2017

Git pull request

1. 1. merge merge commit

<179

Java A merge commit B

Android Studio JDT-View

180>2017 Eclipse JDT JDT JDT Java AB MD5MD5
JDT ASM ASM JDT Lambda ClassName ClassName ClassName RetroLambda ClassName ...$Index ...$InnerClassName RetroLambda ...$$Lambda$Index JDT Java visit(MethodDeclaration method) visit(AnonymousDeclaration method) endVisit(AnonymousDeclaration method) visit(TypeDeclaration node) endVisit(TypeDeclaration node) visit(LambdaExpress node) Lambda ClassName

<181

MethodInfo

public String className;//hash package

public String md5;

public String methodName;

public List paramList = new ArrayList<>();

public String methodBody; public boolean isLambda;

// Lambda

public int lambdaNumInClass; 1 .

// Class lambda .

public int totalLambdaInClass; // Class lambda

public String lambdaParent;

//lambda

public boolean isLambdaInAnonymous; // lambda public boolean isAnonymousClass; //

JaCoCo JaCoCo JaCoCo JaCoCo C0 coverage C1 coverage

182>2017

JaCoCo JaCoCo
Byte CodeByte Code Offline On-TheFly Offline Class
On-The-FlyJVM -javaagent Jar Instrumentation ClassLoader class class class JVM
On-The-Fly JVM

<183 Offline Offline 1. 2. 3. class

184>2017 JaCoCo ASM Probe BOOL true false

Probe Insertion Strategy

JaCoCo JaCoCo ClassProbesAdapter ASM visitMethod visitMethod ClassProbeVisitor visitMethod MethodInstrumenter @Override public final MethodVisitor visitMethod(final int access, final String name, final String desc, final String signature, final String[] exceptions) { final MethodProbesVisitor methodProbes; final MethodProbesVisitor mv = cv.visitMethod(access, name, desc, signature, exceptions); if (mv == null) { methodProbes = EMPTY_METHOD_PROBES_VISITOR; } else { methodProbes = mv; } return new MethodSanitizer(null, access, name, desc, signature, exceptions) { @Override public void visitEnd() { super.visitEnd(); LabelFlowAnalyzer.markLabels(this); final MethodProbesAdapter probesAdapter = new MethodProbesAdapter( methodProbes, ClassProbesAdapter.this); if (trackFrames) { final AnalyzerAdapter analyzer = new AnalyzerAdapter( ClassProbesAdapter.this.name, access, name, desc, probesAdapter); probesAdapter.setAnalyzer(analyzer); this.accept(analyzer); } else { this.accept(probesAdapter);

<185 } } }; } JaCoCo ClassIn- strumenter ClassProbesAdapter visitMethod

@Override public final MethodVisitor visitMethod(final int access, final String name, final String desc, final String signature, final String[] exceptions) { if (Utils.shoudHackMethod(name,desc,signature,changedMethods,cv. getClassName())) { ... } else { return cv.getCv().visitMethod(access, name, desc, signature, exceptions); } }

Analyzer

@Override public void analyzeClass(final ClassReader reader) { if (Utils.shoudHackMethod(reader.getClassName(),changedMethods)) { ... } } ReportClassProbesAdapter @Override public final MethodVisitor visitMethod(final int access, final String name, final String desc, final String signature, final String[] exceptions) { if (Utils.shoudHackMethod(name, desc, signature, changedMethods, this.className)) { ...

186>2017 } else { return null; } } commit FoodPoiDetailActivity onCreate initCustomTitle

<187 JDT vs ASM

JDT ASM classNamemethodName paramList public static boolean shoudHackMethod(String methodName, String desc, String signature, HashSet changedMethods, String className) { Map<String, List> changedLambdaMethods = getChangedLambdaMethods(changedMethods); List changedLambdaMethodNames = changedLambdaMethods. get(className.replace(" /", ".")); updateLambdaNum(methodName, className); int indexMethods = 0; outer: for (; indexMethods < changedMethods.size(); indexMethods++) { MethodInfo methodInfo = changedMethods[indexMethods] if (methodInfo.className.replace(".","/").equals(className)) { if (methodName.startsWith(lambda$') && methodInfo.isLambda && changedLambdaMethodNames != null && changedLambdaMethodNames.size() > 0) { // if (methodInfo.methodName.equals(methodName)) { changedLambdaMethodNames.remove(methodInfo. methodName) return true; } else if (!changedLambdaMethodNames. contains(methodName)) { // , lambda , int lastIndex = methodInfo.methodName. lastIndexOf($'); if (lastIndex <= 0) { continue; } String tmpMethodName = methodInfo.methodName. substring(0, lastIndex); if (tmpMethodName.equals(sAsmMethodInfo.methodName) && (methodInfo.lambdaNumInClass == (methodInfo.totalLambdaInClass - sAsmMethodInfo.lambdaNumInClass + 1) || judgeSoleLambda(changedMethods, methodInfo, methodName, className. replace("/",".")))) { changedLambdaMethodNames.remove(methodInfo. methodName) return true; }

188>2017

}

} else {

if (methodInfo.methodName.equals(methodName) || (!methodInfo.methodBody.trim().equals("{}") && methodName.equals("") && methodInfo.methodName.equals(methodInfo. className.split("\.|\$")[methodInfo.className.split("\.|\$").

length - 1]))) {

if (signature == null) signature = desc;

TraceSignatureVisitor v = new

TraceSignatureVisitor(0);

new SignatureReader(signature).accept(v);

String declaration = v.getDeclaration(); int rightBrace = declaration.indexOf("("); int leftBrace = declaration.lastIndexOf(")");

if (rightBrace > 0 && leftBrace > rightBrace) { //

declaration = declaration.substring(rightBrace +

1, leftBrace);

"");

} // \[\] , String paraStr = declaration.replaceAll("[(){}]",

if (paraStr.length() > 0) {

String[] parasArray = getAsmMethodParams(paraStr. split(","), className, methodInfo.paramList);

List paramListAst =

getAstMethodParams(methodInfo.paramList);

if (parasArray.length == paramListAst.size()) {

for (int i = 0; i < paramListAst.size(); i++)

{ // < > .

String[] methodInfoParamArray = paramListAst.get(i).split("<|>|\.");

for (String param : methodInfoParamArray)

{

if (!parasArray[i].contains(param) ||

(parasArray[i]. contains(param) && parasArray[i].contains("[]") && !param. endsWith("[]"))) { // , ( , class , ) outer

continue outer;

}

} else {

continue;

}

<189

!= null) { methodName) } } } } return false; }

} if (methodInfo.isLambda && changedLambdaMethodNames changedLambdaMethodNames.remove(methodInfo. } return true;

Gradle DEX Debug

JaCoCo
org.jacoco.agent.rt.RT.getAgent().getExecutionData(false)

onDestory task

190>2017 JaCoCo mergeTask ClassId merge JaCoCo ClassId merge probe Pre-Push apply Pre-Push .git push 95%

Gradle push 95% abort

<191

JaCoCo-Source-Code 2. Java JaCoCo-

Android 2013 7 App 2015 2016

192>2017

App
App [1] [2] QA

[1] smoke testing [2]

Android iOS

<193

/ TDD[3] BDD[4]
Android iOS App [3] Test-driven development TDD [4] Behavior-driven development BDD

Calabash BDD

194>2017 test case Scenario: Then I press "" When I press view with id"city" Then I see "" When I press "" And I press view with id"start_search" When I enter "" into input field number 1 Then I press list item number 1 Then I see "" When I press "" ... FeatureScenarioStep BDD
Feature Scenario case test case Step Calabash ID UI
Android iOS iOS Calabash Android apk iOS iOS App iOS ID iOS ID

<195 xib ID iOS ID Calabash ID Android AndroidiOSQA Calabash

Calabash                QA    Appium  Appium   QA

Calabash BDD Appium Calabash Calabash

Calabash BDD Cucumber Calabash Cucumber BDD Appium Cucumber Appium Appium Cucumber

QA QA App Calabash

196>2017
App common_steps. rb cucumber.yml Android iOS

config/cucumber.yml ##YAML Template --ios: IDEVICENAME='ios' android: IDEVICENAME='android'

Android/config iOS/config Android iOS support Appium env.rb

<197

class AppiumWorld end if ENV[IDEVICENAME']=='android' caps = Appium.load_appium_txt file: File.expand_path("./../android/ appium.txt", __FILE__), verbose: true elsif ENV[IDEVICENAME']=='ios' caps = Appium.load_appium_txt file: File.expand_path("./../ios/ appium.txt", FILE), verbose: true else caps = Appium.load_appium_txt file: File.expand_path(`./', FILE), verbose: true end Appium::Driver.new(caps) Appium.promote_appium_methods AppiumWorld World do AppiumWorld.new end

cucumber -p android/ios Cucumber Calabash Calabash Calabash Steps FeatureScenarioStep Calabash Steps appium_lib Steps selenium-cucumber testdata.rb

gem ` appium_lib',

`~> 9.4.2'

gem ` rest-client',

`~> 2.0.2'

gem ` rspec',

`~> 3.5.0'

gem ` cucumber',

`~> 2.4.0'

gemrspec-expectations',~> 3.5.0'

gem ` spec',

`~> 5.3.4'

gemselenium-cucumber', ~> 3.1.5'

198>2017

RD QA QA

Appium Calabash ID QA Appium Webview Calabash Webview Native Appium
iOS Android iOS Android apk iOS ipa Calabash iOS

Steps App iOS ID Appium IDclassnameXPath XPath XPath Then I press view with xpath"//android.widget.LinearLayout[1] /android.widget.FrameLayout[1]/android.widget.LinearLayout[1]

<199 /android.widget.FrameLayout[1]/android.widget.LinearLayout[2] /android.widget.FrameLayout[1]/android.widget.ListView[1] /android.widget.LinearLayout[1]/android.widget.LinearLayout[1]" XPath Appium-inspector XPath

Calabash Steps Calabash Appium BDD Calabash homepage homepage

200>2017

Jenkins Cucumber --format html --out reports.html --format pretty HTML JSON Jenkins

<201

Calabash Calabash

202>2017 7 30 Scenario

20 QA

scroll or swipe UIAutomation Android scroll_ uiselector Then /^I scroll to view with text"([^\"]*)"$/ do |value| text = %Q("#{value}") args = scroll_uiselector("new UiSelector().textContains(#{text})") find_element :uiautomator, args end UI App UIAutomation2 swipe swipe start_x: start_x, start_y: start_y, end_x: start_x, end_y: start_y - pixel.to_i Appium v1.6.5 UIAutomation2 swipe scroll

<203 UIAutomation2 Toast UIAutomation2 Scenario test case Scenario test case Scenario Scenario App App

Jenkins job job App

Appium Doc 2. appium/ruby_lib docs 3. selenium-cucumber-ruby Canned Steps

Android Android

204>2017 App Spider

3 1. Spider 2. 3. Spider Spider

API 2. 3.

<205

App MOCK API
API

206>2017
Spider 1

App App Web API ServerAPI Server App App DB App

Spider

<207

App API "Spider " App App SpiderSpider App Spider API API App

208>2017 App API Spider

Spider QA

<209 Spider Spider Spider App App API Spider App APP Debug App Spider .com Spider Spider

210>2017 App Spider Spider Spider URLScheme URLScheme App SpiderSpider Spider Spider "" Spider MOCK Spider
App UI

<211 Spider UI
UI Spider Spider

30 Eric Evans Domain-Driven Design DDD""DDD "" DDD

CRUD

<213

---- idea

214>2017 ...... DDD

Anemic Domain Object J2EE Action/Service/DAO OO ER

<215 AwardPool Award get set class AwardPool { int awardPoolId; List awards; public List getAwards() { return awards; } public void setAwards(List awards) { this.awards = awards; } ...... } class Award { int awardId; int probability;// ...... } Service LotteryService drawLottery() AwardPool awardPool = awardPoolDao.getAwardPool(poolId);//sql AwardPool for (Award award : awardPool.getAwards()) { // award.getProbability() award } Service Award
AwardPool

216>2017 DDD

    DDD  DDD

DDD
MVC DDD

<217 ---- ---- ---- PHP DDD
DDD DDD

218>2017

DDD

<219

Eric Evans Vaughn Vernon

C M

220>2017 M C C

<221 /

C

DDD

·

222>2017 1.
2.
Partnership Shared Kernel - Customer-Supplier Development Conformist Anticorruption Layer Open Host Service Published Language OHS Big Ball of Mud SeparateWay

<223 ""PartnerShip PS Anticorruption LayerACL Open Host Service Published Language Data Coupling

DDD

224>2017 Entity

Value Object {"name":" ""css":"#000000"}

Aggregate( Aggregate Root

<225

List< > 1:N id id

226>2017 (DrawLottery) IDLotteryIdAwardPool UserGroupAward SendResult UserLotteryLog

DDD DDD

Module DDD

<227 {com. . . . .} import com.company.team.bussiness.lottery.;// import com.company.team.bussiness.riskcontrol.;// import com.company.team.bussiness.counter.;// import com.company.team.bussiness.condition.;// import com.company.team.bussiness.stock.;// 1

import com.company.team.bussiness.lottery.domain.valobj.;// - import com.company.team.bussiness.lottery.domain.entity.;// - import com.company.team.bussiness.lottery.domain.aggregate.;// import com.company.team.bussiness.lottery.service.;// import com.company.team.bussiness.lottery.repo.;// import com.company.team.bussiness.lottery.facade.;// 2

DrawLotteryAwardPool id DrawLotteryContext chooseAwardPool chooseAwardPool DrawLotteryContext DrawLottery AwardPool

228>2017 package com.company.team.bussiness.lottery.domain.aggregate; import ...; public class DrawLottery { private int lotteryId; // id private List awardPools; // //getter & setter public void setLotteryId(int lotteryId) { if(id<=0){ throw new IllegalArgumentException(" id"); } this.lotteryId = lotteryId; } // context public AwardPool chooseAwardPool(DrawLotteryContext context) { if(context.getMtCityInfo()!=null) { return chooseAwardPoolByCityInfo(awardPools, context.getMtCityInfo()); } else { return chooseAwardPoolByScore(awardPools, context.getGameScore()); } } // private AwardPool chooseAwardPoolByCityInfo(List awardPools, MtCifyInfo cityInfo) { for(AwardPool awardPool: awardPools) { if(awardPool.matchedCity(cityInfo.getCityId())) { return awardPool; } } return null; } // private AwardPool chooseAwardPoolByScore(List awardPools, int gameScore) {...} } 3DrawLottery

AwardPool package com.company.team.bussiness.lottery.domain.valobj; import ...;

<229 public class AwardPool { private String cityIds;// private String scores;// private int userGroupType;// private List awards;// // public boolean matchedCity(int cityId) {...} // public boolean matchedScore(int score) {...} // public Award randomGetAward() { int sumOfProbablity = 0; for(Award award: awards) { sumOfProbability += award.getAwardProbablity(); } int randomNumber = ThreadLocalRandom.current().nextInt(sumOfProbablity); range = 0; for(Award award: awards) { range += award.getProbablity(); if(randomNumber<range) { return award; } } return null; } } 4AwardPool gettersetter **Service

Repository

230>2017 // import com.company.team.bussiness.lottery.repo.dao.AwardPoolDao;// - import com.company.team.bussiness.lottery.repo.dao.AwardDao;// - import com.company.team.bussiness.lottery.repo.dao.po.AwardPO;// - import com.company.team.bussiness.lottery.repo.dao.po.AwardPoolPO;// - import com.company.team.bussiness.lottery.repo.cache.DrawLotteryCacheAccessObj; // - import com.company.team.bussiness.lottery.repo.repository.DrawLotteryRepository; // - 5Repository Repository

Cache Aside Pattern

package com.company.team.bussiness.lottery.repo; import ...; @Repository public class DrawLotteryRepository { @Autowired private AwardDao awardDao; @Autowired private AwardPoolDao awardPoolDao; @AutoWired private DrawLotteryCacheAccessObj drawLotteryCacheAccessObj; public DrawLottery getDrawLotteryById(int lotteryId) { DrawLottery drawLottery = drawLotteryCacheAccessObj.get(lotteryId); if(drawLottery!=null){ return drawLottery; }

<231 drawLottery = getDrawLotteyFromDB(lotteryId); drawLotteryCacheAccessObj.add(lotteryId, drawLottery); return drawLottery; } private DrawLottery getDrawLotteryFromDB(int lotteryId) {...} } 6DrawLotteryRepository

(UserCityInfoFacade) (LotteryContext) (MtCityInfo) package com.company.team.bussiness.lottery.facade; import ...; @Component public class UserCityInfoFacade { @Autowired private LbsService lbsService;// RPC public MtCityInfo getMtCityInfo(LotteryContext context) { LbsReq lbsReq = new LbsReq(); lbsReq.setLat(context.getLat()); lbsReq.setLng(context.getLng());

232>2017 LbsResponse resp = lbsService.getLbsCityInfo(lbsReq); return buildMtCifyInfo(resp); } private MtCityInfo buildMtCityInfo(LbsResponse resp) {...} } 7UserCityInfoFacade

issueLottery

package com.company.team.bussiness.lottery.service.impl import ...; @Service public class LotteryServiceImpl implements LotteryService { @Autowired private DrawLotteryRepository drawLotteryRepo; @Autowired private UserCityInfoFacade UserCityInfoFacade; @Autowired private AwardSendService awardSendService; @Autowired private AwardCounterFacade awardCounterFacade; @Override public IssueResponse issueLottery(LotteryContext lotteryContext) { DrawLottery drawLottery = drawLotteryRepo.getDrawLotteryById (lotteryContext.getLotteryId());// awardCounterFacade.incrTryCount(lotteryContext);// AwardPool awardPool = lotteryConfig.chooseAwardPool(bulidDrawLotteryContext (drawLottery, lotteryContext));// Award award = awardPool.randomChooseAward();// return buildIssueResponse(awardSendService.sendAward(award, lotteryContext));// }

<233 private IssueResponse buildIssueResponse(AwardSendResponse awardSendResponse) {...} } 8LotteryService

DTO DO

234>2017 PODTO DO DO PO

HTTP -

Vernon Vernon

<235

CQRS

236>2017

package ...; import ...; @Service public class LotteryApplicationService { @Autowired private LotteryRiskService riskService; @Autowired private LotteryConditionService conditionService; @Autowired private LotteryService lotteryService; // public Response<PrizeInfo, ErrorData> participateLottery(LotteryContext lotteryContext) { // validateLoginInfo(lotteryContext); // RiskAccessToken riskToken = riskService.accquire(buildRiskReq (lotteryContext)); ... // LotteryConditionResult conditionResult = conditionService. checkLotteryCondition(otteryContext.getLotteryId(),lotteryContext. getUserId()); ... // IssueResponse issueResponse = lotteryService.issurLottery (lotteryContext); if(issueResponse!=null && issueResponse.getCode()==IssueResponse. OK) { return buildSuccessResponse(issueResponse.getPrizeInfo()); } else { return buildErrorResponse(ResponseCode.ISSUE_LOTTERY_FAIL, ResponseMsg.ISSUE_LOTTERY_FAIL) } } private void validateLoginInfo(LotteryContext lotteryContext){...} private Response<PrizeInfo, ErrorData> buildErrorResponse (int code, String msg){...}

<237 private Response<PrizeInfo, ErrorData> buildSuccessResponse (PrizeInfo prizeInfo){...} } 9LotteryApplicationService

DDD
SmartUI DDD
DDD DDD DDD

Eric Evans. . . 2016. 2. Vaughn Vernon. . . 2014.

wenbin.lu@dianping.com

238>2017 MGW

MGW LVS Gbps MGW

<239 DNS IP DNS DNS

240>2017 OSI

OSI TCP/IP
DR NAT TUNNEL FULLNAT DR MAC VIP NAT IP IP IP VIP IP IP IP

<241 IP VIP NAT
TUNNEL DR TUNNEL TUNNEL FULLNAT NAT SNAT SNAT SNAT IP FULLNAT localip SNAT IP localip IP IP localip localip IP localip IP localip localip

242>2017 IP FULLNAT NAT SNAT SNAT NAT FULLNAT MGW

LVS

bug
LVS LVS+Nginx LVS Nginx LVS MGW LVS MGW LVS MGW

<243

LVS 600 6 100 cache miss LVS netfilter netfilter LVS LVS
kernel bypass DPDK PMD DPDK numamemory channelDDIO DPDK MGW

244>2017

MGW RSSReceive Side ScalingRSS CPU CPU MGW FULLNAT FULLNAT RSS CPU session
SNAT lport0 RSS(cip0, cport0, vip0, vport0) = RSS(dip0, dport0, lip0, lport0) CPU localip SNAT

<245 IP CPU localip lip CPU IP flow director session flow director RSS CPU localip cpu0 lip0 cpu1 lip1 cpu2 lip2 cpu3 lip3 cip0, cport0, vip0, vport0 RSS 0 cpu0 cpu0 fullnat cpu0 localip lip0lip0, lport0, dip0, dport0 dip0, dport0, lip0, lport0 flow director IP lip0 0 0 cpu0 CPU session

246>2017

CPU CPU CPU CPU CPU CPU Linux kernelSSH CPU MGW MGW MGW MGW

<247

MGW OSPF+ECMP ECMP OSPF ecmp failover ecmp MGW session

248>2017 hash session
session session session

<249 100ms 0 MGW 0 0 500ms 500ms

250>2017 session session MGW MGW MGW session session
request session session session session finish finish session finish session finish session

<251

MGW

RS

252>2017 RS MGW RS RS RS MGW MGW RS MGW RS RS
IP Hash IP Hash IP Hash IP Hash
hash

Hash

<253 2. MGW MGW MGW MGW IP Hash Google Maglev Hash paper

MGW overlay Web MGW MGW

DPDK. 2. LVS. 3. Eisenbud D E, Yi C, Contavalli C, et al. Maglev: A Fast and Reliable Software Network Load Balancer.

254>2017 Docker

Docker "" 2015 Docker 45 Web

O2O
" " IT

<255 IT
2015 Docker Docker KubernetesDocker Swarm Docker

Web master CPU SSH "" Docker
SLA 99.99% 3 1. 2. 3.

256>2017

1 PaaS IaaS
PaaS HTTP API

<257 HTTP API Web IaaS API PaaS HTTP API Docker HostServerDockerVolume OVS Cgroup API HostServer API API PaaS API API set1. 2. API API Host-SRV API 3 Docker Host-SRV Host-SRV
Host-SRV Docker Load Docker Host-SRV Unix Socker Docker Daemon

258>2017 Logsexec Rootfs Volume Host-SRV Docker Daemon Unix Socket Dock- er-Containerd Host-SRV

Docker Registry: Docker Hub Mirror
Glance: Openstack Glance Docker Docker Glance Glance UUID Glance API UUID Docker ID Parent ID Glance Glance Parent ID Glance Docker Glance Docker Docker
Glance Glance Glance Docker "" API Docker Glance UUID Docker Image ID Docker

<259 Glance "" Glance 200 Glance 16,000 Docker

Docker NoneBridgeContainer Host 2015 Docker 1.9 Libnetwork Docker Docker Docker
2

260>2017 OVS-DPDK CPU OVS-DPDK OVS-DPDK CPU
OVS Controller OVS
MosBridge MosBridge None None

--net=None 2. eth-pair 3. eth-pair OVS Bridge 4. nsenter Namespace eth-pair
Namespace IP None Docker Host-SRV Host-SRV VPC Host-SRV Docker LibnetworkLibnetwork Docker Libnetwork Docker IP Libnetwork MosBridge

<261 Docker --net=mosbridge IP OVS Bridge Docker MosBridge MosBridge MosBridge MosBridge Host-SRV Docker
Docker Docker Docker Volume mount bind "" Volume Volume 3LVM-Volume LVM Volume LVM VG Volume VG LV Volume LVM LVM

262>2017 Volume LVM Volume Volume LVM Volume LVM Linux Devicemapper Devicemapper Linux 2.6 IO

ZabbixFalcon CAT Agent

4

<263 Libcontainer Mos-DockerAgent procCGroup driver GO Libcontainer Docker Daemon Daemon

Agent Agent Docker

Docker Docker
Docker
Kubernetes Pod Pod Pod

264>2017 VolumeIPC Pod
Pod set set 5Set set / set BusyBox BusyBox set Volume IPC set Volume IPCset JSON ( 6 ) set Container ListContainer
Index ImageDocker Glance name ID Options CPU MEM set CPU 4 set CPU:80 3.2

<265 6set json set set set set set Busybox Privileged sysctl SSH Busybox Volume set Volume Busybox Volume set set set API set

266>2017 Docker MosDocker Docker 2 4 Docker LTS Bug Bug 1.11 Bug 1.12 1.12 Bug 1.13 Docker Bug BugFix

Docker 1.11 MosDocker 1.11 Docker
Docker Daemon DaemonContainerd runC 3 Binary Daemon OCI rootfs spec Libnetwork Docker ID Hash Docker MosDocker

MosBridge , IP VPC

<267 2. Cgroup Docker Update CGroup CGroup 3. Docker Save Docker MosDocker Docker

Docker
Docker
IT Docker 32 8 8G 3 + 32 QPS 85% 44-56%( 78 ) Docker CPU Docker CPU Linux CGroup CGroup

268>2017 7 QPS 8

<269

Docker Docker IT Docker Docker Docker IO Buffered IO Docker Daemon OOM OOM_kill_disabled Docker Docker Docker

270>2017 HULK

O2O
JDK

1-2 Docker
Docker
HULK 2015 ----HULK Docker IT

<271 HULK "" HULK HULK
HULK HULK

272>2017 1. 2. 3. IaaS HULK

HULK

30%~70% 10%

<273 /

CAP CAP Consistency Availability Partition Tolerance
" "

Mesos Mesos Framework Mesos Kubernetes Mesos Mesos Framework Offer Framework

274>2017 Framework Offer Mesos FrameworkMesos Offer Framework Framework Mesos Mesos Framework Framework
Omega Omega Mesos Omega MVCC Omega Omega Omega

1. Namespace

Borg Kubernetes Borg Omega Kubernetes plugin Kubernetes HULK CAP AP C

<275 CA P

HULK HULK HULK

HULK ID 2. HULK ID

276>2017 3. Omega Actor

HULK + "" HULK Host pool / cgroups FilterPredicates RankPriorities rank rank

<277

HULK IaaS CPUI/O MemoryDisk

IaaS HULK

Load 2. HULK / Redis Redis 25% / Borg prod non-prod

278>2017 Borg non-prod Google C/C++ Java OOM HULK

HULK HULK HULK
HULK CPULoad MemoryIO rank n Actor

<279 Actor n
Actor mailbox Actor

n n n 1 n 1 n

280>2017 n n n

HULK Omega Omega MVCC HULK Actor HULK n

HULK IaaS
Kubernetes Kubernetes

2015

<281

"" 6 400

""

282>2017

"" ""
""

100 iPhone, AndroidH5PC

<283 ----

"" ""

284>2017

"" " "

"" "" ---- ----

<285 ""
""

286>2017

""" "

---- " """

<287

""

288>2017

<289

""---- ·" "

Samet O. Introduction to Online Payments Risk Management. O'Reilly. 2013. 2. . ArchSummit. . 2014.

290>2017

MTTRSLA 2016 C
"""" 45% case

<291 2016 3 10 Tair ID ID ID

""

switch "Faultdrill"

""

292>2017 11 618 6

/ / N

Thrift JMeter HTTP Thrift TCPCopy "" loading_test

<293 method
:) AppKey AppKey AppKey xx.xx.xx.order.search

Netflix SimianAmy MonkeyKing casekiller SimianAmy casekiller Linux "tc"

294>2017 "iptables" root root
root Hystrix Redis :) """" AppKey "" root method

1. 1. /DB/Cache/MQ

<295

    DubboCopy           SDK     TCPCopy

SDK @Copy simplingRate 100%

296>2017 @Copy(attribute = CopyMethodAttribute.READ_METHOD, simplingRate = 1.0f) public Result toCopiedMethod() { } copy-server copy-server interfacemethodserverAppKey AppKey Thrift IDL IDL ThriftCopy :) RPC HTTP RPC Thrift "" total length 2B header length+ + + 4B header length
header TraceInfo RequestInfo clientAppKey interfaceNamemethodName

client

<297

copyServer IP list IP 2. 3. @Copy AOP

RPC 1. 2. copyServer JoinPoint method args method
Thrift send_xx TSocket

298>2017 Client Client Thrift clientAppKey requestMethodName "" "${rawMethodName}_copy" "" attribute
/ server

fromAppKeytargetAppKey 2. targetAppKeys IP list IP

<299

" " fromAppKeyinterfaceName methodName 2. interfacemethodserverAppKey AppKey n n ByteBuf

AppKey

Thrift RPC mybatis mapper method Redis/Tair method Kafka ES

xxx ms Exception xxx ms Exception xxx ms exception / xxx ms exception

300>2017

Linux /IO/Load HystrixCommand AOP

root
Hystrix

classloader /Spring LTW/javaagent

javaagent

<301 client
JVM -javaagent:WEB-INF/lib/hotel-switchfaultdrill-agent-1.0.2.jar

client RedisDefaultClientMapperRegistryDefaultConsumerProcessorDefaultProducerProcessorMTThriftMethodInterceptorThriftClientProxy
script"java.lang.Thread.sleep(2000L);" "throw new org.apache.thrift.TException("rpc error")" script
script AppKey faultType md key 4. script method invoke script insertBefore value 5. key value Map 6. RedisDefaultClient method map object invoke 1. map object 2. object invoke "java.lang.Thread.sleep(2000L);" "throw new org.apache.thrift.TException("rpc error");"

302>2017 server server

hotel-swtich-api

sourceAppKey targetAppKey

com.sankuai.hotel. com.sankuai.hotel.sw.api.

5

1.0

sw.api.beta03

beta04

beta03 distributeGoodsService.queryPrepayList 5000

beta04 25000

CAT Receive Dispatch

<303 Redis MTThrift filter 1s

304>2017 Thrift com.meituan.hotel.goods.service. IGoodsService.queryGoodsStrategyModel 3s 6s

""
"" response request response daily build

<305

. 2. TCPCopy Dubbo -DubboCopy. 3. 0 1 . 4. javassit.

2013 2015 2015 2014

306>2017 Leaf ID

ID ID ID ID ID

ID 2. MySQL InnoDB
RDBMS B-tree 3. ID IDIM 4. ID URL ID 123 3 4
ID ID ID
ID

1. TP999 2. 5 9 3. QPS

<307

UUID UUID(Universally Unique Identifier) 32 16 8-4-4-4-12 36 550e8400-e29b41d4-a716-446655440000 5 UUID IETF UUID A Universally Unique IDentifier (UUID) URN Namespace

UUID 16 128 36
MAC UUID MAC
ID DB UUID MySQL [4]36 UUID All indexes other than the clustered index are known as secondary indexes. In InnoDB, each record in a secondary index contains the primary key columns for the row, as well as the columns specified for the

308>2017 secondary index. InnoDB uses this primary key value to search for the row in the clustered index. If the primary key is long, the secondary indexes use more space, so it is advantageous to have a short primary key. MySQL InnoDB UUID
snowflake UUID ID 64-bit snowflake 64-bit 41-bit 1L<<41/(1000L360024*365)=69 10-bit 1024 IDC 10-bit 5-bit IDC 5-bit 32 IDC IDC 32 12 2^12 ID snowflake QPS 409.6w/s IDC ID ID ID

<309 bit

Mongdb objectID MongoDB ObjectID snowflake " + +pid+inc" 12 4+3+2+3 24

MySQL auto_increment_increment auto_ increment_offset ID SQL MySQL ID begin; REPLACE INTO Tickets64 (stub) VALUES ('a'); SELECT LAST_INSERT_ID(); commit;

310>2017

DBA ID ID DB DB ID MySQL MySQL step 2TicketServer1 11357911... TicketServer2 2246810... Flickr 2010 Ticket Servers: Distributed Unique Primary Keys on the Cheap TicketServer1 1 TicketServer2 2 2 TicketServer1: auto-increment-increment = 2 auto-increment-offset = 1 TicketServer2: auto-increment-increment = 2 auto-increment-offset = 2 N N 0,1,2...N-1

<311

1,2,3,4,5 1 14 14 2 14 ID 7 2 100 ID ID
Leaf Leaf

312>2017

There are no two identical leaves in the world " "

Leaf Leaf-segment Leaf-snowflake Leaf-segment Leaf-segment ID proxy server segment(step )
biz_tag biz-tag ID biz_tag

+-------------+--------------+------+-----+-------------------+-----------------------------+

| Field

| Type

| Null | Key | Default

| Extra

|

+-------------+--------------+------+-----+-------------------+-----------------------------+

| biz_tag | varchar(128) | NO | PRI |

|

| max_id | bigint(20) | NO | | 1

|

| step

| int(11) | NO | | NULL

|

| desc

| varchar(256) | YES | | NULL

|

+-------------+--------------+------+-----+-------------------+-----------------------------+

biz_tag max_id biz_tag ID step ID step 1000 1000 1 1/step

<313 test_tag Leaf 1~~1000 step=1000 3001~~4000 biz_tag max_id 3000 4000 SQL Begin UPDATE table SET max_id=max_id+step WHERE biz_tag=xxx SELECT tag, max_id, step FROM table WHERE biz_tag=xxx Commit Leaf ID 8byte 64 Leaf DB Leaf

314>2017 max_id ID ID TP999 hang I/O tg999 DB buffer Leaf-segment Leaf ID DB DB DB DB DB DB DB DB TP999

<315 buffer Leaf segment 10% segment
biz-tag segment QPS 600 10 DB Leaf 10-20

Leaf "DB " Master Slave [5] Atlas ( DBProxy) 100% " Paxos " MySQL MySQL 5.7 GA MySQL Group Replication

316>2017 Leaf IDC "MTthrift RPC" Leaf IDC Leaf Leaf OCTO
Leaf-snowflake Leaf-segment ID ID ID 12 id Leaf-snowflake Leaf-snowflake snowflake bit "1+41+10+12" ID workerID Leaf Zookeeper snowflake wokerID Leaf-snowflake

Leaf-snowflake Zookeeper leaf_forever
workerIDzk int ID
workerID

<317 ZooKeeper ZK workerID ZooKeeper SLA

ID

318>2017 ZooKeeper leaf_forever

leaf_forever/${self} leaf_forever/${self}

<319

leaf_forever/${self} Leaf leaf_temporary ( Leaf-snowflake ) IPPort RPC sum(time)/nodeSize
abs( -sum(time)/nodeSize ) < leaf_temporary/${self}
1. (3s) leaf_forever/${self}

NTP NTP ERROR_CODE

// if (timestamp < lastTimestamp) {

} // ID

long offset = lastTimestamp - timestamp; if (offset <= 5) { try { // 5ms wait(offset << 1);//wait timestamp = timeGen(); if (timestamp < lastTimestamp) { // throwClockBackwardsEx(timestamp); } } catch (InterruptedException e) { throw e; } } else { //throw throwClockBackwardsEx(timestamp); }

320>2017 2017 Leaf-snowflake Leaf Leaf Leaf 4C8G QPS 5w/ sTP999 1ms SLA

Mtrace ID Leaf 2016 7 zhangjinlu#meituan.com

. MySQL[M]. , 2010:162-171. 2. UUID. 3. snowflake. 4. MySQL: Clustered and Secondary Indexes. 5. Semisynchronous Replication.

<321

322>2017
2014.06~2015.09 (1) (2) (3) (4)

<323 (5)
(6) (7)

(1) (2) (3) (4)

(1) 95% (2)

324>2017
App (1) (2) (3)

<325

2015.10~2016.10 (1)
(2) (3) (4)

326>2017 (5) (6) (7)

(1)
(2) (3)

(1) (2)

<327

(1)

328>2017 (2)

(3)

<329

"" "" "" " API """"" (1) "" """" ""---- "" "" "" ""

330>2017 (2) (1) ----Java PHP ---- RPC (2) "" (3) ----""

<331 ---- a. b.

2016.11

332>2017 (1)
(2)
(3)
(4) 2
(5)
(6)

(1)
(2)
(3) (4)

<333

20min 60min 180min

1min 43min 180min

1min 2min 20min

(1) 95% 99% (2) (3) (4)

334>2017

   Debug

QPS 8000 1 IO CPU

<335 Log4j Buffer

INFO

Log4jLog4j2 Logback RD

336>2017

JAR XML

<337

SLF4J SLF4J Log4jLogback Log4j2 (slf4j-api-1.7.7) private final static void bind() { try { // classpath StaticLoggerBinder Set staticLoggerBinderPathSet = findPossibleStaticLoggerBinderPathSet(); reportMultipleBindingAmbiguity(staticLoggerBinderPathSet); // slf4j org.slf4j.impl.StaticLoggerBinder LoggerFactoryBinder // the next line does the binding StaticLoggerBinder.getSingleton(); INITIALIZATION_STATE = SUCCESSFUL_INITIALIZATION; reportActualBinding(staticLoggerBinderPathSet); fixSubstitutedLoggers(); ... }

338>2017 findPossibleStaticLoggerBinderPathSet classpath org.slf4j.impl.StaticLoggerBinder slf4j
StaticLoggerBinder LoggerFactoryBinder

ChangeLogLevelProcessUnit

Logger Map String type = StaticLoggerBinder.getSingleton(). getLoggerFactoryClassStr(); if (LogConstant.LOG4J_LOGGER_FACTORY.equals(type)) { logFrameworkType = LogFrameworkType.LOG4J; Enumeration enumeration = org.apache.log4j.LogManager. getCurrentLoggers(); while (enumeration.hasMoreElements()) { org.apache.log4j.Logger logger = (org.apache.log4j.Logger) enumeration.nextElement(); if (logger.getLevel() != null) { loggerMap.put(logger.getName(), logger); } } org.apache.log4j.Logger rootLogger = org.apache.log4j.LogManager. getRootLogger(); loggerMap.put(rootLogger.getName(), rootLogger); } else if (LogConstant.LOGBACK_LOGGER_FACTORY.equals(type)) { logFrameworkType = LogFrameworkType.LOGBACK; ch.qos.logback.classic.LoggerContext loggerContext = (ch.qos.logback. classic.LoggerContext) LoggerFactory.getILoggerFactory(); for (ch.qos.logback.classic.Logger logger : loggerContext. getLoggerList()) { if (logger.getLevel() != null) { loggerMap.put(logger.getName(), logger); } } ch.qos.logback.classic.Logger rootLogger = (ch.qos.logback.classic. Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); loggerMap.put(rootLogger.getName(), rootLogger); } else if (LogConstant.LOG4J2_LOGGER_FACTORY.equals(type)) { logFrameworkType = LogFrameworkType.LOG4J2; org.apache.logging.log4j.core.LoggerContext loggerContext = (org.apache. logging.log4j.core.LoggerContext) org.apache.logging.log4j.LogManager. getContext(false);

<339 Map<String, org.apache.logging.log4j.core.config.LoggerConfig> map = loggerContext.getConfiguration().getLoggers(); for (org.apache.logging.log4j.core.config.LoggerConfig loggerConfig : map.values()) { String key = loggerConfig.getName(); if (StringUtils.isBlank(key)) { key ="root"; } loggerMap.put(key, loggerConfig); } } else { logFrameworkType = LogFrameworkType.UNKNOWN; LOG.error("Log : type={}", type); } 2. Logger Map private String getLoggerList() { JSONObject result = new JSONObject(); result.put("logFramework", logFrameworkType); JSONArray loggerList = new JSONArray(); for (ConcurrentMap.Entry<String, Object> entry : loggerMap.entrySet()) { JSONObject loggerJSON = new JSONObject(); loggerJSON.put("loggerName", entry.getKey()); if (logFrameworkType == LogFrameworkType.LOG4J) { org.apache.log4j.Logger targetLogger = (org.apache.log4j.Logger) entry.getValue(); loggerJSON.put("logLevel", targetLogger.getLevel().toString()); } else if (logFrameworkType == LogFrameworkType.LOGBACK) { ch.qos.logback.classic.Logger targetLogger = (ch.qos.logback. classic.Logger) entry.getValue(); loggerJSON.put("logLevel", targetLogger.getLevel().toString()); } else if (logFrameworkType == LogFrameworkType.LOG4J2) { org.apache.logging.log4j.core.config.LoggerConfig targetLogger = (org.apache.logging.log4j.core.config.LoggerConfig) entry.getValue(); loggerJSON.put("logLevel", targetLogger.getLevel().toString()); } else { loggerJSON.put("logLevel","Logger , !"); } loggerList.add(loggerJSON); } result.put("loggerList", loggerList); LOG.info("getLoggerList: result={}", result.toString()); return result.toString(); }

340>2017 3. Logger private String setLogLevel(JSONArray data) { LOG.info("setLogLevel: data={}", data); List loggerList = parseJsonData(data); if (CollectionUtils.isEmpty(loggerList)) { return ""; } for (LoggerBean loggerbean : loggerList) { Object logger = loggerMap.get(loggerbean.getName()); if (logger == null) { throw new RuntimeException(" Logger "); } if (logFrameworkType == LogFrameworkType.LOG4J) { org.apache.log4j.Logger targetLogger = (org.apache.log4j.Logger) logger; org.apache.log4j.Level targetLevel = org.apache.log4j.Level. toLevel(loggerbean.getLevel()); targetLogger.setLevel(targetLevel); } else if (logFrameworkType == LogFrameworkType.LOGBACK) { ch.qos.logback.classic.Logger targetLogger = (ch.qos.logback. classic.Logger) logger; ch.qos.logback.classic.Level targetLevel = ch.qos.logback. classic.Level.toLevel(loggerbean.getLevel()); targetLogger.setLevel(targetLevel); } else if (logFrameworkType == LogFrameworkType.LOG4J2) { org.apache.logging.log4j.core.config.LoggerConfig loggerConfig = (org.apache.logging.log4j.core.config.LoggerConfig) logger; org.apache.logging.log4j.Level targetLevel = org.apache.logging. log4j.Level.toLevel(loggerbean.getLevel()); loggerConfig.setLevel(targetLevel); org.apache.logging.log4j.core.LoggerContext ctx = (org.apache. logging.log4j.core.LoggerContext) org.apache.logging.log4j.LogManager. getContext(false); ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. } else { throw new RuntimeException("Logger , !"); } } return "success"; } Logger Logger

<341

Web RPC HTTP Thrift Thrift JSON String
dynamic-invoker.xml Thrift ZooKeeper
HTTP Web RPC HTTP

2016 9 V1.0 20

Logback SLF4J Logback Log4j Logback.xml SLF4J Debug ROOT 1130 Load

342>2017 ERROR

Simple Logging Facade for Java (SLF4J) 2. Log4j 2 Architecture - Apache Log4j 2 3. Hash-based message authentication code - Wikipedia

<343 0 1

2016 07 ""
CRM """" KPI

344>2017

3 "==""<"

if (StringUtil.isBlank(fieldA) || StringUtil.isBlank(fieldB) || StringUtil.isBlank(fieldC) || StringUtil.isBlank(fieldD)) { return ResultDOFactory.createResultDO(Code.PARAM_ERROR, " "); } if (fieldA.length() < 10) { return ResultDOFactory.createResultDO(Code.PARAM_ERROR, " 10 "); } if (!isConsistent(fieldB, fieldC, fieldD)) { return ResultDOFactory.createResultDO(Code.PARAM_ERROR," xxx "); }

<345

"=="

346>2017 ---- Drools Drools Drools DSL rule "1.1" when poi : POI( source == 1 && brandType == 1 ) then System.out.println( "1.1 matched" ); poi.setPassedNodes(1); end rule "1.2" when poi : POI( source == 1 && brandType == 2 ) then System.out.println( "1.2 matched" ); end rule "2.1" when poi : POI( source == 2 && brandType == 1 ) then System.out.println( "2.1 matched" ); poi.setPassedNodes(2); end

<347

rule "2.2"

when

poi : POI( source == 2 && brandType == 2 )

then

System.out.println( "2.2 matched" );

poi.setPassedNodes(3);

end

Drools

DSL Java, Groovy, Python then when...then

Drools

2016 10 70%

348>2017

SQL +

Spark

<349

""

350>2017 Drools

"" 2017 02 "" "" 3 """" "Maze " "Maze " Maze

"" "Maze "

n 1 3 y = fx1, x2, ..., xn ""

<351 FACT LHSLeft Hand Side RHSRight Hand SideLHS RHS FACT [ 1] && [ 2] 1 3 FACT IntegerLongFloatDoubleShortStringBoolean

"" 3

352>2017 3 "" $ 1 > $ 2
"" $ 1 + $ 2 > $ 3 ""

<353

/** */

|

| > $ 3 /** */

|

$ 1 + $ 2

/** */

    DroolsAviator  ANTLR

Maze " " Maze Maze
Maze MazeGO MazeQL MazeGO SQLC MazeQL BD 30 A B MazeQL VectorC MazeGO ID *

354>2017 VectorC 1. 2. 3. GroupBy SQLC SQL MazeGO MazeGO 3 MazeGO
3 """ """

<355

MazeGO ZooKeeper
```
                        hbase 
```

MazeQL MazeQL 3 MazeQL

356>2017

MazeQL ZooKeeper QL SQLC VectorC SQLC " " QL "" QL
Spark QL QL SQL QL Mysql Derby Spark Spark SQL
"" SQL

<357 SQL VectorC VectorC "Maze "

Maze Maze

Maze // N FACT function(Fact[] facts) { // FACT String xx= facts[0].xx; // SQLC sql SQLC +SQL

358>2017 List moreFacts = connection.executeQuery("select * from xxx where xx like`%"+ xx +"%'); // FACT UserDefinedClass userDefinedObj = userDefinedFuntion(facts, moreFacts); // int compareResult = userDefinedObj.getFieldXX().compare(XX); // UserDefinedResultClass userDefinedResultObj = new UserDefinedResultClass(); // if (compareResult == 0) { userDefinedResultObj.setCompareResult(Boolean.FALSE); } else if (compareResult > 0) { userDefinedResultObj.setCompareResult(Boolean.FALSE); } else { userDefinedResultObj.setCompareResult(Boolean.TRUE); } // return userDefinedResultObj; }

MazeGO jar
MazeGO client

// MazeGO client MazeGOReactor reactor = new MazeGOReactor(); reactor.setMazeIds(Arrays.asList());

reactor.init(); // MazeGO client reactor.go(, ); // MazeGO client reactor.destroy();

<359

MazeGO

360>2017

Maze

2015 CRM CRM

MySQL
MMM 2015 MMMMaster-Master replication manager for MySQL MMM MMM

362>2017 MySQL 1 VIPVirtual IP NN>=1 VIP MySQL Agentmmm-agent mmm-agent mmm-manager mmm-manager MySQL mmm-manager mmm-agent mmm-manager

mmm-manager VIP VIP
Dead Master flush tables with read lock Dead Master VIP

<363 mmm-manager master1 VIP master2 MMM VIP 1 6 7 VIP VIP VIP mmm-agent VIP mmm-agent mmm-manager MySQL mmm-manager VIP ARP MMM Google Google MMM Bug Bug

364>2017 MHA 2015 MySQL MHA MMM MHAMySQL Master High Availability Facebook Yoshinori Matsunobu MySQL MHA MySQL MHA Dead Master Binlog VIP MHA MHA

<365 DB MHA VIP VIP 2 VIP MHA Manager
MHA+Zebra (DAL) Zebra Java c3p0 SQL MHA MySQL MHA+Zebra MHA Zebra monitorZebra monitor ZooKeeper

366>2017 Zebra monitor 10s ~ 40s ZooKeeper
Zebra monitor MHA ^_^ VIP MHA MHA VIP Zebra monitor Write new master IP Dead Master VIP

<367

368>2017 MHA Master-Slave Binlog MHA Master-Slave
Proxy Zebra Proxy MHA MHA Proxy Proxy Zebra Java Response Time GitHub

MHA MySQL Semi-Sync 95% Agent Master MHA Manager

Binlog Server Server Slave Binlog Binlog Server ACK Binlog Server

<369 Binlog Server Binlog Server Agent MHA MySQL Agent Agent Slave Manager MHA

370>2017 MGR Agent Binlog Server Binlog Server Binlog Server Agent MySQL Raft Paxos Paxos MGR MySQL Paxos MGR single-primary

<371 MySQL topo Zebra monitorZebra monitor Binlog Server ACK MGR 1 MGR MySQL

MySQL MMM MHA+Zebra MHA+Proxy

14

372>2017 DBProxy DBA ( )

   DBA  DBA

360 Atlas DBA DBProxy DBProxy DBProxy Github https://github. com/Meituan-Dianping/DBProxy DBProxy

MySQL 2. 3. 4.Slave Master MHA 5. 6. SQL 7.

8. 9. 10.Client IP

<373

DBProxy Atlas ( show proxy status/variables) save config percentile DBProxy kill session DBProxy admin kill session backend backend DBProxy root root admin admin host DBProxy 1. normal KILL -SIGTERM pid of mysql-proxy; admin : shutdown [normal] 2. immediate

374>2017 KILL -SIGINT pid of mysql-proxy; admin : shutdown immediate 3. shutdown_timeout: normal shutdown_ timeout s, 600 SQL SQL SQL query-filter-time-threshold SQL SQL query-filter-frequent-threshold SQL 60s 60s access-num-per-time-window SQL auto-filter-flag OFF: ON: Load SQL manual-filter-flag OFF: ON:

<375 admin / MySQL thread running DBProxy MySQL MySQL thread running MySQL backend-max-thread-running MySQL thread running thread-running-sleep-delay thread running backend-max-thread-running set backend offline set transaction isolation level use db set option set session db-connection-idle-timeout keepalive SQL DBProxy SQL SQL database SQL rotate

376>2017 SQL sql-log-file-size sqllog-file-num MySQL MySQL server server mysql-version SQL SQL database COM_INIT_DB sysbench QPS 7 22 QPS sql log autocommit false Bugs DBProxy MySQL socket keepalive rpm conf rpm mysql-proxyd proxy-dir, rpm KILL MySQL KILL DBProxy rollback NULL DBProxy merge NULL JDBC IN int32 int64

<377 lock 24 admin admin "[admin] we only handle text-based queries (COM_QUERY)" set backend offline/online DBProxy SQL_CALC_FOUND_ROWS SQL SQL DBProxy Bug release notes

378>2017 SQL SQLAdvisor DBA

SQL DBA SQL
SQL SQL SQL DBA SQLAdvisor DBA SQL SQL MySQL SQL where Join SQLAdvisor SQLAdvisor GitHub https://github. com/Meituan-Dianping/SQLAdvisor GitHub SQL SQLAdvisor

<379 SQLAdvisor sql: SELECT id FROM crm_loan WHERE id_card = '1234567cmd: ./sqladvisor -h xx -P xx -u xx -pxx -d xx -q"SELECT id FROM crm_ loan WHERE id_card =1234567'" SQLAdvisor : alter table crm_loan add index idx_id_card(id_card)

380>2017 SQLAdvisor SQLAdvisor MySQL SQL(Insert/Delete/Update/Select) Join Order by Group by
SQLAdvisor Join

Join Join on Join using Join on where
Join nested_join table list join_using_fields Join on Join using
table list
right Join left Join 5.Join
Join mysql_sql_parse_join(TABLE_LIST join_ table) mysql_sql_parse_join(Item join_condition)

<381 where

SQL where where AND OR OR AND
where Join 3. where like
30 where
"show table status like" table_count 2. best_index Primary key > Unique key > 3. offset rand_rows

382>2017 offset = (table_count / 2) > 10W ? 10W : (table_count / 2) rand_rows =(table_count / 2) > 1W ? 1W : (table_count / 2) select count(1) from (select field from table force index(best_index) order by cl.. desc limit rand_rows) where field_print rows cardinality = rows == 0 ? rand_rows : rand_rows / rows; mysql_sql_parse_field_cardinality_new()

<383

mysql_sql_parse_index() 2. Group Order
Group Order Group by Order by Order by Order by Order by Order by Order by

384>2017 2. >(group by | order by )> 3. mysql_sql_parse_group() Group mysql_sql_parse_order() Order mysql_sql_parse_group_order_add()

<385

where Join SQL

386>2017 3. explain select * from table where field 4. 5. final_table_drived() get_ join_table_result_set()

1. Join 3. mysql_index_add_condition_field()

<387

print_index()

388>2017 SQLAdvisor Functionality Added or Changed SQLParser SQLAdvisor Join find_join_elements() where like Order by Order by primary key
Bugs Fixed SQL where find_best_index() MySQL API result free

SQL Bug Github issue SQLAdvisor SQLAdvisor QQ: 231434335 SQLAdvisor

SQLAdvisor . 2. SQLAdvisor . 3. SQLAdvisor release notes. 4. SQLAdvisor . 5. FAQ.

<389 MyFlash MySQL

DBA bug DBA binlog MySQL mysqlbinlog 2 DBA binlog2sql binlog2sql ----MyFlash https://github.com/Meituan-Dianping/ MyFlash

mysqlbinlog sedawk binlog SQL sedawk SQL SQL SQL SQL SQL HTML awksed patch mysqlbinlog Flashback

390>2017 MySQL Server binlog
MySQL MySQL 5.6 patch MySQL 5.7 patch MySQL MySQL binlog SQL binlog2sql binlog SQL delete awksed

a. binlog b. SQL c. MySQL d. e. binlog binlog binlog
binlog binlog binlog format description event rotate

event event

<391

binlog

392>2017 event event header event data binlog event formart description event

170905 01:59:33 server id 10 end_log_pos 123 CRC32 0xed1ec563 Start: binlog v 4, server v 5.7.18-log created 170905 01:59:33

table map event

<393

394>2017
170905 01:59:33 server id 10 end_log_pos 339 CRC32 0x3de40c0d Table_map: test.test4 mapped to number 238 update row event

<395 170905 01:59:33 server id 10 end_log_pos 385 CRC32 0x179ef6dd Update_rows: table id 238 flags: STMT_END_F UPDATE test.test4 WHERE @1=3 SET @1=13; binlog event binlog binlog event event header type_code insert 30update 31delete 32 insert delete type_code binlog event update BI before imageAI after image update update

396>2017 AIBI

int 4 bingint 8 table map event decimal189 9 table map event varchar10 1 varchar280 2
binlog length encoded integer binlog timestamp 4 event 64 3 192<255 1 2 3 8 8 length encoded integer

<397 varchar 251 varchar 0xFC varchar decimal decimal 9 4 9 Fnum decimal18,10 1. 8 int 4 2. (10 /9)*4+Fnum(10%9)=5 3. 4+5=9

binlog event binlog binlog

398>2017 MyFlash binlog binlog event binlog binlog event event binlog binlog event binlog event binlog event table_map event row_event( write_event delete_eventupdate_event) least execution event unit

table_map event row_event

<399

binlog table_map_event update_row_ event update_row_event event table_map_event write_row_eventupdate_row_evendelete_row_event event event table_map event row event binlog

a. row event b.

400>2017 SQL binlog binlog event next_position binlog

testFlashback2 100 CREATE TABLE testFlashback2 ( id int(11) NOT NULL AUTO_INCREMENT, nameShort varchar(20) DEFAULT NULL, nameLong varchar(260) DEFAULT NULL, amount decimal(19,9) DEFAULT NULL, amountFloat float DEFAULT NULL, amountDouble double DEFAULT NULL, createDatetime6 datetime(6) DEFAULT NULL, createDatetime datetime DEFAULT NULL, createTimestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, nameText text, nameBlob blob, nameMedium mediumtext, PRIMARY KEY (id) ) ENGINE=InnoDB mysql> select count() from testFlashback2; +----------+ | count() |

+----------+ | 1048576 | +----------+ 1 row in set (0.16 sec) delete from testFlashback2;

<401

MyFlash

402>2017

MySQL 1,2,3. 2. binlog2sql. 3. mysqlbinlog Flashback for 5.6. 4. MySQL .

Sysbench

<403

sysbench sysbench JSON sysbench

I/O CPU DBA DBA MySQL sysbench sysbenchTPCC-MySQL

404>2017

sysbench

sysbench DBA 2. MySQL OraclePercona
sysbench 3. sysbench MySQL 8.0
sysbench Lua C Lua DBA sysbench

sysbench sysbench MySQL sync_binlog=1innodb_flush_log_at_ trx_commit=2innodb_io_capacity=2000 sysbecnh

tables

16

table_size

25,000,000

threads

16

time

3600

warmup_time 600

rate

0

histogram

on

percentile

99

sbtest1... sbtest16 0 MySQL OLTP 99

<405 sysbench MySQL SATASSDPCIe IO CPU GB MySQL 5.7 DBA DBA MySQL 5.7 slave_ parallel_workers sysbench sysbench oltp_write_only.lua TPS:33,336 TPS 8 TPS TPS binlog_group_commit_ sync_delay binlog_group_commit_sync_no_delay_count

406>2017 sysbench

sysbench Lua sysbench 10 sysbench
sysbench prepare warmup Lua sysbench Lua
MySQL 5.7 JSON MongoDB MongoDB JS map/ reduce TPS MySQL 5.7+JSON MongoDB MySQL JSON sysbench Lua

<407

sysbench

TPS QPS QPS

408>2017 TPS 10000 TPS sysbench

TPS QPS SQL SQL SQL sysbench
MySQL buffer pool instance

<409 sysbench sb_counter.c SQL 90 95 99 90%95% 99% SQL DBA 0.01ms 0.01ms ""

410>2017

sysbench sysbench k=floor((log(response_time) +6.908) * 55.35 + 0.5) k response_time=0.001 k 0response_time=0.01 k=128; response_time=10 k=509 response_time=100 k=1023 k k ms k respone_time = exp((k/55.535)-6.908)

MySQL QPS 50%

<411 response[1024] sysbench response[1024] 128

412>2017

sysbench MySQL sysbench JSON sysbench sysbench

MySQL DBA2012 2017 MySQL 2014 ""

2015 10 ""
Hadoop SQL
"1+1>2" App UGC
App App
App

414>2017 SQL
" " Excel

<415 Client

"  "   Blackhole DataX     10Gb

416>2017
9 10 SQL PrestoHive 22 work Excel...... 6 7 4
BAT

<417 500 11 P 3000 6000 50% 50%
owner owner 100 240 1000

418>2017
DB Server log
DistCp ready ready DistCp Hadoop DistCp MapReduce A B A A B DistCp

<419

""

Kerberos Kerberos Kerberos KDC Client ServerClient HDFS ClientServer Name NodeKDC Client Server Client Server Kerberos Kerberos Kerberos

420>2017
Client KDC Server ticket Server 3 2 Kerberos KDC TGS krbtgt A realm @ B realm KDC KDC
Client Server Client KDC ticket 1 krbtgt CREALM @ REALM Client ticket Service KDC Service ticket Service Kerberos KDC krb5.conf domain_realm DNS

<421 Hadoop Server Client Hadoop Debug Hadoop logClient realm realm realm

RD " "

422>2017 7000 8000 10 1~~4 3~~5 ODS 1 Task 1Task 2Task 3Task 8 Task 123 Task 8 Task 8 Task 7 Task 4 50

<423 HiveSpark UDF

Hadoop Block 10

424>2017 Hadoop 2.7.1 Hadoop HDFS
patch bug feature Git review patch Hive
DistCp HDFS YARN HDFS DataNode 4 10Gbps

<425 NameNode YARN trick 1000 Hadoop Hadoop Hadoop Hadoop Rack Hadoop 3~4 3 3 DataNode pipeline NameNode zone Rack NameNode pipeline hack pipeline Client zone Application mapper A reducer B YARN zone Fair Scheduler zone zone

426>2017 job job job balancer Hack ZoneTransfer DataNode block Slave DN(DataNode) NM(NodeManager) App RM(ResourceManager) zone App DN pipeline root DN transfer

<427 4 70% 70% 30%
RPC Active NameNode FsImage standby FsImage block DataNode NNNameNode SafeMode 5~6
3 "" NN federation cluster ID mount table cluster ID Hive MySQL pipeline KDC

428>2017 KDC KDC

Hadoop Hive

<429 Hadoop
insert into BI select from BI where group by
8000 10

430>2017 a o1 b o1o2 c o2 mart

<431

Hive Hack bi.o2 mart_b.o2 mart_b.o2 bi.o2 bi.o2 Hive Hack bi. run Hive Hive metaserver Spark Presto Hive SQL
7000 90%SQL Hive metaserver

432>2017

NN

Block Cache Cache Cache Cache Cache server zone Server Client NameNode

RPC SLA
"" " "

""

<433

deadline Hive

brainstorming

11 12 11 101112

434>2017 12 11 7 8

<435

436>2017
Kimball

ODS Bill Inmon

<437

BDWMFS-LDMMLDM O2O

438>2017

BD BD BD

App PC I

POI O2O POI POI POI

POI

<439

6

3NF 3NF

440>2017

- B POI POI 80%

<441

- ODS

442>2017

2003 2015

<443 Flink Storm

Apache Flink Apache Storm Apache Storm"Storm" Storm API Storm Apache Flink"Flink"
Flink Storm Flink Flink """" Flink SLA Flink Storm

Storm

Flink

At Most Once At Least Once

At Most Once At Least Once Exactly Once

ACK

API API Storm

444>2017 2. Flink Storm Flink
2.1 " - " " - " " "

100 Flink Storm API
"" Storm ""(At Most Once) ""(At Least Once) Flink ""(Exactly Once)

<445

2.2 Throughput
/
20 / 2 /
Latency
5 25 30 60
3. Storm Flink 1 2 Standalone Flink on Yarn

446>2017

3.1

CPU Core Memory Disk OS

QEMU Virtual CPU version 1.1.2 2.6GHz 8 16GB 500G CentOS release 6.5 (Final)

3.2

Version Master Memory Slave Memory Parallelism

Storm

Flink

Storm 1.1.0-mt002 Flink 1.3.0

2600M

1600M * 16

12800M * 2

2 supervisor

2 Task Manager

16 worker

16 Task slots

4. 4.1

<447

Data Generator id eventTime Kafka TopicTopic Data Storm Task Flink Task Kafka Topic Data Offset inTimeoutTime Topic Topic Storm Topic Flink

448>2017 Metrics Collector outTime Topic MySQL Metrics Collector outTime outTime - eventTime outTime inTime 99 MySQL MySQL 99 4.2 Storm Flink At Least Once Storm ACKACKer 1 Flink Checkpoint 30 StateBackend Memory Kafka Kafka Kafka eventTime Kafka Topic Topic

4.3 Identity Identity " - " "msgId, eventTime" eventTime 20 B inTime outTime Kafka Topic Data Kafka "msgId, eventTime, inTime, outTime" 50 B

<449 Sleep Sleep Identity 1 ms

450>2017 Windowed Word Count Windowed Word Count Flink JSON msgIdeventTime 150 B JSON eventTime inTime CountWindow eventTime inTime outTime Kafka Topic Spout/Source OutputBolt/Output/Sink 1 JSONParserCountWindow Storm window CountWindow HashMap Flink CountWindow Reduce

5. 5.1Identity

<451

Storm Flink
Identity Storm 8.7 / Flink 35 / Kafka Data Partition 1 Flink Storm 3.2 Partition 8 Flink Storm 4.6 Flink Storm 3-5

452>2017 5.2Identity outTime - eventTime Storm Flink 99 Identity 99 Storm Flink QPS 80000 Storm Storm Flink Storm QPS 100 99 700 Flink 50 99 300 Flink Storm

5.3Sleep

<453

Sleep 1 Storm Flink 900 /

5.4Sleep

454>2017 outTime - eventTime Sleep 1 Flink Storm 5.5Windowed Word Count 10 Storm 1.2 / Flink Standalone 4.3 / Flink Storm 3

<455 5.6Windowed Word Count Flink At Least Once Exactly Once Flink Exactly Once JSON Parser CountWindowCountWindow Output Source/Output/Sink 1 JSONParser/CountWindow Windowed Word Count At Least Once Exactly Once Exactly Once At Least Once 6.3%

456>2017 5.7Windowed Word Count Storm At Least Once At Most Once Storm ACKer ACK Bolt ACK At Most Once At Least Once At Most Once At Most Once At Least Once 16.8%

<457 5.8Windowed Word Count Identity Sleep outTime - eventTime Thread.sleep() outTime - inTime Windowed Word Count outTime - inTime outTime - eventTime outTime - eventTime outTime - InTime Flink Storm outTime - inTime outTime - eventTime inTime eventTime Storm Flink Storm Flink Flink Flink outTime - inTime QPS Flink

458>2017 5.9Windowed Word Count Flink At Least Once Exactly Once 99 At Least Once Exactly Once Flink Exactly Once At Least Once

<459 5.10Windowed Word Count Storm At Least Once At Most Once 99 At Least Once At Most OnceQPS 4000 QPS 6000 QPS 8000 At Least Once Storm QPS Storm At Most Once At Least Once QPS At Most Once

460>2017 5.11Windowed Word Count Flink StateBackends Flink Standalone on Yarn Memory FileSystemRocksDB StateBackends StateBackends Standalone JobManager on Yarn HDFS FileSystem Memory RocksDB Standalone on Yarn FileSystem Memory on Yarn RocksDB Standalone

<461 5.12Windowed Word Count Flink StateBackends FileSystem Memory Backends RocksDB Backends on Yarn 6. 6.1 5.15.5 Storm 8.7 / Flink 35 / Flink Storm 3-5 5.25.8 Storm QPS Kafka 100 99 700 Flink 50 99 300 Flink Storm QPS Flink

462>2017

Flink Storm 6.2 5.1 5.35.2 5.4 Bolt Sleep 1 Flink Storm

6.3 5.65.75.95.10 Flink Exactly Once At Least Once 6.3%Storm At Most Once At Least Once 16.8% Storm ACKFlink At Least Once Flink Exactly Once Flink Storm At Most Once Flink 6.4Flink Flink RocksDB StateBackends 5.11 5.12

StateBackend

Memory

TM Memory

    JM Memory 3-5  Storm

FileSystem

TM Memory

FS/HDFS

KV 3-5 Storm Backend

RocksDB

RocksDB on TM FS/HDFS

0.3-0.5 Storm

KV

<463 6.5 Flink Flink Exactly Once
7. Exactly Once 1ms Thread.sleep() Flink

Flink RocksDBStateBackend Flink API Table API & SQL CEP

---- . 2. intel-hadoop/HiBench: HiBench is a big data benchmark suite. 3. Yahoo . 4. Extending the Yahoo! Streaming Benchmark.

464>2017

O2O /push
POI 2000 2.5
1
1

<465

RPC

2

466>2017 2

Thrift RPC

3 3 123 A BC AB C A B 2 3 C

<467

4 4

468>2017

NoSQL
5 5 ID level ID+level Key JSON ID+level

POI

<469

6 6

7 7 ID t

470>2017 1 7 3t 4t 6t 13t
2 Apache Storm Bolt Bolt Bolt Bolt Storm 7 t 3t Storm Bolt Storm 8 8Srm

<471 8 ChinaSpout Spout ProvinceBolt 40 CityBolt 300 BareaBolt POI Bolt POI POI POI Storm ----DRPC Storm Bolt

KV Squirrel Tair Squirrel Redis Cluster squirrel KV Tair Squirrel Tair

472>2017

9 Rhino Hystrix
a. b. c. 9 Rhino request1 squirrel request2 squirrel tair squirrel Rhino squirrelrequest3 squirrel Rhino squirrel

<473

RPC Rhino
1

2016 2017 BI

474>2017

"" dashboard 1 1 Kylin MySQLElasticsearchDruid

<475

2

476>2017 2 1. 2. 3. master-work

3 3

<477 3 B=(L+H)*Y https://tech.meituan.com/ dsmf.html 4 E , A
4

478>2017

5 UV SKU GMV GMV GMV= UV* * * *SKU

A/(A+B)

5

6

<479 6 7
7

480>2017

8
9

<481 8 9

482>2017 BI

Cube Cube SQL UI Cube

BI SQL UI

----

<483

1 1.

1. OLAP

484>2017 2 SQL MySQL Kylin SQL ESElasticsearch ES API API 2
2. SQL SQL 1. SQL SQL SQL SQL 2. SQL SQL Dialect SQL SQL SQL SQL SQL

<485 SQL Apache Calcite Avatica Alibaba Druid

SQL API
ANSI SQL-92 SQL SQL Java
join Kylin Cube join inner joinleft outer join full outer join inner join left outer join full outer join
inner join private void join(List<Map<String,String>>[] contents,List sharedList,final int n,int[] rowsStatus,LinkedList result){ if(this.cubeJoin==1){ throw new java.lang.IllegalArgumentException("left join call leftJoin method,not call join method"); } if(n<contents.length){ List<Map<String,String>> list = contents[n]; for(int k=0;k<list.size();k++) { boolean equal = true; if(n!=0) { Map<String, String> prev = contents[n - 1].get (rowsStatus[n - 1]); Map<String, String> cur = list.get(k);

486>2017 for (Project proj : sharedList) { String key = proj.fieldName.toUpperCase(); if (key.matches("^\d+$") || key.equals("*")) { key = "_"; } key = proj.isCompanion() ? key + proj.getFactId() : key; String prevValue = prev.get(key); String curValue = cur.get(key); if (prevValue == curValue) { continue; } if (prevValue == null || curValue == null || !prevValue.equals(curValue)) { equal = false; break; } } } if (equal) { rowsStatus[n] = k; if(n==contents.length-1){//last dataset match MatchRow mr = new MatchRow(); List<MatchRow.DatasetRow> tmp = new ArrayList<>(); for(int i=0;i<rowsStatus.length;i++){ MatchRow.DatasetRow dr = new MatchRow.DatasetRow(); dr.setDatasetIndex(i); dr.setRowIndex(rowsStatus[i]); tmp.add(dr); } mr.addMatchRow(tmp); result.add(mr); }else{ join(contents,sharedList,n+1,rowsStatus,result); } } } } } inner join contents sharedList

<487 n rowsStatus result join MatchRow

sharedList inner join join

left outer join private boolean leftJoin(List<Map<String,String>>[] contents,List sharedList,final int n,int[] rowsStatus,LinkedList result){ boolean leftJoinMatch = false; if(n<contents.length){ List<Map<String,String>> list = contents[n]; for(int k=0;k<list.size();k++) { boolean equal = true; if(n!=0) { //in left join,compare with the first dataset. Map<String, String> prev = contents[0].get(rowsStatus[0]); Map<String, String> cur = list.get(k); for (Project proj : sharedList) { String key = proj.fieldName.toUpperCase(); if (key.matches("^\d+$") || key.equals("*")) { key = "_"; } key = proj.isCompanion() ? key + proj.getFactId() : key; String prevValue = prev.get(key); String curValue = cur.get(key); if (prevValue == curValue) { continue; } if (prevValue == null || curValue == null || !prevValue.equals(curValue)) { equal = false; break; } } } if (equal) {

488>2017 leftJoinMatch = true; rowsStatus[n] = k; if(n==contents.length-1){//last dataset match MatchRow mr = new MatchRow(); List<MatchRow.DatasetRow> tmp = new ArrayList<>(); for(int i=0;i<rowsStatus.length;i++){ MatchRow.DatasetRow dr = new MatchRow.DatasetRow(); dr.setDatasetIndex(i); dr.setRowIndex(rowsStatus[i]); tmp.add(dr); } mr.addMatchRow(tmp); result.add(mr); }else{ //if next dataset is not match,use the next's next... for(int loopFlag=n+1;loopFlag<rowsStatus. length;loopFlag++){ boolean match = leftJoin(contents,sharedList, loopFlag,rowsStatus,result); if(match){ break; } rowsStatus[loopFlag]=-1; if(loopFlag==contents.length-1){ MatchRow mr = new MatchRow(); List<MatchRow.DatasetRow> tmp = new ArrayList<>(); for(int i=0;i<rowsStatus.length;i++){ MatchRow.DatasetRow dr = new MatchRow.DatasetRow(); dr.setDatasetIndex(i); dr.setRowIndex(rowsStatus[i]); tmp.add(dr); } mr.addMatchRow(tmp); result.add(mr); } } } } } } return leftJoinMatch; }

<489 left outer join inner join 2

left outer join
left outer join
Java ScriptEngine ScriptEngine
"" ScriptEngine

490>2017

UPM UPM UPM UPM UPM
UI

UI UI

AI

O2O
"" """" """"" "

492>2017 "" "" O2O / ClickNet

AI<493

Query DNN/CNN/RNN/LSTM/GRU

Label Query Bias / /

Yelp

494>2017 :

AlexNet High Level Low Level 2

AI<495 POI Deal UGC POI 3 OCR O2O OCR OCR BD

496>2017 OCR OCR

OCR 1.
2. HOG 3.
OCR OCR 1. Faster R-CNN FCN Faster R-CNN : ZF 5 3 RPN Anchor

AI<497 FCN / Layer Layer

498>2017 2.

AI<499 +

2010 2015 4 NLP 2015 NLP 2015 OCR

500>2017

2012 ImageNet SIFTHOG SVM SIFT 26% CNN 26% 15% ImageNet 2012 4.94%

Google  2016

AI<501 Wide & Deep Learning
2.

Item

User-Based User X N User N User Item X Item Jaccard Similarity

502>2017 Model-Based u i
Item-Based word2vec Item Cosine Similarity u Item Item i Top N Query-Based Query WiFi
Location-Based

AI<503 AUCNDCGPrecision AB
3. Item Item

504>2017 3.1 GBDTGBDT+LR GBDT AB CTR

AI<505 Item "" Item
Google 2016 Wide & Deep Learning

506>2017

3.2 / 10% Noise ( ) App
Item Item Item

3.3

AI<507 2.

""

3.3.1 " " " " " "

508>2017
3.3.2 "" Min-Max Min Max Cumulative Distribution FunctionCDFCDF x CDF Min-MaxCDF AUC 0.1%01 CDF Min-Max Min-Max 3.3.3 super-liner sub-liner x 2

AI<509 2 AUC 2 3.4Optimizer

3.4.1Stochastic Gradient Descent (SGD) SGD Mini-Batch
3.4.2Momentum SGD Momentum SGD SGDMomentum

510>2017
3.4.3Adagrad SGDAdagrad Adagrad gt gt 3.4.4Adam Adam Momentum Adagrad Adam

AI<511 Adam Adagrad Momentum SGDAdagrad SGD 3.5 Mean Squared ErrorMean Absolute ErrorCross Entropy Cross Entropy W b Sigmoid MSE y a a=(Wx+b) W b

512>2017 Sigmoid (z) z Cross Entropy

n i Logistic :

w b
3.6 5 DNN / AUC DNN CTR DNN Item Item

AI<513 Item-Feature-Label
TheanoTensorflow Keras Min-Max Adam Cross Entropy Wide & Deep Learning Deep Wide Deep 3 ReLU Sigmoid Wide Deep 7000 3000 Batch Size 50000Epoch 20

514>2017 4. / Base AB / 256->128->64 Base

AI<515 Item 5. Item CTR

RNN Deep & Wide DNN

516>2017

H. Cheng, L. Koc, J. Harmsen et al, Wide & Deep Learning for Recommender Systems, DLRS 2016 Proceedings of the 1st Workshop on Deep Learning for Recommender Systems.
P. Covington, J. Adams, E. Sargin, Deep Neural Networks for YouTube Recommendations, RecSys`16 Proceedings of the 10th ACM Conference on Recommender Systems.
H. Wang, N. Wang, D. Yeung, Collaborative Deep Learning for Recommender Systems, KDD`15 Proceedings of the 21th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining.

2015 2016 12

AI<517

1 1 ""Kaggle ......

518>2017 2 +

KPI deadline bad case low

AI<519 2. 10min

garbage ingarbage out word2vec embeddingword2vec ID ......

520>2017 3

1. "" RD "" ...... ...... IPWiFi...... ""

AI<521 4 WiFi deep learning deep learning ----

5 roadmap

522>2017 Glmnet>LASSO>=Ridge>LR/Logistic LR/Logisticridge LR lasso L1 Glmnet Stanford Glmnet RPythonSpark RFRandom Forest <=GBDT<=XGBoost29 Kaggle winner solution 17 GBDT Boosting DNNDeep Neural NetworkRF Kaggle RF GBDT CARTClassification And Regression Trees L Breiman J Friedman 1984 90 the ensemble stacking/ Bagging & Boosting CARTrandomized variablesbootstrap samples Boosting GBDT RF XGBoost

AI<523

    20ms

KVKey-Value Tair ETL Tair

524>2017 Thrift RPC
Service DomainDomain=ABC ID Domain 1 ETL

AI<525

Domain Domain ETL ETL

ETL
IP Key Value
ETL Tair Squirrel Redis KV

526>2017 2 Console Settings MySQL Scheduler Settings Hive Map Reduce Job
Service Domain Settings Domain

AI<527 Domain

T+1
ETL Hive Hive Hive SQL

528>2017 3 Console Settings Scheduler Settings Storm Kafka Topic

AI<529

2

SUM SUM COUNT MAX MIN AVG DISTINCT COUNT LAST LIST ID 24

1. 1

530>2017

Key Key ${ ID}_${ }
2
RPC
Key-Value Storm Calc Bolt

AI<531 Mafka Delay Topic 4 24 Storm Third Party

532>2017 5 O(N) N
DISTINCT COUNT HyperLogLog

Domain Storage Domain Storage ID Storage Domain Storage

AI<533 6 Storage 1. Domain Storage Domain Storage 2. Storage Storage Storage 3. Domain Key-Value KV Storage Domain

534>2017 ---- SNAPSHOT KV KV Period ArchiveSyncArchive SNAPSHOT Diff SNAPSHOT Diff
7

AI<535 Sync SNAPSHOT KV SNAPSHOT Lease Diff Lease Lease Lease 0 Lease

KV

0 2. 3. Sync Sync Sync Sync

536>2017

Updater 5. Updater

KV

Key Storm
Third Party Updater CASCompare And Swap CAS

AI<537

Storm At Least OnceAt Most Once Exactly Once

538>2017 8

Key

AI<539 9 10 SNAPSHOT SNAPSHOT SNAPSHOT SNAPSHOT SNAPSHOT

540>2017
SNAPSHOT 11

AI<541

2011 107 CTO2016 2015

542>2017

     Pipeline

1.1---- Key-ValueKV

AI<543

Hive MySQL Kafka

544>2017

"""" 1.2---- HashMap HashMap Datahub

QPS 1 QPS 100
IO 500Mbps 1.5Gbps
Hive 10 TB
TP99 10ms

AI<545

2.1 TB KV KV Redis/MemcacheHBaseTair
----

2.2

2.2.1

546>2017

JSON - JSON 2. Hive String

IntegerDouble String

JSON A B
JSON

JSON 2~10 2.2.2 Pattern Java GzipSnappyDeflateLZ4

AI<547 2 10 300~~400 600~~800 Deflate 1~9 LZ4 01 LZ4 JNIJava UnsafeJava Safe https://github. com/lz4/lz4-java

548>2017 / / LZ4 Java UnsafeJava Safe Snappy 2.2.3 Pattern Pattern
2.3

2.3.1 KV

AI<549 ---- PushPullPush ----Push Pull Pull
Pull Pull Diff Pull ----RedoLog Diff Diff

550>2017 id MergeMerge Merge Merge Diff Diff ---- Diff Merge MerkleTree ---- RedoLog Diff RedoLog Merge Log

AI<551 RedoLog Merkle Tree Diff Hash Hash ---- Diff Hash Merkle Tree Merkle Tree Merkle Tree hash Hash Hash Hash Hash Hash Merkle Tree
Merkle Tree hash hash hash LogN

552>2017 2.3.2 ---- ThriftHTTP ListMap ...... 1. 1. Future 2. Future

AI<553
""update cache" "merge data search cacheupdate cache

554>2017

---- JVM GC Full GC JVM QPS Young GC Least Recently UsedLRU Young GC Full GC

AI<555 GC GC QPS QPS
JVM GC JVM Off Heap JVM EhcacheBigMemory JVM GC

2011 107 CTO2016 2015

556>2017 ETA

ETAEstimated time of Arrival 1 ETA ETA ETA ETA
1ETA ETA ETA 2 ---- ----

AI<557 ETA ETA 2ETA ETA GBDTGradient Boost Decision Tree RFRandomForest GBDT DT Boosting RF DT Bagging

558>2017 ETA GBDT Facebook 2014 GBDT LR [1] GBDT Facebook GBDT 0-1 0-1 n k k 1 n-1 0 3 3GBDT(Gradient Boost Decision Tree) GBDT

AI<559

GBDT ETA ETA ETA

4 ETA ETA
(1) a. b. (2) a. N N b. c. d.

560>2017 GBDT GBDT GBDT CARTClassification And Regression Trees CART CART (1) a. n b. GBDT

GridSearch+CrossValidation +
c. GBDT 4~6

(2) 50% 50% GBDT 50% GBDT x GBDT x [25,20,22,....,30,28] GBDT

OneHotEncoder

AI<561

5.OneHotEncoder( ) GBDT x [25,20,22,....,30,28] ID ETA OneHotEncoder 0-1 5 3 bit {100} {0010} {1000010} CART 7 GBDT CART 1000+

562>2017

ETA C R N N

: N N K K N N C N C 2. : Xi i =abs( - ) N P(Xi <= N) 6 7 6

AI<563 7N N N 1. N N N N ETA N C R N 2. Xi i =( - ) Xi < 0 0 < Xi <= N N Xi > N N P(Xi <= N) N ( 7)

5% 40%

564>2017

GBDT +Ridge base modelGBDT ETA N (MAE) 3.4% 1.7 N 2.2 N (MAE) 2.56% 1 N 1.6 N 3.46 (MAE) 3.1% 1.4 N 1.7 N GBDT ETA

ETA ETA GBDT+OneHotEncoder ETA ETA ETA

AI<565 ETA

[1] He X, Pan J, Jin O, et al. Practical Lessons from Predicting Clicks on Ads at Facebook[C]. Proceedings of 20th ACM SIGKDD Conference on Knowledge Discovery and Data Mining. ACM, 2014: 1-9. [2] https://www.csie.ntu.edu.tw/~r01922136/kaggle-2014-criteo.pdf. [3] GitHubguestwalk.

566>2017

1600

40
""----O2O

AI<567 2015 41 32 28 20%

DVRPDynamic Vehicle Routing Problem 1

568>2017

O2O /
3 1 50 1-2 / /
e.g. LBS

AI<569

570>2017

TSP BP VRP """"

AI<571

572>2017

1 2

AI<573

574>2017
t

"Garbage ingarbage out"

AI<575

App "" /

576>2017 4 10 90%

NP-Hard 50 200 5 pow(200,50)*10 NP-Hard 2~3
"No Free Lunch Theory" Building Blocks

AI<577 KM 50% 99% 30ms 2~3 50

578>2017
/ / /

AI<579

"No measurement, No improvement" A/B

580>2017

A/B 5 6 2.9 4.7
19% 5 4

O2O

AI<581

3 KTV
O2O 1
2 O2O
3
4

582>2017 LTVlife-time value)
1 2 3" O2O "

kv

AI<583
O2O

1 2 3

584>2017

. 5 5% 25%~85% : COX-PH T y X y P(y|X) COX T T t<T t t' t' T (flag) (t') P(t'|X)

AI<585 Tx Tx COX XX

586>2017

AI<587

         look-alike

1 2

588>2017 3

AI<589

30% A B
Deal POI

1~4 6

590>2017

2015
1.0 Deal ""Deal
4 1 Deal deal_score = ((count(payorder) * ^ i) count(payorder) Deal i 1 28 (<1)Deal
Top N Deal Deal POI 200km Deal Deal Deal Deal UV/ UV /

AI<591 POI

Deal

POI Deal Deal Deal Deal Deal Deal Deal 2016 POI POI Deal POI POI

Deal POIPOI 30 Deal >=30 POI 30
Deal POI POI 2016 Q2 F POI F POI F POI ID POI 2.0 1.0

592>2017 2.0 POI
3 3 POI POI Rerank

POI POI ID

POI POI ""

AI<593

POI Top POI
Top POI POI POI POI

POI ItemCF UserCF ItemCF POI Item/User POI
POI POI POI
POI UUID POI

594>2017 | N (i) | POI i | N (i) Ç N ( j) | POI i j POI u POI j N (u) POI S( j, K ) POI j K POI wji POI j i rui u POI i " """"" POI POI POI POI POI Query POI Query 10 POI POI Query+City Key POI Query Key""Query POI """" POI A B C A B A B B A POI A B

A C

AI<595

l POI i j Nijl POI i j (<1)

ALS POI POI POI

Case 2. POI Deal POI

596>2017 3. POI POI 4. POI 5. POI 2 3

Rerank Location-Based 1357 2468 Location-Based 2016 Q1 Rerank Deal Deal 30/180 /
Q2 POI POI

AI<597 Q2 Rerank
HOUR_OF_DAYDAY_OF_WEEKCITY_ID onehot one-hot HOUR_OF_DAY 0~11 12 ~18 19 ~23 DAY_OF_WEEK CITY_ID ID
110 100 1

598>2017 10 POI 1 1000 XGBoost Rerank POI POI Rerank User-POI User 7 / POI User-POI POI POI

AI<599

User POI CTR/CVR

            Hive     ISample  label  ETL

ISample POI ID POI ID POI UUID Feature Sample & XGBoost Scala Spark XGBoost Spark XGBoost AUC ABTest

600>2017

Rerank Deal POI Deal POI

Hive ETL POI POI POI /
Spark User CFPOI CF ETL
Storm user__action_basic / POI/ Deal POI/Deal

AI<601

DataSet

ElasticSearch ES POI/Deal POI ES Location-Based ES KV ES
DataHub DataHub Hive DataHub Tair Key Tair DataHub HDFS Tair
Tair DataHub value Tair Spark Tair Hive Ready

API API ID App App

602>2017 Rerank rank Rerank Rerank POI

RecommendServicePublisher Client Request Booth UUID

AI<603 2. Context global ID UUID ABTest Strategy 3. Booth Handler AbstractHandler rerankpost rerank 4. Handler Client Handler

604>2017

Handler Strategy SelectRule Booth Strategy ABTest Baseline Strategy Strategy SelectRuleBaseline SelectRuleLocation-Based SelectRule Rule
SelectRule Selector Selector Location-Based Rule POI POI Selector ES DataHub Cache
Merge Merge Rerank Location-Based 246
SelectRule POI
POI POI Rerank Deal Deal Rerank
PostRerank Case

AI<605 Falcon JVM FullGC Thread block ES ES
Booth Hystrix Rerank Rerank

Debug UUID POI/Deal PM RD Case

/ 20

606>2017

Deal POI Deal POI """"

POI POI

POI " " POI Deal "" POI """" "" POI / POI POI / POI =1 POI POI CF POI Query POI Query CF Query

AI<607

                            Query  POI   POI  POI

POI POI
POI POI CF PUSH POI PUSH

20 Rerank FFMUser CF LLR /
User /Listwise DNN UV/ UV / / Selector/Rerank Filter/ Merge ""

608>2017 POI/Deal

/ ""

""

AI<609 "" / Appi PC App /

2010 2011

610>2017

O2O POI """" """" """"""" """"" " CS"Deal ""POI " CS""" POI

2015 Q2

AI<611 / UV Query-> POI/Deal -> ->
PV/ PVPage View Deal
/ 20 POI POI 1 0.5 0

612>2017 bad case case

/ Query
POI
A/B Testing

2015 Q2 Query "" 30% A B

AI<613 Query POI Deal ""

POI Deal POI POI Deal

614>2017 POI " " POI Deal Deal POI Deal POI """"POI " CS"Deal ""POI 2015 Q3 Query Query 56% POI """ """ POI "" POI ""Term"

AI<615 "POI Deal Deal
""""" "" - """4 "" " - " "" POI
POI Deal
POI Deal
Query 4
"" POI

616>2017 POI Deal

2015 Q4 Query 32% POI POI Deal27% POI POI 30% Query case 15% Query """" 7% Query "" "" 5% Query "" """" 3% Query "798 3D " " 3D "

AI<617 & Query Query Query & Session
POI 2016 Q2 case 3

618>2017 case POI "" Deal POI case case Query Chunk Term Chunk Chunk

Query """"""

AI<619 """" """""" """ "2015 Q4 Query 8 POI POI
Query Term Query Term """ """""" " POI Term tag

620>2017 tagPOI POI tag"" POI " " Term POI "" ChunkChunk Term Chunk Query Term Chunk Chunk tag Chunk BMESBeginMiddle EndSingle" "Chunk "" B"" E"" Chunk S- "" S-POI "" B- Chunk 123 CRF
Term tag DealPOI POI

AI<621 Chunk tag Chunk tag Term tag CRF++ CRF++ Query" " Query Term tagChunk Term Query Chunk Chunk tag Query >POI> > """ " POI "" POI "" Rerank

622>2017

Query POI Deal case 2016 Q3 POI >=300KM 0300KM
POI
POI

TF-IDF RQ,D t Q Term H t tft, f t f l f f wf f POI Deal idft Term t

AI<623 "" POI "" """"""3 Term"" """"2 Term POI 1.5 POI "" ""POI (Brand Name) """ "POI """" """" """"POI"" Deal Term""""POI "" POI "" IDF Term Term Query
k1 b BM25 max if Term Query idft¢ Term Query Term Term Query Query 4 Super important POI

624>2017 Required" """" " Important" """ "" Unimportant "" Query 4 Query Term Term Query PMIIDF query / Term Query Chunk tag XGBoost

AI<625

Rerank POI POI

""

1. """"""" " 3. "" 4. Term """" 5. chunk tag"""" "" 6. Chunk Query 7. Term Query "" 0.48"" 0.39 "" 0.55 8. 9. POI "" POI POI 10. POI "" Chunk "" 11. """"

626>2017

""""

2010 2011

AI<627

DSP CTR

LBS

P2P

628>2017

 ADX ADX

" / "" / "

ID IMEI IDFA Hive Redis

AI<629 1.
2.
Spark ML Spark ML A B (A B) A " / "" / " " / "

630>2017
MySQL

Redis
A/B base

AI<631

DSPDemand-Side Platform

72

AdaBoost

632>2017 AdaBoost 1 icon-code one-hot 123 13 2
AdaBoost Gentle AdaBoost 1 AdaBoost

AI<633
Spark Tair key value AdaBoost 100

Gentle AdaBoost JSON Tair ADX Tair

634>2017

Tair DSP

/
/ + +

200   8090606030  80+90+60<200    200

AdaBoost A/B testing

AI<635

Query Targeting App

NLP
TF/IDF - -
""" / " c1 = "" c2 = """ / " c3 =

636>2017 c4 = " / " tfidf=(c2/c1)×log(c3/(c4+1)) topN TF-IDF TF-IDF " "
Spark
TF-IDF Tair key TF-IDF value

Tair Kafka ID Tair Tair ID Tair

AI<637 Tair
1 0.5 72 0.5 = 1 × e-*
w w'

ID Tair Tair
A 8:30 Spark Streaming Redis 8:30 1100 Spark

638>2017 Streaming 11:00 11:00 16:00 ADX 16:00 11:00

LR

[0,1]

x1,..., xn

AI<639 CTR A/B testing

Friedman J, Hastie T, Tibshirani R.(2000), Additive Logistic Regression: A Statistical View of Boosting, Annals of Statistics, 28, 337-307.

2012 2016 2011 2014 2016

640>2017 DSP

DSPDemand-Side Platform[1] AdExchange[2] AdExchange DSP Cookie Mapping [3] App DSP DSP / ROI [4] App DSP DSP DSP DSP AB / DSP

DSP

AdExchange 2.

AI<641

1 1 Gateway CTR AdServerAdServer RecServer PredictorServerCTR/ AdServer v ctr v*ctrt top

642>2017 2 2 Gateway Gateway AdServer AdServer RecServer PredictorServer Gateway DSP RecServer PredictorServer 1 2 AdServer RecServer PredictorServer CTR CTR

AI<643 3 DSP CTR

/ / / /

644>2017

O2O O2O
1.

PC IP PC
2.
1 2 userid userid 3. / / userid userid

user-based item-based

AI<645 item-based item item A item B item A item B item B item B item A item A item-based
user-based

O2O
FM /

CTR AdServer PredictorServer AdServer v ctr v*ctrt top

k a* vi *ctrit + b i =1

1

k t,a,b t

646>2017 ctr t ctr a,b DSP ROI a,b ROI 1 CTR CTR CTR
CTR RecServer AdExchange DSP DSP ROICTR
useradpublisher [5] 1.

deal deal CTRCVRPVUV 3.

AI<647 4. match match

LR+ + + + FM FFM
/
LR+
id one-hot encoding

648>2017 DSP top20
4.
5. CTR

q: p: w:

2

CTR

/ FFM[6] FFM CTR

AI<649 position bias position bias

CTR CVR ctr_p cvr_p i item ctr_0/ctr_p cvr_0/ctr_p 0

vctrt 1 v
eCPM[7]eCPM vctr*1000 eCPM [8] CTR eCPM AdExchange
t a b CTR CPC ROI CTR CPC ROI

650>2017 DSP CVR CVR FFM + FFM (knn)k n
1 k label GBDT FM GBDT

AUC ROC AUC
Facebook NENormalized Entropy[9] NE

AI<651 3 N yi i lable +1 -1 pi i P AUC NE AB ABtest CTR 30% CTR 13% CVR 10%

AUC CTR CVR AUC AUC

DSP

652>2017

CTR
DSP CTR adx CPC CPC CPC [10]CPC
/ / DSP AdExchange DSP DSP
https://en.wikipedia.org/wiki/Demand-side_platform 2. https://en.wikipedia.org/wiki/Ad_exchange 3. https://developers.google.com/ad-exchange/rtb/cookie-guide?hl=en

AI<653 4. https://en.wikipedia.org/wiki/Return_on_investment 5. http://www.xiutx.cn/archives/263 6. https://www.csie.ntu.edu.tw/~cjlin/libffm/ 7. http://baike.baidu.com/view/1666309.htm 8. http://book.douban.com/subject/26596778/ 9. http://www.herbrich.me/papers/adclicksfacebook.pdf 10. https://en.wikipedia.org/wiki/PID_controller

2015 DSP 2014 7 CPS DSP DSP

---- 2013 11 1600 4

1

<655 " O2O " 1 App --> --> -->

2 2~3 2

656>2017

3 2013 2017 10 4 2000 1600 3 120 + QPS 40

4

<657 4
IM

TP99

5

658>2017 5

--> -->
->

<659

6
6

660>2017 7 7

Holt-Winters

<661

8 TP99 SOP CPU 8

662>2017

& 9 & & 9

DB

<663 Cache Hystrix

10

664>2017 10

11

<665 SLA 11 N

12 Hystrix 12

666>2017

13 13

Tair

<667

14 14

668>2017

2016

MCC CAT CAT DIGGER DIGGER FALCON MtFalcon----Open-Falcon

SRE

<669

SRESite Reliability Engineering Google 2003

"" SRE

SLA
SRE
SRE 2. 3. SRE SRE

4 Varnish/Squid LAMP

670>2017

Web API PHP JavaPython C++ 2014
SRE SRE

<671

VM CPU VM VM VM VM CPU
VM VM VM

672>2017 10 VM 5 SRE VM 90% VM SRE 3 9

BGP
MGW"MGW----"NAT API

<673

Web 2. SOP

674>2017

<675 SRE TODO

676>2017

& 300 Nginx rewrite set rewrite break rewrite set Nginx

<677 SOP 5 DAU

678>2017 session session IMGW1 TCP

SLA HTTP accesslog

<679 TODO

SOP 2. 3. TODO

Server Server HTTP 1.0/1.1 HTTP

BGP BGP BGP
HTTP DNS Shark

680>2017 Server App SDK App DNS Shark "" Shark 3. SPDY HTTP 2.0Server HTTP 2.0 bug

AI

2014 SRE

<681 Mt-FalconOpen-Falcon

Zabbix 2W+ 450W+ Zabbix Zabbix

API

Open-Falcon Mt-Falcon Open-Falcon

682>2017 Open-Falcon Open-Falcon Mt-Falcon

<683 Mt-Falcon Open-Falcon ACK OpenTSDB

Agent 1. Agent Agent Transfer Agent 0.5 Transfer 1W 60s <100W 2. Agent Tag 4
3. coredump core
4. Agent Falcon-Agent 100 G Go-LoggerGo-Logger Golang Log

684>2017 Go-Logger Log Go-Logger Go-Logger https://github.com/donnie4w/go-logger 5. hostname hostname Endpoint host hostname hostname hostname hostname Falcon-Agent hostname /etc/ sysconfig/network hostname IP 6. Falcon-Agent Falcon-Agent HBS Falcon-Agent Redis Falcon-Agent Redis Falcon-Agent Falcon-Agent 5 Falcon-Agent
HBS 1.

HBSHeartBeat Server Judge JSON-RPC JSON-RPC RPCRPC

<685 TCP Go encoding/json encoding/json HBS 50 G RPC+MessagePack JSON-RPC encoding/json MessagePack MessagePack encoding/json HBS 6G RPC MessagePack https://github.com/ugorji/ go/tree/master/codec#readme 2. Group Group HBS HostID 3.
HBS ID
HBS ID+ActionID
4. 5

686>2017

10 30 1 3 6 1 3 Transfer 1. Endpoint Falcon Transfer Endpoint Endpoint
Transfer Endpoint Endpoint Endpoint xxx Endpoint
2. OpenTSDB OpenTSDB Transfer

<687 Redis Transfer Redis Transfer Transfer Judge 1. 80% Judge Judge 11 Judge Judge
2. Judge Judge Judge Judge K M Load Judge Alarm
3. Action 20
4. ACK ACK Zabbix ACK ACK

688>2017 ACK Alarm Endpoint+Metric+Tags ACK
ACK Transfer ACK Transfer ACK Endpoint+Metric+Tags Judge Judge ACK Judge ACK EventID ACK 5. Tag Tag Metric df.bytes.free.percent Tags mount=/dev/shm Tag Metric df.bytes.free.percentTags ^mount=/dev/shm Judge
6. plus_judge Judge plus_judge plus_judge Graph 1. MySQL Redis+Tair

<689 Redis Cluster Redis Cluster Go client https://github.com/chasex/redis-go-cluster BoltDB Git https://github.com/laiwei/falcon-index RedisTairBoltDB 2. 1 Graph Tair indexcache
Graph 6 unindexcache 3.

690>2017 12 RRD RRD 12 12 RRD 12

12 12 RRD RRD RRD RRD RRD 12 RRD
Alarm 1. Alarm Metric 3 5 2.

net.if.in.Mbps net.if.out.Mbps icmp.ping.alive cpu.steal CPU

prod SRE +RD staging RD test

<691

Metric Metric
Alarm Redis Alarm Redis Redis Alarm sender
IM
p0: p1: p2:

692>2017 p3: p9: + + + 6. InfluxDB InfluxDB MySQL Top10 7 BG Top20 7. Top10 >=90 50~90
8. Action Action
9. base
" net.if.in.Mbps", " net.if.out.Mbps", " cpu.idle", " df.bytes.free.percent",

<693 " df.inodes.free.percent", " disk.io.util", " icmp.ping.alive", " icmp.ping.msec", " load.1minPerCPU", " mem.swapused.percent", " cpu.steal", " kernel.files.percent", "kernel.coredump", " df.mounts.ro", " net.if.change", Portal/Dashboard 1.

API
shift Dashboard shift
Dashboard

694>2017 7. Dashboard Endpoint Metric
8. screen screen
9. prod prod staging test

Ping Fping Ping Ping Ping Ping
Portal HBS string_judge
nodata diff pdiff 10

<695 OpenTSDB OpenTSDB 4. Judge Judge plus_judge
ID 3
Judge plus_judge plus_judge

Mt-Falcon Zabbix QPS 100W+ Open-Falcon Feature Merge PR

SRE 2015

Python+Nmap

Wikipedia

<697

1

Nmap 3

698>2017 filtered TCP SYN TCP SYN

TCP Fin/Null/Xmas/Maimon Idle TCP openclosed filtered

TCP/IP 3

<699 TCP SYN SYN TCP 3

2

700>2017 Nmap Python Nmap Nmap ---- 2 Masscan Masscan Masscan Masscan Zmap Nmap TCP SYN IP Masscan

<701 3. 4. 100% 100% 5. IP IP IP 6. Masscan+Nmap Masscan Banner Masscan Nmap Masscan Nmap

702>2017 Masscan+Nmap & DPDK+Storm+Nmap ----
Masscan / Nmap

IP
API

Masscan Nmap

<703 4.

Web Web Web Zabbix Web Web Web

704>2017

UDP TCP UDP ---- 2016 8 The Shadow Brokers Adaptive Security ApplianceASA PIX ExtraBacon SSH telnet ASA

UDP / IP

2015

<705 Android Binder

Android Android 2016 10 Android 6.0 6.0.17.0
Android Android https://v.qq.com/x/page/u0543o9s9fk.html 6.06.0.17.0 Android expexp APK bin Android pin password CVECommon Vulnerabilities and Exposures CVE-2016-3749 CVE-2016-3908 2016 5 Google6 1 Google 4 13 duplicate

706>2017 Android 6.0.1 Android Google Google patch Google 7 CVE CVE-2016-3749 patch setLockPassword() setLockPattern()

<707 Google Google 2016 7 20 8 31 CVE-2016-3908 10

708>2017

https://v.qq.com/x/page/o0543t0uwkw.html CVE-2016-9567 setmDNIeScreenCurtain()

<709 https://v.qq.com/x/page/y0543skh3t7.html

https://v.qq.com/x/page/a0543ilaytw.html NFC tag Android NFC Android Binder Android Binder Android Binder "In the Android platform, the binder is used for nearly everything that happens across processes in the core platform." Dianne Hackborn,Google https://lkml.org/lkml/2009/6/25/3 Android Binder Dianne Hackborn OpenBinder Android IPC Android Linux socket Client-Server Android Binder /dev/binder Binder Token Binder Proxy PatternMediator Pattern Bridge Pattern Binder BinderBinder ObjectBinder ProtocolIBinder interface Binder TokenAIDLAndroid interface definition languageServiceManager Binder kernel

710>2017 BinderAndroid Binder Android Interprocess Communication drozer drozer MWR Android Metasploitdrozer ConsoleAgentServer Agent Console PC

<711 Agent internet App ServerSocket 31415 Console Server Agent Dalvik App IPC intent Agent Dalvik drozer drozer demo new Java Java drozer .java APK drozer APK Agent Java

drozer Android Agent USB Console Agent Embedded Server Agent

712>2017 Server host port SSL

<713 launcher Agent Agent server IP port Agent session Android drozer Server Agent Console Server Agent Sdcard drozer-modules curesecmetall0id Android 4.3

714>2017 drozer

<715 drozer drozer 1. 2.execute() drozer fuzzing fuzzer drozer fuzzing drozer Android Binder fuzzing fuzzing intentfuzzing fuzzing intent 15 360

716>2017 Android crash Android Android Google fuzzing Android adb shell service list shell Nexus 5X 7.12 126 [ ]
lock_settings int fuzzingfuzzing

<717 shell fuzzingadb shell service call lock_settings CODE i32 -1 CODE i32 32 drozer apk Java Ramada drozer execute() fuzzing drozer

718>2017 module install MODULE_NAME drozer Python Java Python fuzzing fuzzing logcat Crash Crash lock_settings fuzzer

AIDL Android Android SDK tools AIDL Java AIDL Bound Services

Java

<719

AIDL Java

Android Android BinderDemo

720>2017 Java shell

Runtime runtime = Runtime.getRuntime(); Process proc = runtime.exec(command); shell service call lock_settings 10 i32 0

2017 Android 4 Android

<721 Android Code Arbiter

Android Code Arbiter Android Studio
1. Android Studio IDE Android Studio

IDE API UI

PMD FindBugs PMD Java FindBugs Java class FindBugsFindBugs JAR JAR

722>2017

FindBugs FindBugs JAR FindBugs JAR JAR FindBugs Find Security Bugs Web Android

Android Webview Broadcast

<723 Find Security Bugs
2. FindBugs class FindBugs .class class FindBugs class
2.1 sawOpcode() Android
Android getExternalCacheDir() getExternalCacheDirs() getExternalFilesDir() getExternalFilesDirs() getExternalMediaDirs() Environment.getExternalStorageDirectory() Environment.getExternalStoragePublicDirectory()

public class ExternalFileAccessDetector extends OpcodeStackDetector { private static final String ANDROID_EXTERNAL_FILE_ACCESS_TYPE = "ANDROID_EXTERNAL_FILE_ACCESS"; private BugReporter bugReporter;

724>2017 public ExternalFileAccessDetector(BugReporter bugReporter) { this.bugReporter = bugReporter; } @Override public void sawOpcode(int seen) { //printOpCode(seen); if (seen == Constants.INVOKEVIRTUAL && ( getNameConstantOperand().equals("getExternalCacheDir") || getNameConstantOperand().equals("getExternalCacheDirs") || getNameConstantOperand().equals("getExternalFilesDir") || getNameConstantOperand().equals("getExternalFilesDirs") || getNameConstantOperand().equals("getExternalMediaDirs") )) { // System.out.println(getSigConstantOperand()); bugReporter.reportBug(new BugInstance(this, ANDROID_EXTERNAL_FILE_ ACCESS_TYPE, Priorities.NORMAL_PRIORITY).addClass(this).addMethod(this). addSourceLine(this)); } else if(seen == Constants.INVOKESTATIC && getClassConstantOperand().equals("android/os/Environment") && (getNameConstantOperand().equals("getExternalStorageDirectory") || getNameConstantOperand().equals("getExternalStoragePublicDirectory"))) { bugReporter.reportBug(new BugInstance(this, ANDROID_ EXTERNAL_FILE_ACCESS_TYPE, Priorities.NORMAL_PRIORITY).addClass(this). addMethod(this).addSourceLine(this)); } } } OpcodeStackDetector FindBugs
sawOpcode printOpCode(seen)
Constants.INVOKEVIRTUAL Constants. INVOKESTATIC getNameConstantOperand getClassConstantOperand
getSigConstantOperand bugReporter.reportBug BugInstance

<725

HIGH_PRIORITY NORMAL_PRIORITY LOW_PRIORITY EXP_PRIORITY IGNORE_PRIORITY

1 2 3 4 5

addClassaddMethodaddSourceLine

2.2 visitClassContext Android TrustManager TrustManager Server checkServerTrusted HTTPS

@Override public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { }

checkServerTrusted

public class WeakTrustManagerDetector implements Detector { ... public WeakTrustManagerDetector(BugReporter bugReporter) { this.bugReporter = bugReporter; } @Override public void visitClassContext(ClassContext classContext) {

726>2017 JavaClass javaClass = classContext.getJavaClass(); //The class extends X509TrustManager boolean isTrustManager = InterfaceUtils.isSubtype(javaClass,"javax.net. ssl.X509TrustManager"); boolean isHostnameVerifier = InterfaceUtils. isSubtype(javaClass,"javax.net.ssl.HostnameVerifier"); // if (!isTrustManager && !isHostnameVerifier) return; if (!isTrustManager && !isHostnameVerifier){ for (Method m : javaClass.getMethods()) { allow_All_Hostname_Verify(classContext, javaClass, m); } } Method[] methodList = javaClass.getMethods(); for (Method m : methodList) { MethodGen methodGen = classContext.getMethodGen(m); if (DEBUG) System.out.println(">>> Method:" + m.getName()); if (isTrustManager && (m.getName().equals("checkClientTrusted") || m.getName().equals("checkServerTrusted") || m.getName().equals("getAcceptedIssuers"))) { if(isEmptyImplementation(methodGen)) { bugReporter.reportBug(new BugInstance(this, WEAK_TRUST_MANAGER_TYPE, Priorities.NORMAL_PRIORITY). addClassAndMethod(javaClass, m)); } ...... classContext.getJavaClass javaClass.get- Methods classContext. getMethodGen isEmptyImplementation

private boolean isEmptyImplementation(MethodGen methodGen){ boolean invokeInst = false; boolean loadField = false; for (Iterator itIns = methodGen.getInstructionList(). iterator();itIns.hasNext();) { Instruction inst = ((InstructionHandle) itIns.next()). getInstruction();

<727 if (DEBUG) System.out.println(inst.toString(true)); if (inst instanceof InvokeInstruction) { invokeInst = true; } if (inst instanceof GETFIELD) { loadField = true; } } return !invokeInst && !loadField; } return true/false
2.3 source Android Intent source

EditText android/widget/EditText.getText()Landroid/text/Editable;:TAINTED - Intent android/content/Intent.getAction()Ljava/lang/String;:TAINTED android/content/Intent.getStringExtra(Ljava/lang/String;)Ljava/lang/ String;:TAINTED ...... - Bundle android/os/Bundle.get(Ljava/lang/String;)Ljava/lang/Object;:TAINTED android/os/Bundle.getString(Ljava/lang/String;)Ljava/lang/String;:TAINTED ...... sink sink

728>2017 public class CommandInjectionDetector extends BasicInjectionDetector { public CommandInjectionDetector(BugReporter bugReporter) { super(bugReporter); loadConfiguredSinks("command.txt", "COMMAND_INJECTION"); } BasicInjectionDetector loadConfiguredSinks sink

java/lang/Runtime.exec(Ljava/lang/String;)Ljava/lang/Process;:0 java/lang/Runtime.exec([Ljava/lang/String;)Ljava/lang/Process;:0 java/lang/Runtime.exec(Ljava/lang/String;[Ljava/lang/String;)Ljava/lang/ Process;:0,1 java/lang/Runtime.exec([Ljava/lang/String;[Ljava/lang/String;)Ljava/lang/ Process;:0,1 java/lang/Runtime.exec(Ljava/lang/String;[Ljava/lang/String;Ljava/io/ File;)Ljava/lang/Process;:1,2 java/lang/Runtime.exec([Ljava/lang/String;[Ljava/lang/String;Ljava/io/ File;)Ljava/lang/Process;:1,2 java/lang/ProcessBuilder.([Ljava/lang/String;)V:0 java/lang/ProcessBuilder.(Ljava/util/List;)V:0 java/lang/ProcessBuilder.command([Ljava/lang/String;)Ljava/lang/ ProcessBuilder;:0 java/lang/ProcessBuilder.command(Ljava/util/List;)Ljava/lang/ ProcessBuilder;:0 dalvik/system/DexClassLoader.loadClass(Ljava/lang/String;)Ljava/lang/ Class;:0 BasicInjectionDetector
getInjectionPoint WebView.loadurl @Override protected InjectionPoint getInjectionPoint(InvokeInstruction invoke, ConstantPoolGen cpg, InstructionHandle handle) { assert invoke != null && cpg != null; String method = invoke.getMethodName(cpg); String sig = invoke.getSignature(cpg); // System.out.println(invoke.getClassName(cpg)); if(sig.contains("Ljava/lang/String;")) { if("loadUrl".equals(method)){ if(sig.contains("Ljava/util/Map;")){

<729 return new InjectionPoint(new int[]{1}, WEBVIEW_LOAD_ DATA_URL_TYPE); }else{ return new InjectionPoint(new int[]{0}, WEBVIEW_LOAD_ DATA_URL_TYPE); } }else if("loadData".equals(method)){ return new InjectionPoint(new int[]{2}, WEBVIEW_LOAD_ DATA_URL_TYPE); }else if("loadDataWithBaseURL".equals(method)){ //BUG return new InjectionPoint(new int[]{4}, WEBVIEW_LOAD_DATA_URL_TYPE); } } return InjectionPoint.NONE; } InjectionPoint sink webView.loadUrl(url) new int[]{0} getPriorityFromTaintFrame

@Override protected int getPriorityFromTaintFrame(TaintFrame fact, int offset) throws DataflowAnalysisException { Taint stringValue = fact.getStackValue(offset); // System.out.println(stringValue.getConstantValue()); if (stringValue.isTainted() || stringValue.isUnknown()) { return Priorities.NORMAL_PRIORITY; } else { return Priorities.IGNORE_PRIORITY; } } fact.getStackValue isTainted Priorities. NORMAL_PRIORITY Priorities. IGNORE_PRIORITY

730>2017 2.4 2.2 Android Intent try catch FindBugs class class javap
class Intent Map Key Value Map

<731 private Map<String, List> get_line_location(Method m, ClassContext classContext){ HashMap<String, List> all_line_location = new HashMap<>(); ConstantPoolGen cpg = classContext.getConstantPoolGen(); CFG cfg = null; try { cfg = classContext.getCFG(m); } catch (CFGBuilderException e) { e.printStackTrace(); return all_line_location; } for (Iterator i = cfg.locationIterator(); i.hasNext(); ) { Location loc = i.next(); Instruction inst = loc.getHandle().getInstruction(); if(inst instanceof INVOKEVIRTUAL) { INVOKEVIRTUAL invoke = (INVOKEVIRTUAL) inst; if(all_line_location.containsKey(invoke.getMethodName(cpg))){ all_line_location.get(invoke.getMethodName(cpg)). add(loc); }else { LinkedList loc_list = new LinkedList<>(); loc_list.add(loc); all_line_location.put(invoke.getMethodName(cpg), loc_list); } // } } } return all_line_location; } Exception FindBugs Exception

exceptionTable[i].getStartPC try catch excep- tionTable[i].getEndPC try catch public int[] getExceptionScope(){ try { CodeException[] exceptionTable = this.code. getExceptionTable(); int[] exception_scop = new int[exceptionTable.length * 2]; for (int i = 0; i < exceptionTable.length; i++) {

732>2017

} }

exception_scop[i * 2] = exceptionTable[i].getStartPC(); exception_scop[i * 2 + 1] = exceptionTable[i].getEndPC(); } return exception_scop; }catch (Exception e){ return new int[0];

class FindBugs code-length

public int get_Code_Length(String firstLineCode){ try{ String[] split1 = firstLineCode.split("code_length"); // System.out.println(split1[split1.length-1]); byte[] code_length_bytes = split1[split1.length-1].getBytes(); byte[] new_code_bytes = new byte[code_length_bytes.length]; for(int i=0; i<code_length_bytes.length; i++){ // System.out.println(); if(code_length_bytes[i]<48 || code_length_bytes[i]>57){ new_code_bytes[i] = 32; }else{ new_code_bytes[i] = code_length_bytes[i]; } } return Integer.parseInt(new String(new_code_bytes).trim()); }catch(Exception e){ e.printStackTrace(); } return 0; }

try catch Intent try catch get_code_line_index

<733 private void analyzeMethod(JavaClass javaClass, Method m, ClassContext classContext) throws CFGBuilderException { HashMap<String, List> all_line_location = (HashMap<String, List>) get_line_location(m, classContext); Code code = m.getCode(); StringCodeAnalysis sca = new StringCodeAnalysis(code); String[] codes = sca.codes_String_Array(); int code_length = sca.get_Code_Length(sca.get_First_Code(codes)); int[] exception_scop = sca.getExceptionScope(); for(int i=1; i<codes.length; i++){ int line_index = sca.get_code_line_index(codes[i]); if (line_index < code_length){ if(codes[i].toLowerCase().contains("invokevirtual") && (codes[i].contains("android.content.Intent.get") || codes[i].contains("android.os.Bundle.get"))){ if(exception_scop.length == 0){ ...... }else{ boolean is_scope = false; for(int j=0; j<exception_scop.length; j+=2){ int start = exception_scop[j]; int end = exception_scop[j+1]; if(line_index >= start && line_index <= end){ is_scope = true; } if(is_scope){ break; } } if(!is_scope){ String method_name = get_method_ name(codes[i]); if(all_line_location.containsKey(method_ name)){ for(Location loc : all_line_location. get(method_name)){ bugReporter.reportBug(new BugInstance(this, LOCAL_DENIAL_SERVICE_TYPE, Priorities.NORMAL_PRIORITY). addClass(javaClass).addMethod(javaClass, m).addSourceLine(classContext, m, loc)); } }else { bugReporter.reportBug(new BugInstance(this, LOCAL_DENIAL_SERVICE_TYPE, Priorities.NORMAL_PRIORITY). addClass(javaClass).addMethod(javaClass, m)); } } } }

734>2017 } } }

findbugs.xml

Detector BugPattern "Android " "Android " messages.xml

<Detector class="com.h3xstream.findsecbugs.android.

LocalDenialOfServiceDetector">

Local Denial of Service.

Intent

:

bundle.getString(""); //  try/catch

intent.

getStringExtra(""); //  try/catch

:

 Intent 



try {

bundle.getString("");

intent.getStringExtra("");

}catch (Exception e){}



 ]]>

Maven FindBugs JAR <735 4. Android Android Studio Code Arbiter Code Arbiter Code Arbiter 1. Code Arbiter Android 2. 5. 2016 / Freebuf O2O PV 4 1. <737 2. 3. 4. 1. 10 0 H5 "" Native 33 10 0 738>2017 2. ---- <739 """" Push Push PV UV 3. Web Service Web MAPI / 200 Service 740>2017 ---- <741 " " Web MAPI 742>2017 <743 H5 "" "" Web API API 744>2017 App Native "" 6 API 3 ---- ---- 3 Web 5 <745 CAT Web CAT GitHub PV 1 3 API API API API 1 5 7 746>2017 PV 17 1 DB <747 Web * 748>2017 PTP <749 30+ 750>2017 PTP BETA PPE BETA PPE BETA QA BETA JMeter PPE JMeter <751 PTP PTP + PTP Docker 752>2017 PTP <753 PTP CPU PTP BETA PPE 754>2017 ---- <755 ---- TCPCopy AB A B A TCPCopy B A TCPCopy 40 QPS PTP JMeter 756>2017 JMeter id "" A B <757 Web TCPDump dump Web + Nginx Access Log App Nginx Access Log token 758>2017 csv <759 ---- ----

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

2018年美团点评技术年货（合）.pdf.md

2018年美团点评技术年货（合）.pdf.md

config/cucumber.yml ##YAML Template --ios: IDEVICENAME='ios' android: IDEVICENAME='android'

218>2017

<223 ""PartnerShip PS Anticorruption LayerACL Open Host Service Published Language Data Coupling

<287

3 "==""<"

SQL +

Block Cache Cache Cache Cache Cache server zone Server Client NameNode

474>2017

AI<553
""update cache" "merge data search cacheupdate cache

AI<567 2015 41 32 28 20%

576>2017 4 10 90%

Files

2018年美团点评技术年货（合）.pdf.md

Latest commit

History

2018年美团点评技术年货（合）.pdf.md

File metadata and controls

config/cucumber.yml ##YAML Template --ios: IDEVICENAME='ios' android: IDEVICENAME='android'

218>2017

<223 ""PartnerShip PS Anticorruption LayerACL Open Host Service Published Language Data Coupling

<287

3 "==""<"

SQL +

Block Cache Cache Cache Cache Cache server zone Server Client NameNode

474>2017

AI<553 ""update cache" "merge data search cacheupdate cache

AI<567 2015 41 32 28 20%

576>2017 4 10 90%

AI<553
""update cache" "merge data search cacheupdate cache