Optimizing the Pairs-Trading Strategy Using Deep Reinforcement Learning with Trading and Stop-Loss Boundaries

<table class="fixed-width table-group" id="tab3"><tr><td><table class="table"><colgroup><col style="width:9.07em"/><col style="width:3.39em"/><col style="width:3.39em"/><col style="width:3.39em"/><col style="width:3.39em"/><col style="width:3.39em"/><col style="width:3.39em"/></colgroup><tr><td class="thead-hr" colspan="7"><hr/></td></tr><tr class="thead"><td class="align_left"> </td><td class="align_center" colspan="6"><i>Action</i></td></tr><tr class="thead"><td class="align_left"> </td><td class="align_center">A0</td><td class="align_center">A1</td><td class="align_center">A2</td><td class="align_center">A3</td><td class="align_center">A4</td><td class="align_center">A5</td></tr><tr><td class="thead-hr" colspan="7"><hr/></td></tr><tr><td class="align_left">Trading boundary</td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M70" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.229,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.193,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M71" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.229,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.193,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M72" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.229,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.193,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M73" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M412 140C382 77 369 73 315 73H129L270 222C362 320 402 379 402 466C402 571 322 635 234 635C177 635 130 609 99 576L42 495L64 475C90 514 133 568 201 568C274 568 318 519 318 435C318 349 255 267 193 193C144 135 87 78 32 23V0H405C417 45 427 89 440 131L412 140Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.229,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.193,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M74" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M412 140C382 77 369 73 315 73H129L270 222C362 320 402 379 402 466C402 571 322 635 234 635C177 635 130 609 99 576L42 495L64 475C90 514 133 568 201 568C274 568 318 519 318 435C318 349 255 267 193 193C144 135 87 78 32 23V0H405C417 45 427 89 440 131L412 140Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.229,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.193,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M75" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M285 378C315 398 338 416 353 432C373 451 384 474 384 503C384 579 325 635 236 635H235C182 635 136 610 108 579L65 516L85 496C110 533 150 575 205 575C258 575 300 543 300 481C300 407 232 369 141 339L147 310C163 315 188 321 211 321C268 321 338 284 338 192C338 94 288 40 217 40C160 40 119 68 93 91C85 98 77 97 69 91C60 84 47 71 46 58C44 46 48 35 62 22C75 10 116 -12 162 -12C234 -12 424 62 424 224C424 297 373 359 285 376V378Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.23,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.194,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g></svg></span></td></tr><tr><td class="align_left">Stop-loss boundary</td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M76" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M412 140C382 77 369 73 315 73H129L270 222C362 320 402 379 402 466C402 571 322 635 234 635C177 635 130 609 99 576L42 495L64 475C90 514 133 568 201 568C274 568 318 519 318 435C318 349 255 267 193 193C144 135 87 78 32 23V0H405C417 45 427 89 440 131L412 140Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.229,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.193,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M77" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M285 378C315 398 338 416 353 432C373 451 384 474 384 503C384 579 325 635 236 635H235C182 635 136 610 108 579L65 516L85 496C110 533 150 575 205 575C258 575 300 543 300 481C300 407 232 369 141 339L147 310C163 315 188 321 211 321C268 321 338 284 338 192C338 94 288 40 217 40C160 40 119 68 93 91C85 98 77 97 69 91C60 84 47 71 46 58C44 46 48 35 62 22C75 10 116 -12 162 -12C234 -12 424 62 424 224C424 297 373 359 285 376V378Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.23,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.194,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M78" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M285 378C315 398 338 416 353 432C373 451 384 474 384 503C384 579 325 635 236 635H235C182 635 136 610 108 579L65 516L85 496C110 533 150 575 205 575C258 575 300 543 300 481C300 407 232 369 141 339L147 310C163 315 188 321 211 321C268 321 338 284 338 192C338 94 288 40 217 40C160 40 119 68 93 91C85 98 77 97 69 91C60 84 47 71 46 58C44 46 48 35 62 22C75 10 116 -12 162 -12C234 -12 424 62 424 224C424 297 373 359 285 376V378Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.23,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.194,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M79" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M456 178V225H360V632H320C217 496 115 347 20 206V178H280V106C280 40 276 34 189 27V0H445V27C364 34 360 39 360 106V178H456ZM280 225H82C149 335 214 431 278 520H280V225Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.23,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.194,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M80" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M456 178V225H360V632H320C217 496 115 347 20 206V178H280V106C280 40 276 34 189 27V0H445V27C364 34 360 39 360 106V178H456ZM280 225H82C149 335 214 431 278 520H280V225Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.23,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.194,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g></svg></span></td><td class="align_center"><span style="width: 27.6194ptpx;"><svg height="8.69875pt" id="M81" style="vertical-align:-0.3499298pt" version="1.1" viewbox="-0.0498162 -8.34882 27.6194 8.69875" width="27.6194pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M535 290V340H323V533H265V340H52V290H265V84H323V290H535ZM535 -22V28H52V-22H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.989,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.23,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,21.194,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g></svg></span></td></tr><tr class="table-tr"><td colspan="7"><hr class="tbody-hr"/></td></tr></table></td></tr></table>

<div>Setting a discrete action space.</div>

Complexity

tab3

Table 3

Table 3: Optimizing the Pairs-Trading Strategy Using Deep Reinforcement Learning with Trading and Stop-Loss Boundaries 